all repos — pba-zig @ main

Protobufs But Awesomer as rewritten in Zig

src/lex.zig (view raw)

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 
const std = @import("std");
const fail = @import("fail.zig");

pub const Position = struct {
    line: usize = 0,
    column: usize = 0,
};

pub const Token = struct {
    pos: Position,
    tok: union(enum) {
        identifier: []const u8,
        scalar: usize,
        lbrace: void,
        rbrace: void,
        lparen: void,
        rparen: void,
        lbracket: void,
        rbracket: void,
        colon: void,
        comma: void,
        bang: void,
        eof: void,
    },

    pub fn describe(self: Token) []const u8 {
        return switch (self.tok) {
            .identifier => "identifier",
            .scalar => "scalar",
            .lbrace => "left brace",
            .rbrace => "right brace",
            .lparen => "left parenthesis",
            .rparen => "right parenthesis",
            .lbracket => "left bracket",
            .rbracket => "right bracket",
            .bang => "exclamation point",
            .colon => "colon",
            .eof => "end of file",
        };
    }
};

fn is_ident_start(c: u8) bool {
    return std.ascii.isAlphabetic(c) or (c == '_');
}

fn is_ident_continue(c: u8) bool {
    return std.ascii.isAlphanumeric(c) or (c == '_');
}

pub fn tokenize(buf: []const u8, alloc: std.mem.Allocator) !std.ArrayList(Token) {
    var res = std.ArrayList(Token).empty;
    var current_pos: Position = .{ .line = 1, .column = 1 };
    var current_idx: usize = 0;
    while (current_idx < buf.len) {
        if (is_ident_start(buf[current_idx])) {
            const start = current_idx;
            while (current_idx < buf.len and is_ident_continue(buf[current_idx])) {
                current_idx += 1;
            }
            try res.append(alloc, .{
                .tok = .{
                    .identifier = buf[start..current_idx],
                },
                .pos = current_pos,
            });
            current_pos.column += current_idx - start;
        } else if (std.ascii.isDigit(buf[current_idx])) {
            const start = current_idx;
            while (current_idx < buf.len and std.ascii.isDigit(buf[current_idx])) {
                current_idx += 1;
            }
            try res.append(alloc, .{
                .tok = .{
                    .scalar = try std.fmt.parseInt(usize, buf[start..current_idx], 10),
                },
                .pos = current_pos,
            });
            current_pos.column += current_idx - start;
        } else if (buf[current_idx] == '\n') {
            current_idx += 1;
            current_pos.line += 1;
            current_pos.column = 1;
        } else if (std.ascii.isWhitespace(buf[current_idx])) {
            current_idx += 1;
            current_pos.column += 1;
        } else if (buf[current_idx] == '#') {
            while (current_idx < buf.len and buf[current_idx] != '\n') {
                current_idx += 1;
                current_pos.column += 1;
            }
        } else {
            switch (buf[current_idx]) {
                '{' => try res.append(alloc, .{ .tok = .lbrace, .pos = current_pos }),
                '}' => try res.append(alloc, .{ .tok = .rbrace, .pos = current_pos }),
                '(' => try res.append(alloc, .{ .tok = .lparen, .pos = current_pos }),
                ')' => try res.append(alloc, .{ .tok = .rparen, .pos = current_pos }),
                '[' => try res.append(alloc, .{ .tok = .lbracket, .pos = current_pos }),
                ']' => try res.append(alloc, .{ .tok = .rbracket, .pos = current_pos }),
                ':' => try res.append(alloc, .{ .tok = .colon, .pos = current_pos }),
                ',' => try res.append(alloc, .{ .tok = .comma, .pos = current_pos }),
                '!' => try res.append(alloc, .{ .tok = .bang, .pos = current_pos }),
                else => fail.fail(current_pos, "invalid token"),
            }
            current_idx += 1;
            current_pos.column += 1;
        }
    }
    try res.append(alloc, .{ .tok = .eof, .pos = current_pos });
    return res;
}