tokenizer.zig

const std = @import("std");
const ArrayList = std.ArrayList;
const Allocator = std.mem.Allocator;
const TestAllocator = std.testing.allocator;
const t = std.testing;

const Token = struct {
    tag: Tag,
    value: ?[]const u8 = null,

    const Tag = enum {
        lpar,
        rpar,
        sym,
    };
};

const Tokenizer = struct {
    index: u64 = 0,
    allocator: Allocator,

    pub fn init(allocator: Allocator) Tokenizer {
        return Tokenizer{ .allocator = allocator };
    }

    pub fn scan(self: *Tokenizer, input: [:0]const u8) !ArrayList(Token) {
        var tokens = ArrayList(Token).init(self.allocator);
        var pos: usize = 0;
        while (pos < input.len) : (pos += 1) {
            var char: u8 = input[pos];
            const tk = switch (char) {
                '(' => Token{ .tag = .lpar },
                ')' => Token{ .tag = .rpar },
                ' ' => continue,
                'A'...'Z', 'a'...'z' => blk: {
                    while (true) {
                        switch (input[pos + 1]) {
                            'A'...'Z', 'a'...'z', '0'...'9' => {
                                pos += 1;
                            },
                            else => break,
                        }
                    }
                    break :blk Token{ .tag = .sym, .value = input[self.index + 1 .. pos + 1] };
                },
                else => unreachable,
            };
            self.index = pos;
            try tokens.append(tk);
        }
        return tokens;
    }
};

test "Tokenize example program" {
    const program = "(x)";

    var tokenizer = Tokenizer.init(TestAllocator);
    var tokens = try tokenizer.scan(program);
    defer tokens.deinit();

    const expected = [_]Token{ Token{ .tag = .lpar }, Token{ .tag = .sym, .value = "x" }, Token{ .tag = .rpar } };

    // Fails
    // try t.expectEqualSlices(Token, &expected, tokens.items);

    // Fails
    // try t.expect(std.meta.eql(expected[1].value.?, tokens.items[1].value.?));

    // Works
    try t.expectEqualDeep(expected[1], tokens.items[1]);

    // Works
    // for (expected[1].value.?, tokens.items[1].value.?) |e, a| {
    //     try t.expect(e == a);
    // }
}