Add tokenizer test

1 files changed, 64 insertions, 3 deletions

diff --git a/src/tokenizer.zig b/src/tokenizer.zig
index a69b4eb..f2b013e 100644
--- a/src/tokenizer.zig
+++ b/src/tokenizer.zig
@@ -1,3 +1,5 @@
+const std = @import("std");
+
 pub const Tokenizer = struct {
     buf: []u8,
     offset: u32,
@@ -7,10 +9,69 @@ pub const Tokenizer = struct {
     }
 
     pub fn next(_: *Tokenizer) ?Token {
-        return Token.HI;
+        return null;
+        // return Token{
+        //     .LET = void{},
+        // };
     }
 };
 
-pub const Token = enum {
-    HI,
+const TokenType = enum {
+    // Keywords
+    LET,
+    PRINT,
+
+    // Identifiers
+    IDENTIFIER,
+
+    // Literals
+    NUMBER,
+    STRING,
+
+    // Operators
+    EQUALS,
+
+    // Punctuation
+    SEMICOLON,
+    LPAREN,
+    RPAREN,
 };
+
+pub const Token = union(TokenType) {
+    LET: void,
+    PRINT: void,
+    IDENTIFIER: []u8,
+    NUMBER: i64,
+    STRING: []u8,
+    EQUALS: void,
+    SEMICOLON: void,
+    LPAREN: void,
+    RPAREN: void,
+};
+
+test "simple" {
+    const buf =
+        \\ let i = 2;
+        \\
+        \\ print(i);
+    ;
+
+    var tokenizer = try Tokenizer.init(@constCast(buf));
+
+    var token_list = std.ArrayList(Token).init(std.testing.allocator);
+    while (tokenizer.next()) |token| {
+        try token_list.append(token);
+    }
+    try std.testing.expectEqualSlices(Token, &.{
+        Token{ .LET = void{} },
+        Token{ .IDENTIFIER = @constCast("i") },
+        Token{ .EQUALS = void{} },
+        Token{ .NUMBER = 2 },
+        Token{ .SEMICOLON = void{} },
+        Token{ .PRINT = void{} },
+        Token{ .LPAREN = void{} },
+        Token{ .IDENTIFIER = @constCast("i") },
+        Token{ .RPAREN = void{} },
+        Token{ .SEMICOLON = void{} },
+    }, token_list.items);
+}