Tokenizer: Add support for tokenizing functions

3 files changed, 27 insertions, 5 deletions

diff --git a/examples/5.src b/examples/5.src
new file mode 100644
index 0000000..3582794
--- /dev/null
+++ b/examples/5.src
@@ -0,0 +1,7 @@
+let printOne = () => {
+	print(1);
+};
+
+printOne();
+
+return 0;
diff --git a/src/main.zig b/src/main.zig
index bc88aeb..e7ebba3 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -51,11 +51,8 @@ fn process_buf(buf: []u8, allocator: std.mem.Allocator, arena: *std.heap.ArenaAl
 
     var source_tokenizer = try tokenizer.Tokenizer.init(buf);
     while (try source_tokenizer.next()) |token| {
-        try token_list.append(token);
-    }
-
-    for (token_list.items) |token| {
         std.debug.print("{any}\n", .{token});
+        try token_list.append(token);
     }
 
     const source_parser = try parser.Parser.init(token_list.items, arena.allocator());
diff --git a/src/tokenizer.zig b/src/tokenizer.zig
index aab8aa2..17833e0 100644
--- a/src/tokenizer.zig
+++ b/src/tokenizer.zig
@@ -9,6 +9,7 @@ pub const TokenType = enum {
     LET,
     PRINT,
     RETURN,
+    ARROW,
 
     // Identifiers
     IDENTIFIER,
@@ -24,12 +25,15 @@ pub const TokenType = enum {
     SEMICOLON,
     LPAREN,
     RPAREN,
+    LBRACE,
+    RBRACE,
 };
 
 pub const Token = union(TokenType) {
     LET: void,
     PRINT: void,
     RETURN: void,
+    ARROW: void,
     IDENTIFIER: []u8,
     NUMBER: i64,
     EQUALS: void,
@@ -37,11 +41,13 @@ pub const Token = union(TokenType) {
     SEMICOLON: void,
     LPAREN: void,
     RPAREN: void,
+    LBRACE: void,
+    RBRACE: void,
 };
 
 pub const Tokenizer = struct {
     buf: []u8,
-    offset: u32,
+    offset: u64,
 
     pub fn init(buf: []u8) !Tokenizer {
         return Tokenizer{ .buf = buf, .offset = 0 };
@@ -55,9 +61,12 @@ pub const Tokenizer = struct {
 
         const c = self.buf[self.offset];
 
+        if (self.accept_substr("=>")) return Token{ .ARROW = void{} };
         if (c == ';') return Token{ .SEMICOLON = void{} };
         if (c == '(') return Token{ .LPAREN = void{} };
         if (c == ')') return Token{ .RPAREN = void{} };
+        if (c == '{') return Token{ .LBRACE = void{} };
+        if (c == '}') return Token{ .RBRACE = void{} };
         if (c == '=') return Token{ .EQUALS = void{} };
         if (c == '+') return Token{ .PLUS = void{} };
 
@@ -95,6 +104,15 @@ pub const Tokenizer = struct {
             self.offset += 1;
         }
     }
+
+    fn accept_substr(self: *Tokenizer, substr: []const u8) bool {
+        if (self.offset + substr.len > self.buf.len) return false;
+        if (std.mem.eql(u8, self.buf[self.offset .. self.offset + substr.len], substr)) {
+            self.offset += substr.len;
+            return true;
+        }
+        return false;
+    }
 };
 
 test "simple" {