Tokenizer: Add support for tokenizing functions

1 files changed, 1 insertions, 4 deletions

diff --git a/src/main.zig b/src/main.zig
index bc88aeb..e7ebba3 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -51,11 +51,8 @@ fn process_buf(buf: []u8, allocator: std.mem.Allocator, arena: *std.heap.ArenaAl
 
     var source_tokenizer = try tokenizer.Tokenizer.init(buf);
     while (try source_tokenizer.next()) |token| {
-        try token_list.append(token);
-    }
-
-    for (token_list.items) |token| {
         std.debug.print("{any}\n", .{token});
+        try token_list.append(token);
     }
 
     const source_parser = try parser.Parser.init(token_list.items, arena.allocator());