From de594ada98e053d3c2bdd4e17ad8b6e2aa6505a2 Mon Sep 17 00:00:00 2001 From: Baitinq Date: Mon, 13 Jan 2025 00:33:56 +0100 Subject: Tokenizer: Add support for tokenizing functions --- src/main.zig | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'src/main.zig') diff --git a/src/main.zig b/src/main.zig index bc88aeb..e7ebba3 100644 --- a/src/main.zig +++ b/src/main.zig @@ -51,11 +51,8 @@ fn process_buf(buf: []u8, allocator: std.mem.Allocator, arena: *std.heap.ArenaAl var source_tokenizer = try tokenizer.Tokenizer.init(buf); while (try source_tokenizer.next()) |token| { - try token_list.append(token); - } - - for (token_list.items) |token| { std.debug.print("{any}\n", .{token}); + try token_list.append(token); } const source_parser = try parser.Parser.init(token_list.items, arena.allocator()); -- cgit 1.4.1