Tokenizer: Support newlines and tabs for strings

2 files changed, 23 insertions, 5 deletions

diff --git a/src/main.zig b/src/main.zig
index cb32838..3ac7f39 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -68,7 +68,7 @@ fn process_buf(buf: []u8, allocator: std.mem.Allocator, arena: std.mem.Allocator
     var token_list = std.ArrayList(tokenizer.Token).init(allocator);
     defer token_list.deinit();
 
-    var source_tokenizer = try tokenizer.Tokenizer.init(buf);
+    var source_tokenizer = try tokenizer.Tokenizer.init(buf, arena);
     while (try source_tokenizer.next()) |token| {
         std.debug.print("{any}\n", .{token});
         try token_list.append(token);
diff --git a/src/tokenizer.zig b/src/tokenizer.zig
index b959738..1c88bf2 100644
--- a/src/tokenizer.zig
+++ b/src/tokenizer.zig
@@ -57,8 +57,10 @@ pub const Tokenizer = struct {
     buf: []u8,
     offset: u64,
 
-    pub fn init(buf: []u8) !Tokenizer {
-        return Tokenizer{ .buf = buf, .offset = 0 };
+    arena: std.mem.Allocator,
+
+    pub fn init(buf: []u8, arena: std.mem.Allocator) !Tokenizer {
+        return Tokenizer{ .buf = buf, .offset = 0, .arena = arena };
     }
 
     pub fn next(self: *Tokenizer) TokenizerError!?Token {
@@ -163,18 +165,34 @@ pub const Tokenizer = struct {
             return null;
         }
 
-        const res = self.consume_until_condition(struct {
+        const string = self.consume_until_condition(struct {
             fn condition(c: u8) bool {
                 return c == '"';
             }
         }.condition);
 
+        var res = std.ArrayList(u8).init(self.arena);
+
+        var i: usize = 0;
+        while (i < string.len) : (i += 1) {
+            if (string[i] == '\\') {
+                i += 1;
+                switch (string[i]) {
+                    'n' => res.append('\n') catch unreachable,
+                    't' => res.append('\t') catch unreachable,
+                    else => unreachable,
+                }
+                continue;
+            }
+            res.append(string[i]) catch unreachable;
+        }
+
         if (!self.accept_string("\"")) {
             self.offset = prev_offset;
             return null;
         }
 
-        return res;
+        return res.items;
     }
 
     fn create_token(self: *Tokenizer, token_type: TokenType) Token {