Tokenizer: Support newlines and tabs for strings

18 files changed, 47 insertions, 29 deletions

diff --git a/examples/0.src b/examples/0.src
index 8d0f7c1..db7a87f 100644
--- a/examples/0.src
+++ b/examples/0.src
@@ -2,7 +2,7 @@
 extern printf = (*i64, varargs) => i64;
 
 let main = (argc: i64, argv: *i64) => i64 {
-	printf("%d", argc);
+	printf("%d\n", argc);
 	
 	return 2;
 };
diff --git a/examples/1.5.src b/examples/1.5.src
index 4391639..17c9b35 100644
--- a/examples/1.5.src
+++ b/examples/1.5.src
@@ -1,14 +1,14 @@
 extern printf = (*i64, i64) => i64;
 
 let x = () => i64 {
-	printf("%d", 22);
+	printf("%d\n", 22);
 	return 11;
 };
 
 let main = () => i64 {
 	let i = 4;
 
-	printf("%d", i);
+	printf("%d\n", i);
 
 	return x();
 };
diff --git a/examples/1.src b/examples/1.src
index 05e0121..5307e50 100644
--- a/examples/1.src
+++ b/examples/1.src
@@ -3,7 +3,7 @@ extern printf = (*i64, varargs) => i64;
 let main = () => i64 {
 	let s = "hello";
 
-	printf("%s", s);
+	printf("%s\n", s);
 
 	return 0;
 };
diff --git a/examples/10.src b/examples/10.src
index b196846..a3abc39 100644
--- a/examples/10.src
+++ b/examples/10.src
@@ -4,7 +4,7 @@ let main = () => i64 {
 	let counter = 0;
 
 	while counter < 10 {
-		printf("%d", counter);
+		printf("%d\n", counter);
 		counter = counter + 1;
 	};
 
diff --git a/examples/11.src b/examples/11.src
index cfefad8..cc23719 100644
--- a/examples/11.src
+++ b/examples/11.src
@@ -6,8 +6,8 @@ let main = () => i64 {
 	};
 
 	let id = (a: i64) => i64 {
-		printf("%d", a);
-		printf("%d", 12);
+		printf("%d\n", a);
+		printf("%d\n", 12);
 		return a;
 	};
 
diff --git a/examples/12.src b/examples/12.src
index 3ecbaa3..1819335 100644
--- a/examples/12.src
+++ b/examples/12.src
@@ -31,10 +31,10 @@ let main = () => i64 {
     };
 
     let fact_val = factorial(6);
-    printf("%d", fact_val);
+    printf("%d\n", fact_val);
 
     let even_sum = sum_if(is_even, 20);
-    printf("%d", even_sum);
+    printf("%d\n", even_sum);
 
     return 0;
 };
diff --git a/examples/13.src b/examples/13.src
index 5bd0863..ed05a16 100644
--- a/examples/13.src
+++ b/examples/13.src
@@ -24,10 +24,10 @@ let main = () => i64 {
     };
 
     let fib_val = fibonacci_iter(10);
-    printf("%d", fib_val);
+    printf("%d\n", fib_val);
 
     let gcd_val = gcd(48, 18);
-    printf("%d", gcd_val);
+    printf("%d\n", gcd_val);
 
     return 0;
 };
diff --git a/examples/2.src b/examples/2.src
index bb5dca2..aef918e 100644
--- a/examples/2.src
+++ b/examples/2.src
@@ -5,7 +5,7 @@ let main = () => i64 {
 
 	let uwu = test;
 
-	printf("%d", uwu);
+	printf("%d\n", uwu);
 
 	return 0;
 };
diff --git a/examples/3.src b/examples/3.src
index 266dad5..fcd4b1b 100644
--- a/examples/3.src
+++ b/examples/3.src
@@ -5,7 +5,7 @@ let main = () => i64 {
 
 	seventeen = seventeen + 1;
 
-	printf("%d", seventeen);
+	printf("%d\n", seventeen);
 
 	return seventeen;
 };
diff --git a/examples/4.src b/examples/4.src
index 3ba2ef9..119fce1 100644
--- a/examples/4.src
+++ b/examples/4.src
@@ -2,13 +2,13 @@ extern printf = (*i64, varargs) => i64;
 
 let main = () => i64 {
 	let print_one = () => i64 {
-		printf("%d", 1);
+		printf("%d\n", 1);
 		return 4;
 	};
 
 	let y = print_one() + 2;
 
-	printf("%d", y);
+	printf("%d\n", y);
 
 	return y;
 };
diff --git a/examples/5.src b/examples/5.src
index f149e92..a8d341d 100644
--- a/examples/5.src
+++ b/examples/5.src
@@ -5,14 +5,14 @@ let x = 18;
 
 let foo = () => i64 {
 	let x = 1;
-	printf("%d", x);
+	printf("%d\n", x);
 	return x;
 };
 
 let main = () => i64 {
-	printf("%d", x);
+	printf("%d\n", x);
 	let x = 2;
 	let y = foo();
-	printf("%d", x);
+	printf("%d\n", x);
 	return x + y;
 };
diff --git a/examples/6.5.src b/examples/6.5.src
index fd645af..2820f70 100644
--- a/examples/6.5.src
+++ b/examples/6.5.src
@@ -3,7 +3,7 @@ extern printf = (*i64, varargs) => i64;
 let main = () => i64 {
 	let x = !(1 == 1);
 	if !x {
-		printf("%d", x);
+		printf("%d\n", x);
 	};
 	return 0;
 };
diff --git a/examples/6.src b/examples/6.src
index ccfa585..3c90471 100644
--- a/examples/6.src
+++ b/examples/6.src
@@ -1,8 +1,8 @@
 extern printf = (*i64, varargs) => i64;
 
 let print_input = (input_a: i64, input_b: i64) => i64 {
-	printf("%d", input_a);
-	printf("%d", input_b);
+	printf("%d\n", input_a);
+	printf("%d\n", input_b);
 	return input_a + input_b;
 };
 
diff --git a/examples/7.src b/examples/7.src
index e887d9c..e4db114 100644
--- a/examples/7.src
+++ b/examples/7.src
@@ -10,7 +10,7 @@ let main = () => i64 {
 	let i = 4;
 	
 	if (1 - -1 * 2) == 5 - (-1 + 1 + ten() / 2) + 3 {
-		printf("%d", i);
+		printf("%d\n", i);
 		return i;
 	};
 
diff --git a/examples/8.src b/examples/8.src
index 339dcba..012005c 100644
--- a/examples/8.src
+++ b/examples/8.src
@@ -12,6 +12,6 @@ let main = () => i64 {
 	};
 
 	let result = fib(30);
-	printf("%d", result);
+	printf("%d\n", result);
 	return result;
 };
diff --git a/examples/9.src b/examples/9.src
index 55b1797..dc7084b 100644
--- a/examples/9.src
+++ b/examples/9.src
@@ -1,7 +1,7 @@
 extern printf = (*i64, varargs) => i64;
 
 let print_boolean = (b: bool) => i64 {
-	printf("%d", b);
+	printf("%d\n", b);
 	return 0;
 };
 
diff --git a/src/main.zig b/src/main.zig
index cb32838..3ac7f39 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -68,7 +68,7 @@ fn process_buf(buf: []u8, allocator: std.mem.Allocator, arena: std.mem.Allocator
     var token_list = std.ArrayList(tokenizer.Token).init(allocator);
     defer token_list.deinit();
 
-    var source_tokenizer = try tokenizer.Tokenizer.init(buf);
+    var source_tokenizer = try tokenizer.Tokenizer.init(buf, arena);
     while (try source_tokenizer.next()) |token| {
         std.debug.print("{any}\n", .{token});
         try token_list.append(token);
diff --git a/src/tokenizer.zig b/src/tokenizer.zig
index b959738..1c88bf2 100644
--- a/src/tokenizer.zig
+++ b/src/tokenizer.zig
@@ -57,8 +57,10 @@ pub const Tokenizer = struct {
     buf: []u8,
     offset: u64,
 
-    pub fn init(buf: []u8) !Tokenizer {
-        return Tokenizer{ .buf = buf, .offset = 0 };
+    arena: std.mem.Allocator,
+
+    pub fn init(buf: []u8, arena: std.mem.Allocator) !Tokenizer {
+        return Tokenizer{ .buf = buf, .offset = 0, .arena = arena };
     }
 
     pub fn next(self: *Tokenizer) TokenizerError!?Token {
@@ -163,18 +165,34 @@ pub const Tokenizer = struct {
             return null;
         }
 
-        const res = self.consume_until_condition(struct {
+        const string = self.consume_until_condition(struct {
             fn condition(c: u8) bool {
                 return c == '"';
             }
         }.condition);
 
+        var res = std.ArrayList(u8).init(self.arena);
+
+        var i: usize = 0;
+        while (i < string.len) : (i += 1) {
+            if (string[i] == '\\') {
+                i += 1;
+                switch (string[i]) {
+                    'n' => res.append('\n') catch unreachable,
+                    't' => res.append('\t') catch unreachable,
+                    else => unreachable,
+                }
+                continue;
+            }
+            res.append(string[i]) catch unreachable;
+        }
+
         if (!self.accept_string("\"")) {
             self.offset = prev_offset;
             return null;
         }
 
-        return res;
+        return res.items;
     }
 
     fn create_token(self: *Tokenizer, token_type: TokenType) Token {