Tokenizer: Fix tokenizer invalid strings

5 files changed, 9 insertions, 5 deletions

diff --git a/src/main.zig b/src/main.zig
index e95bd48..bc88aeb 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -50,7 +50,7 @@ fn process_buf(buf: []u8, allocator: std.mem.Allocator, arena: *std.heap.ArenaAl
     defer token_list.deinit();
 
     var source_tokenizer = try tokenizer.Tokenizer.init(buf);
-    while (source_tokenizer.next()) |token| {
+    while (try source_tokenizer.next()) |token| {
         try token_list.append(token);
     }
 
diff --git a/src/tokenizer.zig b/src/tokenizer.zig
index 73a90e8..4353ebb 100644
--- a/src/tokenizer.zig
+++ b/src/tokenizer.zig
@@ -1,5 +1,9 @@
 const std = @import("std");
 
+const TokenizerError = error{
+    TokenizingError,
+};
+
 pub const Tokenizer = struct {
     buf: []u8,
     offset: u32,
@@ -8,7 +12,7 @@ pub const Tokenizer = struct {
         return Tokenizer{ .buf = buf, .offset = 0 };
     }
 
-    pub fn next(self: *Tokenizer) ?Token {
+    pub fn next(self: *Tokenizer) TokenizerError!?Token {
         defer self.offset += 1;
         self.skip_whitespace();
 
@@ -22,7 +26,7 @@ pub const Tokenizer = struct {
         if (c == '=') return Token{ .EQUALS = void{} };
 
         const string = self.consume_string();
-        self.offset -= 1;
+        if (string.len == 0) return TokenizerError.TokenizingError;
 
         if (std.mem.eql(u8, string, "let")) return Token{ .LET = void{} };
         if (std.mem.eql(u8, string, "print")) return Token{ .PRINT = void{} };
@@ -42,6 +46,7 @@ pub const Tokenizer = struct {
     }
 
     fn consume_string(self: *Tokenizer) []u8 {
+        defer self.offset = if (self.offset > 0) self.offset - 1 else self.offset;
         const start = self.offset;
         while (true) {
             if (self.offset >= self.buf.len) return self.buf[start..self.offset];
diff --git a/todo/1.src b/todo/1.src
deleted file mode 100644
index e17f230..0000000
--- a/todo/1.src
+++ /dev/null
@@ -1 +0,0 @@
-let x = -;
diff --git a/todo/2.src b/todo/2.src
deleted file mode 100644
index 31354ec..0000000
--- a/todo/2.src
+++ /dev/null
@@ -1 +0,0 @@
-_
diff --git a/todo/3.src b/todo/3.src
new file mode 100644
index 0000000..29ade8c
--- /dev/null
+++ b/todo/3.src
@@ -0,0 +1 @@
+let x = x;