Tokenizer: Add support for comments

2 files changed, 11 insertions, 0 deletions

diff --git a/examples/0.src b/examples/0.src
index 8b21f41..d73769a 100644
--- a/examples/0.src
+++ b/examples/0.src
@@ -1,3 +1,5 @@
+/* HELLO! Welcome to the unnamed language */
+
 let main = () => {
 	print(2);
 	
diff --git a/src/tokenizer.zig b/src/tokenizer.zig
index 6513978..4eb807a 100644
--- a/src/tokenizer.zig
+++ b/src/tokenizer.zig
@@ -55,6 +55,7 @@ pub const Tokenizer = struct {
 
     pub fn next(self: *Tokenizer) TokenizerError!?Token {
         defer self.offset += 1;
+        self.skip_comments();
         self.skip_whitespace();
 
         if (self.offset >= self.buf.len) return null;
@@ -82,6 +83,14 @@ pub const Tokenizer = struct {
         return Token{ .IDENTIFIER = string };
     }
 
+    fn skip_comments(self: *Tokenizer) void {
+        if (!self.accept_substr("/*")) return;
+
+        while (!self.accept_substr("*/")) {
+            self.offset += 1;
+        }
+    }
+
     fn skip_whitespace(self: *Tokenizer) void {
         while (true) {
             if (self.offset >= self.buf.len) return;