From 26529af9bd58623ed6b2ee59dc2dc3659a960555 Mon Sep 17 00:00:00 2001
From: Manuel Palenzuela Merino <manuel.palenzuela@datadoghq.com>
Date: Thu, 16 Jan 2025 22:33:02 +0100
Subject: Tokenizer: Add support for comments

---
 examples/0.src    | 2 ++
 src/tokenizer.zig | 9 +++++++++
 2 files changed, 11 insertions(+)

diff --git a/examples/0.src b/examples/0.src
index 8b21f41..d73769a 100644
--- a/examples/0.src
+++ b/examples/0.src
@@ -1,3 +1,5 @@
+/* HELLO! Welcome to the unnamed language */
+
 let main = () => {
 	print(2);
 	
diff --git a/src/tokenizer.zig b/src/tokenizer.zig
index 6513978..4eb807a 100644
--- a/src/tokenizer.zig
+++ b/src/tokenizer.zig
@@ -55,6 +55,7 @@ pub const Tokenizer = struct {
 
     pub fn next(self: *Tokenizer) TokenizerError!?Token {
         defer self.offset += 1;
+        self.skip_comments();
         self.skip_whitespace();
 
         if (self.offset >= self.buf.len) return null;
@@ -82,6 +83,14 @@ pub const Tokenizer = struct {
         return Token{ .IDENTIFIER = string };
     }
 
+    fn skip_comments(self: *Tokenizer) void {
+        if (!self.accept_substr("/*")) return;
+
+        while (!self.accept_substr("*/")) {
+            self.offset += 1;
+        }
+    }
+
     fn skip_whitespace(self: *Tokenizer) void {
         while (true) {
             if (self.offset >= self.buf.len) return;
-- 
cgit 1.4.1