Bootstrap: Tokenizer: Tokenize ints

3 files changed, 63 insertions, 6 deletions

diff --git a/src/bootstrap/tokenizer.src b/src/bootstrap/tokenizer.src
index ffc7e96..3c420d5 100644
--- a/src/bootstrap/tokenizer.src
+++ b/src/bootstrap/tokenizer.src
@@ -1,5 +1,7 @@
 extern strlen = (*i8) => i64;
 extern memcpy = (*i8, *i8, i64) => void;
+extern sprintf = (*i8, *i8, varargs) => void;
+extern atoi = (*i8) => i64;
 
 import "!stdlib.src";
 
@@ -67,7 +69,6 @@ let tokenizer_skip_whitespace = () => void {
 	while true {
 		if offset >= file_size { return; };
 		let c = (*(buf + offset));
-		printf("C: %c\n", c);
 		if !iswhitespace(c) {
 			return;
 		};
@@ -84,7 +85,7 @@ let tokenizer_accept_string = (str: *i8) => bool {
 	let s = malloc(1000);
 	memcpy(s, buf + offset, str_len);
 
-	printf("Accept string: %s\n", s);
+	printf("Accept string: %s vs %s\n", str, s);
 	if strcmp(s, str) {
 		offset = offset + str_len;
 		return true;
@@ -93,6 +94,46 @@ let tokenizer_accept_string = (str: *i8) => bool {
 	return false;
 };
 
+let tokenizer_consume_until_condition = (condition: (i8) => bool) => *i8 {
+	let start = offset;
+	let res = malloc(1000);
+
+	while true {
+		memcpy(res, buf + start, offset);
+		(*(res + (offset - start))) = '\0';
+
+		if offset >= file_size {
+			return res;
+		};
+
+		let c = (*(buf + offset));
+		/* TODO: calling condition breaks */
+		if !isdigit(c) {
+			return res;
+		};
+		offset = offset + 1;
+	};
+
+	return null;
+};
+
+let isnt_digit = (c: i8) => bool {
+	return !isdigit(c);
+};
+
+let tokenizer_accept_int_type = () => *i64 {
+	let res = tokenizer_consume_until_condition(isnt_digit);
+	if res == null {
+		return null;
+	};
+	if strlen(res) == 0 {
+		return null;
+	};
+	let x = malloc(8);
+	*x = atoi(res);
+	return x;
+};
+
 let tokenizer_skip_comments = () => void {
 	if !tokenizer_accept_string("/*") { return; };
 
@@ -191,6 +232,14 @@ let tokenizer_next = () => *i8 {
 	if tokenizer_accept_string(">") {
 	    return ">";
 	};
+	
+	let maybe_int = tokenizer_accept_int_type();
+	if !(maybe_int == null) {
+		let t = malloc(1000);
+		sprintf(t, "int:%d", *maybe_int);
+
+		return t;
+	};
 
 	let c = (*(buf + offset));
 
diff --git a/src/codegen.zig b/src/codegen.zig
index 05cb877..1d2e877 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -655,6 +655,7 @@ pub const CodeGen = struct {
                 if (std.mem.eql(u8, t.name, "bool")) return llvm.LLVMInt1Type();
                 if (std.mem.eql(u8, t.name, "void")) return llvm.LLVMVoidType();
                 if (std.mem.eql(u8, t.name, "varargs")) return llvm.LLVMPointerType(llvm.LLVMInt64Type(), 0); // Hack for varargs (only used for printf)
+                std.debug.print("Unknown type: {s}\n", .{t.name});
                 unreachable;
             },
             .FUNCTION_TYPE => |t| {
diff --git a/std/stdlib.src b/std/stdlib.src
index 133ed27..91884d5 100644
--- a/std/stdlib.src
+++ b/std/stdlib.src
@@ -31,6 +31,15 @@ let strcmp = (stra: *i8, strb: *i8) => bool {
 	return true;
 };
 
+let isdigit = (c: i8) => bool {
+	if c > '0' {
+		if c < '9' {
+			return true;
+		};
+	};
+	return false;
+};
+
 let isalphanum = (c: i8) => bool {
 	if c > 'a' {
 		if c < 'z' {
@@ -42,10 +51,8 @@ let isalphanum = (c: i8) => bool {
 			return true;
 		};
 	};
-	if c > '0' {
-		if c < '9' {
-			return true;
-		};
+	if isdigit(c) {
+		return true;
 	};
 
 	return false;