bootstrap: tokenizer: more changes

1 files changed, 46 insertions, 1 deletions

diff --git a/src/bootstrap/tokenizer.src b/src/bootstrap/tokenizer.src
index 1583a6d..5ac8948 100644
--- a/src/bootstrap/tokenizer.src
+++ b/src/bootstrap/tokenizer.src
@@ -7,6 +7,9 @@ let buf = 0;
 
 let offset = 0;
 
+let tokens = 0;
+let tokens_len = 0;
+
 let read_file = (filename: *i8) => *i8 {
 	file = fopen(filename, "r");
 
@@ -22,6 +25,40 @@ let read_file = (filename: *i8) => *i8 {
 	return buf;
 };
 
+let add_token = (tokens: *i8, token: *i8) => i64 {
+	let i = 0;
+	while true {
+		let c = (*(token + i));
+
+		(*(tokens + tokens_len)) = c;
+
+		tokens_len = tokens_len + 1;
+		i = i + 1;
+
+		if c == '\0' {
+			return 0;
+		};
+	};
+
+	return 0;
+};
+
+let print_tokens = (tokens: *i8) => i64 {
+	let i = 0;
+	while i < tokens_len {
+		let c = (*(tokens + i));
+		if c == '\0' {
+			c = '\n';
+		};
+
+		printf("%c", c);
+
+		i = i + 1;
+	};
+
+	return 0;
+};
+
 let tokenizer_next = () => *i8 {
 	if offset >= file_size {
 		return "EOF";
@@ -45,19 +82,27 @@ let tokenizer_init = (filename: *i8) => i64 {
 
 	println("%s", buf);
 
+	tokens = malloc(10000);
+
 	while true {
 		let t = tokenizer_next();
 		if strcmp(t, "EOF") {
-			return 0;
+			break;
 		};
 		println("%s", t);
+		add_token(tokens, t);
 		free(t);
 	};
 
+	println("PRINT TOKENS");
+
+	print_tokens(tokens);
+
 	return 0;
 };
 
 let tokenizer_deinit = () => i64 {
+	free(tokens);
 	free(buf);
 	fclose(file);