diff options
| -rw-r--r-- | src/bootstrap/main.src | 39 | ||||
| -rw-r--r-- | src/bootstrap/tokenizer.src | 89 |
2 files changed, 67 insertions, 61 deletions
diff --git a/src/bootstrap/main.src b/src/bootstrap/main.src index bb840b5..c8338fb 100644 --- a/src/bootstrap/main.src +++ b/src/bootstrap/main.src @@ -1,8 +1,41 @@ +extern fopen = (*i8, *i8) => *i8; +extern fgets = (*i8, i64, *i8) => void; +extern feof = (*i8) => bool; +extern fseek = (*i8, i64, i64) => i64; +extern ftell = (*i8) => i64; +extern fread = (*i8, i64, i64, *i8) => i64; +extern fclose = (*i8) => *i8; + import "!stdlib.src"; import "!mem.src"; +let slice = struct { + data: *void, + data_len: i64, +}; + import "tokenizer.src"; +let read_file = (filename: *i8, alloc: *arena) => slice { + let file = fopen(filename, "r"); + + fseek(file, 0, 2); + let file_size = ftell(file); + fseek(file, 0, 0); + + let buf = cast(*i8, arena_alloc(alloc, file_size + 1)); + + let bytes_read = fread(buf, 1, file_size, file); + (*(buf + cast(*i8, bytes_read))) = '\0'; + + fclose(file); + + let sl = slice{}; + sl.data = cast(*void, buf); + sl.data_len = file_size; + return sl; +}; + let main = (argc: i64, argv: **i8) => i64 { if argc < 2 { println("Need filename!"); @@ -15,8 +48,10 @@ let main = (argc: i64, argv: **i8) => i64 { let alloc = arena_init(999999999); - tokenizer_init(alloc, filename); - tokenizer_deinit(); + let file = read_file(filename, alloc); + + let t = tokenizer_init(alloc, file); + let ts = tokenizer_tokenize(t); arena_free(alloc); diff --git a/src/bootstrap/tokenizer.src b/src/bootstrap/tokenizer.src index 72c335e..3cbb6c1 100644 --- a/src/bootstrap/tokenizer.src +++ b/src/bootstrap/tokenizer.src @@ -2,13 +2,6 @@ extern strlen = (*i8) => i64; extern memcpy = (*void, *void, i64) => void; extern sprintf = (*i8, *i8, varargs) => void; extern atoi = (*i8) => i64; -extern fopen = (*i8, *i8) => *i8; -extern fgets = (*i8, i64, *i8) => void; -extern feof = (*i8) => bool; -extern fseek = (*i8, i64, i64) => i64; -extern ftell = (*i8) => i64; -extern fread = (*i8, i64, i64, *i8) => i64; -extern fclose = (*i8) => *i8; import "!stdlib.src"; import "!mem.src"; @@ -63,46 +56,16 @@ let token = struct { let tokenizer = struct { buf: *i8, - file_size: i64, + buf_len: i64, offset: i64, arena: *arena, - tokens: *token, - tokens_len: i64, }; -let read_file = (t: *tokenizer, filename: *i8) => void { - let file = fopen(filename, "r"); - - fseek(file, 0, 2); - (*t).file_size = ftell(file); - fseek(file, 0, 0); - - let buf = cast(*i8, arena_alloc((*t).arena, (*t).file_size + 1)); - - let bytes_read = fread(buf, 1, (*t).file_size, file); - (*(buf + cast(*i8, bytes_read))) = '\0'; - - fclose(file); - - (*t).buf = buf; - - return; -}; - -let add_token = (t: *tokenizer, to: *token) => void { - println("Add token: %d", (*to).type); - - (*((*t).tokens + cast(*token, (*t).tokens_len))) = *to; - (*t).tokens_len = (*t).tokens_len + 1; - - return; -}; - -let print_tokens = (t: *tokenizer) => i64 { +let print_tokens = (ts: *token, ts_len: i64) => i64 { let i = 0; - while i < (*t).tokens_len { - let to = (*((*t).tokens + cast(*token, i))); + while i < ts_len { + let to = (*(ts + cast(*token, i))); if (to.type == TOKEN_IMPORT) { printf("Import\n"); @@ -142,6 +105,7 @@ let print_tokens = (t: *tokenizer) => i64 { }; if (to.type == TOKEN_BOOLEAN) { printf("Boolean: %d\n", cast(i1, to.data)); + /* TODO: FIX */ }; if (to.type == TOKEN_NULL) { printf("Null\n"); @@ -212,7 +176,7 @@ let print_tokens = (t: *tokenizer) => i64 { let tokenizer_skip_whitespace = (t: *tokenizer) => void { while true { - if (*t).offset >= (*t).file_size { return; }; + if (*t).offset >= (*t).buf_len { return; }; let c = (*((*t).buf + cast(*i8, (*t).offset))); if !iswhitespace(c) { return; @@ -225,7 +189,7 @@ let tokenizer_skip_whitespace = (t: *tokenizer) => void { let tokenizer_accept_string = (t: *tokenizer, str: *i8) => bool { let str_len = strlen(str); - if (*t).offset + str_len > (*t).file_size { return false; }; + if (*t).offset + str_len > (*t).buf_len { return false; }; let s = cast(*i8, arena_alloc((*t).arena, 1000)); memcpy(cast(*void, s), cast(*void, (*t).buf + cast(*i8, (*t).offset)), str_len); @@ -243,7 +207,7 @@ let tokenizer_consume_until_condition = (t: *tokenizer, condition: (i8) => bool) let res = cast(*i8, arena_alloc((*t).arena, 1000)); while true { - if (*t).offset >= (*t).file_size { + if (*t).offset >= (*t).buf_len { return res; }; @@ -371,7 +335,7 @@ let tokenizer_next = (t: *tokenizer) => *token { tokenizer_skip_comments(t); tokenizer_skip_whitespace(t); - if (*t).offset >= (*t).file_size { + if (*t).offset >= (*t).buf_len { return cast(*token, null); }; @@ -546,34 +510,41 @@ let tokenizer_next = (t: *tokenizer) => *token { return to; }; -let tokenizer_init = (alloc: *arena, filename: *i8) => i64 { +let tokenizer_init = (alloc: *arena, file: slice) => *tokenizer { let t = cast(*tokenizer, arena_alloc(alloc, sizeof(tokenizer))); (*t).arena = alloc; - (*t).tokens = cast(*token, arena_alloc((*t).arena, sizeof(token) * 1000)); /* why does it not care about type here */ - (*t).tokens_len = 0; (*t).offset = 0; + (*t).buf = file.data; + (*t).buf_len = file.data_len; - read_file(t, filename); - - println("File size: %d", (*t).file_size); + println("File size: %d", (*t).buf_len); println("%s", (*t).buf); + return t; +}; + +let tokenizer_tokenize = (t: *tokenizer) => slice { + let tokens = cast(*token, arena_alloc((*t).arena, sizeof(token) * 1000)); /* why does it not care about type here */ + let tokens_len = 0; + while true { let tk = tokenizer_next(t); if tk == cast(*token, null) { break; }; - add_token(t, tk); + println("Add token: %d", (*tk).type); + + (*(tokens + cast(*token, tokens_len))) = *tk; + tokens_len = tokens_len + 1; }; - println("PRINT TOKENS"); + println("PRINT TOKENS: %d", tokens_len); - print_tokens(t); + print_tokens(tokens, tokens_len); - return 0; -}; - -let tokenizer_deinit = () => i64 { - return 0; + let res = slice{}; + res.data = tokens; + res.data_len = tokens_len; + return res; }; |