extern strlen = (*i8) => i64; extern memcpy = (*i8, *i8, i64) => void; extern sprintf = (*i8, *i8, varargs) => void; extern atoi = (*i8) => i64; import "!stdlib.src"; let file_size = 0; let file = 0; let buf = 0; let offset = 0; let tokens = 0; let tokens_len = 0; let read_file = (filename: *i8) => *i8 { file = fopen(filename, "r"); fseek(file, 0, 2); file_size = ftell(file); fseek(file, 0, 0); buf = malloc(file_size + 1); let bytes_read = fread(buf, 1, file_size, file); (*(buf + bytes_read)) = '\0'; return buf; }; let add_token = (tokens: *i8, token: *i8) => i64 { printf("Add token: %s\n", token); let i = 0; while true { let c = (*(token + i)); (*(tokens + tokens_len)) = c; tokens_len = tokens_len + 1; i = i + 1; if c == '\0' { return 0; }; }; return 0; }; let print_tokens = (tokens: *i8) => i64 { let i = 0; while i < tokens_len { let c = (*(tokens + i)); if c == '\0' { c = '\n'; }; printf("%c", c); i = i + 1; }; return 0; }; let tokenizer_skip_whitespace = () => void { while true { if offset >= file_size { return; }; let c = (*(buf + offset)); if !iswhitespace(c) { return; }; offset = offset + 1; }; return; }; let tokenizer_accept_string = (str: *i8) => bool { let str_len = strlen(str); if offset + str_len > file_size { return false; }; let s = malloc(1000); memcpy(s, buf + offset, str_len); printf("Accept string: %s vs %s\n", str, s); if strcmp(s, str) { offset = offset + str_len; return true; }; return false; }; let tokenizer_consume_until_condition = (condition: (i8) => bool) => *i8 { let start = offset; let res = malloc(1000); while true { memcpy(res, buf + start, offset); (*(res + (offset - start))) = '\0'; if offset >= file_size { return res; }; let c = (*(buf + offset)); /* TODO: calling condition breaks */ if !isdigit(c) { return res; }; offset = offset + 1; }; return null; }; let isnt_digit = (c: i8) => bool { return !isdigit(c); }; let tokenizer_accept_int_type = () => *i64 { let res = tokenizer_consume_until_condition(isnt_digit); if res == null { return null; }; if strlen(res) == 0 { return null; }; let x = malloc(8); *x = atoi(res); return x; }; let tokenizer_skip_comments = () => void { if !tokenizer_accept_string("/*") { return; }; while !tokenizer_accept_string("*/") { offset = offset + 1; }; return; }; let tokenizer_next = () => *i8 { tokenizer_skip_whitespace(); tokenizer_skip_comments(); tokenizer_skip_whitespace(); if offset >= file_size { return "EOF"; }; if tokenizer_accept_string("import") { return "import"; }; if tokenizer_accept_string("let") { return "let"; }; if tokenizer_accept_string("extern") { return "extern"; }; if tokenizer_accept_string("if") { return "if"; }; if tokenizer_accept_string("while") { return "while"; }; if tokenizer_accept_string("return") { return "return"; }; if tokenizer_accept_string("break") { return "break"; }; if tokenizer_accept_string("true") { return "bool:true"; }; if tokenizer_accept_string("false") { return "bool:false"; }; if tokenizer_accept_string("=>") { return "=>"; }; if tokenizer_accept_string(";") { return ";"; }; if tokenizer_accept_string(",") { return ","; }; if tokenizer_accept_string(":") { return ":"; }; if tokenizer_accept_string("(") { return "("; }; if tokenizer_accept_string(")") { return ")"; }; if tokenizer_accept_string("{") { return "{"; }; if tokenizer_accept_string("}") { return "}"; }; if tokenizer_accept_string("=") { return "="; }; if tokenizer_accept_string("+") { return "+"; }; if tokenizer_accept_string("-") { return "-"; }; if tokenizer_accept_string("*") { return "*"; }; if tokenizer_accept_string("/") { return "/"; }; if tokenizer_accept_string("%") { return "%"; }; if tokenizer_accept_string("!") { return "!"; }; if tokenizer_accept_string("<") { return "<"; }; if tokenizer_accept_string(">") { return ">"; }; let maybe_int = tokenizer_accept_int_type(); if !(maybe_int == null) { let t = malloc(1000); sprintf(t, "int:%d", *maybe_int); return t; }; let c = (*(buf + offset)); offset = offset + 1; let t = malloc(13); memcpy(t, "identifier:", 11); (*(t + 11)) = c; (*(t + 12)) = '\0'; return t; }; let tokenizer_init = (filename: *i8) => i64 { let buf = read_file(filename); printf("File size: %d\n", file_size); printf("%s\n", buf); tokens = malloc(10000); while true { let t = tokenizer_next(); if strcmp(t, "EOF") { break; }; add_token(tokens, t); }; printf("PRINT TOKENS\n"); print_tokens(tokens); return 0; }; let tokenizer_deinit = () => i64 { free(tokens); free(buf); fclose(file); return 0; };