src/bootstrap/tokenizer.src


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

import "!stdlib.src";

let file_size = 0;
let file = 0;
let buf = 0;


let offset = 0;

let read_file = (filename: *i8) => *i8 {
	file = fopen(filename, "r");

	fseek(file, 0, 2);
	file_size = ftell(file);
	fseek(file, 0, 0);

	buf = malloc(file_size + 1);

	let bytes_read = fread(buf, 1, file_size, file);
	(*(buf + bytes_read)) = '\0';

	return buf;
};

let tokenizer_next = () => *i8 {
	if offset >= file_size {
		return "EOF";
	};

	let c = (*(buf + offset));

	offset = offset + 1;

	let t = malloc(2);
	(*(t + 0)) = c;
	(*(t + 1)) = '\0';

	return t;
};

let tokenizer_init = (filename: *i8) => i64 {
	let buf = read_file(filename);

	println("File size: %d", file_size);

	println("%s", buf);

	while true {
		let t = tokenizer_next();
		if strcmp(t, "EOF") {
			return 0;
		};
		println("%s", t);
		free(t);
	};

	return 0;
};

let tokenizer_deinit = () => i64 {
	free(buf);
	fclose(file);

	return 0;
};