tokenizer.odin

package tokenizer

import "core:text/scanner"
import "core:unicode"
import "core:strings"
import "core:slice"

Tokenizer :: struct {
	src: string,
}

next_token :: proc(using tokenizer: ^Tokenizer) -> string {
	if len(src) == 0 do return "EOF"

	// skip whitespace
	for {
		character := rune(src[0])
		if character not_in scanner.C_Whitespace do break
		src = src[1:]
		if len(src) == 0 do return "EOF"
	}

	// block comment
	if src[0] == '/' && src[1] == '*' {
		src = src[2:]
		for {
			if src[0] == '*' && src[1] == '/' {
				src = src[2:]
				return "block comment"
			}
			src = src[1:]
			if len(src) <= 1 do return "EOF"
		}
	}

	// comment
	if src[0] == '/' && src[1] == '/' {
		src = src[2:]
		for {
			if src[0] == '\n' {
				src = src[1:]
				return "comment"
			}
			src = src[1:]
			if len(src) == 0 do return "EOF"
		}
	}

	if slice.contains(SYMBOLS, strings.clone_from_bytes([]byte{src[0]}, context.temp_allocator)) {
		if slice.contains(SYMBOLS, src[:2]) {
			s := src[:2]
			src = src[2:]
			return s
		}
		else {
			s := strings.clone_from_bytes([]byte{src[0]}, context.temp_allocator)
			src = src[1:]
			return s
		}
	}

	builder: strings.Builder
	strings.init_builder(&builder, context.temp_allocator)
	for {
		strings.write_byte(&builder, src[0])
		src = src[1:]

		if rune(src[0]) in scanner.C_Whitespace do break

		if slice.contains(SYMBOLS, strings.clone_from_bytes([]byte{src[0]}, context.temp_allocator)) || slice.contains(SYMBOLS, src[:2]) do break

		if len(src) == 0 do return "EOF"
	}

	return strings.to_string(builder)
}