tokenizer.lua

local function tokenize(program)
	program = string.gsub(program, "%-%-.-\n", "\n");
	program = string.gsub(program, "(%g+);", "%1 ;");
	coroutine.yield()
	for token in string.gmatch(program, "[a-z0-9_:;]+") do
		coroutine.yield(token)
	end
	return nil
end

local function get_tokenizer(program)
	tk = coroutine.wrap(tokenize)
	tk(program)
	return tk
end

function M.parse(program)
	local tokenizer = get_tokenizer(program)
	for tok in tokenizer do
		print(tok)
	end
end