Skip to content

Instantly share code, notes, and snippets.

@katsaii
Last active February 7, 2020 11:12
Show Gist options
  • Save katsaii/5eaaec820ea8f5515896365be05cac63 to your computer and use it in GitHub Desktop.
Save katsaii/5eaaec820ea8f5515896365be05cac63 to your computer and use it in GitHub Desktop.
Tokenises a string into an array of strings and numbers, where anything between double quotes is considered a string.
/// @desc Tokenises a string into a list of values.
/// @param str {String} The string to tokenise.
/// @author Kat @katsaii
var str = argument0;
var tokens = [];
var len = string_length(str);
for (var i = 1; i <= len; i++) {
var prefix = string_char_at(str, i);
if (char_is_whitespace(prefix)) then continue;
var value = undefined;
switch (prefix) {
case "\"":
// tokenise string
var start = i + 1;
var count = 0;
while (start + count <= len) {
var char = string_char_at(str, start + count);
if (char == "\"") then break;
if (char == "\\") then count++;
count++;
}
i = start + count;
value = string_copy(str, start, count);
break;
default:
// tokenise number or word
var validNumber = true;
var decimal = false;
var start = i;
var count = 0;
while (start + count <= len) {
var char = string_char_at(str, start + count);
if (char_is_whitespace(char)) then break;
switch (char) {
case "0":
case "1":
case "2":
case "3":
case "4":
case "5":
case "6":
case "7":
case "8":
case "9":
break;
case "-":
validNumber = count == 0;
break;
case ".":
if (decimal) {
validNumber = false;
} else {
decimal = true;
}
break;
default:
validNumber = false;
break;
}
count++;
}
i = start + count;
value = string_copy(str, start, count);
if (validNumber) {
value = real(value);
} else {
// keywords
switch (value) {
case "null":
case "void":
case "empty":
case "()":
case "undefined":
value = undefined;
break;
case "noone":
value = noone;
break;
case "global":
value = global;
break;
case "self":
value = self;
break;
case "other":
value = other;
break;
case "all":
value = all;
break;
case "inf":
case "infinity":
value = infinity;
break;
case "NaN":
value = NaN;
break;
case "yes":
case "enable":
case "true":
value = true;
break;
case "no":
case "disable":
case "false":
value = false;
break;
default:
// check for an asset, otherwise just accept the word itself
var index = asset_get_index(value);
if (index != -1) then value = index;
break;
}
}
break;
}
tokens[@ array_length_1d(tokens)] = value;
}
return tokens;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment