javascript tokenizer sample

tokenizer.js

let print = (...arg) => {
    console.log(...arg);
}

let longString = 'Acesta, este un string... mai lung';
const sep = [' ', ',', '.', '!']

let tokenizer = (str, seps = [' ']) => {
    let tokens = []
    let token = ''
    for (let i = 0; i < str.length; i++) {
        let c = str[i]
        if (seps.includes(c)) {
            if (token) {
                tokens.push(token)
                token = ''
            }
            continue
        }
        token += c;
    }
    if (token) {
        tokens.push(token)
    }
    return tokens;
}

console.log(tokenizer(longString, sep));

Sample application used to learn simple javascript commands: string, array, for, if