Programming language designer: course day 1

es5_tokenizer.js

/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.
* File Name   : es5_tokenizer.js
* Created at  : 2019-03-04
* Updated at  : 2019-03-04
* Author      : jeefo
* Purpose     :
* Description :
* Reference   :
.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.*/
// ignore:start
"use strict";

/* globals */
/* exported */

// ignore:end

const JeefoTokenizer = require("jeefo_tokenizer");

const DELIMITERS = [
    '.', ',',
    '/', '?',
    ';', ':',
    "'", '"',
    '`', '~',
    '-',
    '=', '+',
    '\\', '|', 
    '(', ')',
    '[', ']',
    '{', '}',
    '<', '>',
    '!', '@',
    '#', '%',
    '^', '&',
    '*',
].join('');

const es5_tokenizer = new JeefoTokenizer();

// Delimiters
es5_tokenizer.register({
    type       : "Delimiter",
    precedence : 10,

    is : current_character => {
        switch (current_character) {
            case ':' : case ';' :
            case ',' : case '?' :
            case '(' : case ')' :
            case '[' : case ']' :
            case '{' : case '}' :
                return true;
        }
        return false;
    },

    initialize : (token, current_character, streamer) => {
        token.value = current_character;
        token.start = token.end = streamer.get_cursor();
    }
});

// Number token
es5_tokenizer.register({
    type       : "Number",
    precedence : 5,

    is : current_character => current_character >= '0' && current_character <= '9',

    initialize : (token, current_character, streamer) => {
        const start = streamer.get_cursor();

        let length = 1;
        let next_character = streamer.at(start.index + length);

        while (next_character) {
            if (next_character >= '0' && next_character <= '9') {
                length += 1;
                next_character = streamer.at(start.index + length);
            } else {
                break;
            }
        }

        if (next_character === '.') {
            length += 1;
            next_character = streamer.at(start.index + length);

            while (next_character) {
                if (next_character >= '0' && next_character <= '9') {
                    length += 1;
                    next_character = streamer.at(start.index + length);
                } else {
                    break;
                }
            }
        }

        if (next_character === 'e' || next_character === 'E') {
            length += 1;
            next_character = streamer.at(start.index + length);

            if (next_character === '-') {
                length += 1;
                next_character = streamer.at(start.index + length);
            }
            
            if (current_character < '0' && current_character > '9') {
                throw new SyntaxError("Invalid or unexpected token");
            }

            while (next_character) {
                if (next_character >= '0' && next_character <= '9') {
                    length += 1;
                    next_character = streamer.at(start.index + length);
                } else {
                    break;
                }
            }
        }

        streamer.move(length - 1);

        token.value = streamer.string.substring(start.index, start.index + length);
        token.start = start;
        token.end   = streamer.get_cursor();
    }
});

// Comment
// case 1---------------
// Inline comment : // abc '\n'
//
// token = {
//   value : '',
//   comment : "abc",
//   is_inline : true,
//   start : (Cursor object),
//   end   : (Cursor object),
// }
//
// case 2---------------
// Multi line comment : /* abc */
//
// token = {
//   value : '',
//   comment : "abc",
//   is_inline : false,
//   start : (Cursor object),
//   end   : (Cursor object),
// }
//
es5_tokenizer.register({
    type       : "Comment",
    precedence : 40,

    is : (current_character, streamer) => {
    },
    initialize : (token, current_character, streamer) => {
    }
});

// String Literal
// case 1---------------
// "string"
//
// token = {
//   value : "string",
//   quote : '"',
//   start : (Cursor object),
//   end   : (Cursor object),
// }
//
// case 2---------------
// "abc"
//
// token = {
//   value : "abc",
//   quote : "'",
//   start : (Cursor object),
//   end   : (Cursor object),
// }
es5_tokenizer.register({
    type       : "String",
    precedence : 1,

	is : (current_character, streamer) => {

    },
    initialize : (token, current_character, streamer) => {
        // implement
    }
});

// Initialize source code
es5_tokenizer.init(`// abc`, 2);

// 
const token = es5_tokenizer.get_next_token();