Skip to content

Instantly share code, notes, and snippets.

@tjvr
Last active September 27, 2019 16:10
Show Gist options
  • Save tjvr/7e640d7ec669e6da03b8dec05aa2f983 to your computer and use it in GitHub Desktop.
Save tjvr/7e640d7ec669e6da03b8dec05aa2f983 to your computer and use it in GitHub Desktop.
compile a moo lexer
const moo = require('moo')
function compileClass(constructor, indent) {
let s = ''
s += indent + 'var ' + constructor.name + ' = ' + constructor
for (let key in constructor.prototype) {
s += '\n\n'
const value = constructor.prototype[key]
if (typeof value === 'function') {
s += indent + constructor.name + '.prototype.' + key + ' = ' + value
}
}
if (constructor.prototype[Symbol.iterator]) {
s += '\n\n'
s += indent + constructor.name + '.prototype[Symbol.iterator] = ' + constructor.prototype[Symbol.iterator]
}
return s
}
function compileGroup(group, indent) {
let s = ''
s += '{\n'
for (let key in group) {
const value = group[key]
switch (key) {
case 'type':
case 'value':
case 'next':
case 'push':
if (value === null) continue
break
case 'match': s += indent + ' // ' + value + '\n'; continue
}
let valueStr = JSON.stringify(value)
if (typeof value === 'function') {
valueStr = ('' + value).replace(/\n/g, "\n" + indent)
}
s += indent + ' ' + key + ': ' + valueStr + ',\n'
}
s += indent + '}'
return s
}
function compileLexer(lexer) {
let s = ''
s += '(function() {\n'
// Emit utility functions
s += ' function tokenToString() { return this.value }\n\n'
// Emit Lexer source
const Lexer = lexer.__proto__.constructor
s += compileClass(Lexer, ' ') + '\n\n'
// Emit LexerIterator source
const iterator = lexer[Symbol.iterator]()
const LexerIterator = iterator.__proto__.constructor
s += ' {\n'
s += compileClass(LexerIterator, ' ') + '\n'
s += ' }\n\n'
// Emit lexer instance
let sticky = true
s += ' return new Lexer({\n'
for (let name in lexer.states) {
s += ' ' + JSON.stringify(name) + ': {\n'
const state = lexer.states[name]
sticky = state.regexp.sticky
s += ' regexp: ' + ('' + state.regexp).replace('\n', '\\n') + ',\n'
s += ' groups: [\n'
for (let group of state.groups) {
s += ' ' + compileGroup(group, ' ') + ',\n'
}
s += ' ],\n'
s += ' fast: {\n'
for (let key in state.fast) {
const group = state.fast[key]
s += ' ' + (+key) + ': ' + compileGroup(group, ' ') + ',\n'
}
s += ' },\n'
s += ' error: ' + compileGroup(state.error, ' ') + ',\n'
s += ' },\n'
}
s += ' }, ' + JSON.stringify(lexer.startState) + ')\n\n'
// More utility functions
// assume hasSticky = true
s += ' function eat(re, buffer) {\n'
s += ' return re.exec(buffer)\n'
s += ' }\n'
s += '}())'
return s
}
module.exports = compileLexer
const moo = require('moo')
const compileLexer = require('./compile')
// Define a Lexer
const lexer = moo.compile({
op: ['1', 'x', '='],
foo: 'foo',
ws: {match: /\s+/, lineBreaks: true},
})
// Compile it
const source = compileLexer(lexer)
console.log(source)
// Eval it and check it works
const compiledLexer = eval(source)
compiledLexer.reset("1 foox")
for (let tok of compiledLexer) {
console.log(tok)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment