Skip to content

Instantly share code, notes, and snippets.

@dcbartlett
Created June 24, 2015 07:10
Show Gist options
  • Save dcbartlett/221200796a239b8cfca7 to your computer and use it in GitHub Desktop.
Save dcbartlett/221200796a239b8cfca7 to your computer and use it in GitHub Desktop.
var nlp = require('./nlp');
var tokens = nlp.tokenize("Eligibility: Open to the general public, must have a library card to checkout library materials");
console.log(tokens.tag());
//Outputs "Open" as "NNP"
var tokens = nlp.tokenize("Eligibility: open to the general public, must have a library card to checkout library materials");
console.log(tokens.tag());
//Outputs "open" as "JJ"
var pos = require('pos');
var _ = require('lodash');
module.exports = {
tokenize: function (string) {
this.tokens = new pos.Lexer().lex(string);
return this;
},
tag: function (tokens) {
if (!tokens) {
var tokens = this.tokens;
}
var taggedTokens = [];
_.each(new pos.Tagger().tag(tokens), function(taggedToken) {
var taggedToken = {
token: taggedToken[0],
tag: taggedToken[1]
}
taggedTokens.push(taggedToken);
});
return taggedTokens;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment