Context
Developers often curios about how Node.js ports of OpenAI's tiktoken perform.
This question is usually prompted by Vercel's Edge Runtime and AWS Lambda requiring more work to make ports with Python binding and WASM to work compared to the purely JavaScript js-tiktoken.
Benchmark results
| function asyncCalc (num, delay) { | |
| delay = delay || 500; | |
| return new Promise(function (resolve, reject) { | |
| setTimeout(function () { | |
| resolve(num); | |
| }, delay); | |
| }); | |
| } |
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <link href="style.css" rel="stylesheet" /> | |
| <meta charset=utf-8 /> | |
| <title>Cytoscape.js initialisation</title> | |
| <script src="http://ajax.googleapis.com/ajax/libs/jquery/1/jquery.min.js"></script> | |
| <script src="http://marvl.infotech.monash.edu/webcola/cola.v3.min.js"></script> | |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/springy/2.5.0/springy.min.js"></script> |
| var http = require('http'); | |
| var _ = require('highland'); | |
| var mongoose = require('mongoose'); | |
| var httpServer = http.createServer(); | |
| var server = _('request', httpServer, ['req', 'res']); | |
| mongoose.connect('mongodb://localhost/test'); | |
| var UserSchema = new mongoose.Schema({ |
| var stream = require('stream'); | |
| var _ = require('highland'); | |
| var $ = require('jquery'); | |
| var toConsoleStream = new stream.Writable({ objectMode: true }); | |
| toConsoleStream._write = function (data, encoding, done) { | |
| console.log(data); | |
| done(); |
| var Readable = require('stream').Readable; | |
| var es = require('event-stream'); | |
| var request = require('superagent'); | |
| var React = require('react'); | |
| var data1 = '{ "name": "substack" }\n{ "name": "dominictarr" }\n'; | |
| var data2 = '{ "name": "mikeal" }\n{ "name": "indutny" }\n{ "name": "dead-horse" }'; | |
| function filterOutEmpty (data, cb){ |
| var util = require('util'); | |
| var fs = require('fs'); | |
| var es = require('event-stream'); | |
| var request = require('request'); | |
| function nicely(data, cb) { | |
| cb(null, util.inspect(data)); | |
| } |
The approach and the client side code you can find here: https://github.com/yahoo/flux-examples/blob/master/routing/app.js
| var select = require('../'); | |
| var tokenize = require('html-tokenize'); | |
| var fs = require('fs'); | |
| var s = select('ul > li dt[name*="sample"]', function (e) { | |
| console.log('*** MATCH ***'); | |
| e.createReadStream().on('data', function (row) { | |
| console.log([ row[0], row[1].toString() ]); | |
| }); | |
| }); |
| 'use strict'; | |
| var http = require('http'); | |
| var select = require('html-select'); | |
| var tokenize = require('html-tokenize'); | |
| var url = 'http://www.bbc.com/news/health-27422547'; | |
| http.get(url, function (response) { |