Skip to content

Instantly share code, notes, and snippets.

@substack substack/after-batches.js
Last active Oct 12, 2018

Embed
What would you like to do?
var randombytes = require('crypto').randomBytes
module.exports = function (n, opts) {
var batch = []
var refs = []
for (var i = 0; i < n; i++) {
if (i % 5 === 4) {
batch.push({
type: 'put',
id: randombytes(8).toString('hex'),
links: opts.explicitLinks ? [] : undefined,
value: {
type: 'way',
refs: refs
}
})
refs = []
} else {
var id = randombytes(8).toString('hex')
refs.push(id)
batch.push({
type: 'put',
id: id,
links: opts.explicitLinks ? [] : undefined,
value: {
type: 'node',
lat: Math.random()*20,
lon: Math.random()*20
}
})
}
}
return batch
}
var argv = require('minimist')(process.argv.slice(2), {
boolean: [ 'explicitLinks' ]
})
var kosm = require('kappa-osm')
var kcore = require('kappa-core')
var level = require('level')
var raf = require('random-access-file')
var mkdirp = require('mkdirp')
var randombytes = require('crypto').randomBytes
var tmpdir = require('os').tmpdir()
var src = createOsm()
var generateData = require('./after-batches.js')
var n = Math.floor(argv.n/100)
var batches = []
for (var i = 0; i < n; i++) {
batches.push(generateData(100, argv))
}
var start = Date.now()
;(function next (i) {
var batch = batches[i]
if (!batch) return afterPopulate()
src.batch(batch, function (err) {
if (err) console.error(err)
else next(i+1)
})
})(0)
function afterPopulate () {
src.ready(function () {
var elapsed = (Date.now() - start) / 1000
console.log('populate: ' + elapsed + ' seconds')
var dst = createOsm()
sync(src, dst)
})
}
function sync (a, b) {
var start = Date.now()
replicate(a, b, function (err) {
var elapsed = (Date.now() - start) / 1000
console.log('sync: ' + elapsed + ' seconds')
})
}
function createOsm () {
var dir = tmpdir + '/osm-' + randombytes(8).toString('hex')
mkdirp.sync(dir+'/storage')
return kosm({
index: level(dir+'/index', { valueEncoding: 'binary' }),
core: kcore(dir+'/core', { valueEncoding: 'json' }),
storage: function (name, cb) { cb(null, raf(dir+'/storage/'+name)) }
})
}
function replicate (osm0, osm1, cb) {
var pending = 2
osm0.ready(onready)
osm1.ready(onready)
function onready () {
if (--pending !== 0) return
var r0 = osm0.replicate()
var r1 = osm1.replicate()
pending = 2
r0.pipe(r1).pipe(r0)
r0.once('end', onend)
r1.once('end', onend)
}
function onend () {
if (--pending !== 0) return
var p = 2
osm0.ready(onready)
osm1.ready(onready)
function onready () { if (--p === 0) cb() }
}
}
var randombytes = require('crypto').randomBytes
module.exports = function (n, opts) {
var batch = []
var refs = []
for (var i = 0; i < n; i++) {
if (i % 5 === 4) {
batch.push({
type: 'put',
key: randombytes(8).toString('hex'),
links: opts.explicitLinks ? [] : undefined,
value: {
type: 'way',
refs: refs
},
links: []
})
refs = []
} else {
var id = randombytes(8).toString('hex')
refs.push(id)
batch.push({
type: 'put',
key: id,
links: opts.explicitLinks ? [] : undefined,
value: {
type: 'node',
lat: Math.random()*20,
lon: Math.random()*20
}
})
}
}
return batch
}
var argv = require('minimist')(process.argv.slice(2), {
boolean: [ 'explicitLinks' ]
})
var mkdirp = require('mkdirp')
var randombytes = require('crypto').randomBytes
var tmpdir = require('os').tmpdir()
var osmdb = require('osm-p2p')
var src = createOsm()
var generateData = require('./before-batches.js')
var batches = []
var n = Math.floor(argv.n/100)
for (var i = 0; i < n; i++) {
batches.push(generateData(100, argv))
}
var start = Date.now()
;(function next (i) {
var batch = batches[i]
if (!batch) return afterPopulate()
src.batch(batch, function (err) {
if (err) console.error(err)
else next(i+1)
})
})(0)
function afterPopulate () {
src.ready(function () {
var elapsed = (Date.now() - start) / 1000
console.log('populate: ' + elapsed + ' seconds')
var dst = createOsm()
sync(src, dst)
})
}
function sync (a, b) {
var start = Date.now()
replicate(a.log, b.log, function (err) {
var elapsed = (Date.now() - start) / 1000
console.log('sync: ' + elapsed + ' seconds')
})
}
function createOsm () {
var dir = tmpdir + '/osm-' + randombytes(8).toString('hex')
return osmdb(dir)
}
function replicate (osm0, osm1, cb) {
var pending = 2
osm0.ready(onready)
osm1.ready(onready)
function onready () {
if (--pending !== 0) return
var r0 = osm0.replicate()
var r1 = osm1.replicate()
pending = 2
r0.pipe(r1).pipe(r0)
r0.once('end', onend)
r1.once('end', onend)
}
function onend () {
if (--pending !== 0) return
var p = 2
osm0.ready(onready)
osm1.ready(onready)
function onready () { if (--p === 0) cb() }
}
}

populate and sync time with no updates and implicit links

osm-p2p-db (before) vs kappa-osm (after)

                     5000      27500     50000     275000       500000
                  +---------+---------+---------+----------+--------------+
before | populate |   8.8s  |   49.6s | 1m33.1s |          |   30m20.6s   |
       | sync     |  12.0s  | 1m05.7s | 2m02.4s |          |    > 5h **   |
-------+----------+---------+---------+---------+----------+--------------+
after  | populate |   8.0s  |  39.2s  | 1m12.3s |  9m42.2s |   18m44.1s   |
       | sync     |   9.6s  |  50.0s  | 1m34.2s | 10m00.8s |   19m39.4s   |

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.