Skip to content

Instantly share code, notes, and snippets.

@maxogden

maxogden/index.js

Created Aug 28, 2017
Embed
What would you like to do?
fatal crash
var fs = require('fs')
var request = require('request')
var through = require('through2')
var ndjson = require('ndjson')
var once = require('once')
var pump = require('pump')
var concat = require('concat-stream')
var parallel = require('parallel-transform')
var hyperdb = require('hyperdb')
var db = hyperdb('./npm.db', {valueEncoding: 'json'})
update()
function update (err) {
if (err) {
log('Error: %s - retrying in 5s', err.message)
return setTimeout(update, 5000)
}
latestSeq(function (err, seq) {
if (err) throw err
seq = Math.max(0, seq - 1) // sub 1 incase of errors
if (seq) log('Continuing fetching npm data from seq: %d', seq)
var url = 'https://skimdb.npmjs.com/registry/_changes?heartbeat=30000&include_docs=true&feed=continuous' + (seq ? '&since=' + seq : '')
pump(request(url), ndjson.parse(), save(), update)
})
}
function latestSeq (cb) {
db.get('/latest-seq', function (err, val) {
if (err || !val) return cb(null, 0)
var seq = val[0].value
cb(null, seq)
})
}
function tick (fn, err, val) {
process.nextTick(function () {
fn(err, val)
})
}
function log (fmt) {
fmt = '[dat-npm] ' + fmt
console.error.apply(console, arguments)
}
function save () {
return through.obj(function (data, enc, cb) {
var doc = data.doc
if (!doc) return cb()
if (data.id.match(/^_design\//)) return cb()
var key = doc._id
if (doc._deleted) {
// TODO
return cb()
}
var versions = Object.keys(doc.versions).map(function (v) {
return v
})
db.put('/modules/' + key, versions, function (err) {
if (err) return cb(err)
log('wrote /modules/' + key + '=' + versions + ', seq=' + data.seq)
db.put('/latest-seq', data.seq, cb)
})
})
}
[dat-npm] wrote /modules/solid-ui=0.2.1,0.4.0,0.4.1,0.5.0, seq=2630308
[dat-npm] wrote /modules/react-redux-segments=0.1.0,0.1.1,0.1.2,0.1.3,0.1.4,0.1.5,0.1.6,0.1.7,0.2.0,0.2.1, seq=2630310
[dat-npm] wrote /modules/constant-algebra=0.0.0,0.0.1,0.0.2,0.0.3, seq=2630311
[dat-npm] wrote /modules/cat-graphql=0.0.1,0.0.2,0.0.3,0.0.4,0.0.5,0.0.6,0.0.7,0.0.8,0.0.9,0.0.10,0.0.11,0.0.12,0.0.13,0.0.14,0.1.0,0.1.1,0.1.2,0.1.3,0.2.0, seq=2630313
<--- Last few GCs --->
1942830 ms: Mark-sweep 1176.7 (1404.2) -> 1176.7 (1406.2) MB, 2104.5 / 0.0 ms [allocation failure] [GC in old space requested].
1944864 ms: Mark-sweep 1176.7 (1406.2) -> 1176.7 (1406.2) MB, 2034.0 / 0.0 ms [allocation failure] [GC in old space requested].
1947018 ms: Mark-sweep 1176.7 (1406.2) -> 1186.5 (1404.2) MB, 2154.0 / 0.0 ms [last resort gc].
1949013 ms: Mark-sweep 1186.5 (1404.2) -> 1196.2 (1404.2) MB, 1993.9 / 0.0 ms [last resort gc].
<--- JS stacktrace --->
==== JS stack trace =========================================
Security context: 0x3868b263fa99 <JS Object>
1: parse [native json.js:~61] [pc=0x705fd88b781] (this=0x3868b2646091 <a JSON with map 0x1c587d413b91>,C=0x2b2653cfff39 <Very long string[2284820]>,w=0x3868b2604241 <undefined>)
2: arguments adaptor frame: 1->2
3: mapper(aka parseRow) [/mnt/bigdisk/dat-npm/node_modules/ndjson/index.js:16] [pc=0x705fd8e6f72] (this=0x2453017cc509 <a DestroyableTransform with map 0x161ed6068541>,row=0x...
FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory
1: node::Abort() [node]
2: 0x10d3f9c [node]
3: v8::Utils::ReportApiFailure(char const*, char const*) [node]
4: v8::internal::V8::FatalProcessOutOfMemory(char const*, bool) [node]
5: v8::internal::Factory::NewRawOneByteString(int, v8::internal::PretenureFlag) [node]
6: v8::internal::Handle<v8::internal::String> v8::internal::JsonParser<false>::ScanJsonString<true>() [node]
7: v8::internal::JsonParser<false>::ParseJsonObject() [node]
8: v8::internal::JsonParser<false>::ParseJsonValue() [node]
9: v8::internal::JsonParser<false>::ParseJsonObject() [node]
10: v8::internal::JsonParser<false>::ParseJsonValue() [node]
11: v8::internal::JsonParser<false>::ParseJsonObject() [node]
12: v8::internal::JsonParser<false>::ParseJsonValue() [node]
13: v8::internal::JsonParser<false>::ParseJsonObject() [node]
14: v8::internal::JsonParser<false>::ParseJsonValue() [node]
15: v8::internal::JsonParser<false>::ParseJsonObject() [node]
16: v8::internal::JsonParser<false>::ParseJsonValue() [node]
17: v8::internal::JsonParser<false>::ParseJson() [node]
18: v8::internal::Runtime_ParseJson(int, v8::internal::Object**, v8::internal::Isolate*) [node]
19: 0x705fd6060c7
Aborted (core dumped)
max@useast:/mnt/bigdisk/dat-npm$
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.