Merge multiple profiles in Speedscope's JSON format into one file

merge-speedscope-json.js

#!/usr/bin/env node
"use strict";

const fs = require("fs");

const SPEEDSCOPE_JSON_SCHEMA = "https://www.speedscope.app/file-format-schema.json";

const [node, prog, ...options] = process.argv;

if (options[options.length - 2] !== "-o") {
    console.error(`Usage: ${node} ${prog} <INPUT-FILES> -o <OUTPUT-FILE>`);
    process.exit(1);
}

const outputFile = options.pop();
options.pop();
const inputFiles = options;

const output = {
    frameIndices: {},
    frames: [],
    profiles: [],
};

for (const inputFile of inputFiles) {
    console.log(`Processing input from ${inputFile}...`);
    const json = JSON.parse(fs.readFileSync(inputFile));
    const input = {
        frames: json.shared.frames,
        profiles: json.profiles,
    };

    for (const profile of input.profiles) {
        for (const event of profile.events) {
            const frame = input.frames[event.frame].name;
            let frameIndex = output.frameIndices[frame];
            if (frameIndex === undefined) {
                frameIndex = output.frames.length;
                output.frames.push({ name: frame });
                output.frameIndices[frame] = frameIndex;
            }
            event.frame = frameIndex;
        }
        output.profiles.push(profile);
    }
}

console.log(`Writing output to ${outputFile}...`);
fs.writeFileSync(outputFile, JSON.stringify({
    "$schema": SPEEDSCOPE_JSON_SCHEMA,
    "activeProfileIndex": 0,
    "exporter": "merge-speedscope-json",
    "profiles": output.profiles,
    "shared": { "frames": output.frames },
}));

Hi Martin,
I'm currently seeing an out-of-memory crash.
The size of the source json is 10GB.

Here is the dump:

<--- Last few GCs --->

[17624:0x55e6e46c45e0]    19089 ms: Mark-sweep 1398.9 (1426.5) -> 1398.9 (1426.5) MB, 578.2 / 0.0 ms  allocation failure GC in old space requested
[17624:0x55e6e46c45e0]    19783 ms: Mark-sweep 1398.9 (1426.5) -> 1398.9 (1425.5) MB, 583.9 / 0.0 ms  last resort GC in old space requested
[17624:0x55e6e46c45e0]    20474 ms: Mark-sweep 1398.9 (1425.5) -> 1398.9 (1425.5) MB, 691.2 / 0.0 ms  last resort GC in old space requested


<--- JS stacktrace --->

==== JS stack trace =========================================

Security context: 0x2f2c45218fe1 <JSObject>
    0: builtin exit frame: parse(this=0x2f2c45229c41 <Object map = 0x11f179f8f251>,0x9bee1b05759 <Uint8Array map = 0x11f179fdaeb9>)

    1: /* anonymous */ [/tmp/prof/merge-speedscope-json.js:~1] [pc=0x12bdd9987ebe](this=0x28108db847f9 <Object map = 0x11f179f84eb1>,exports=0x28108db847f9 <Object map = 0x11f179f84eb1>,require=0x28108db847b1 <JSFunction require (sfi = 0x339c7ca62b81)>,module=0x281...

FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory
 1: node::Abort() [node]
 2: 0x55e6e2ec5011 [node]
 3: v8::Utils::ReportOOMFailure(char const*, bool) [node]
 4: v8::internal::V8::FatalProcessOutOfMemory(char const*, bool) [node]
 5: v8::internal::Factory::NewJSObject(v8::internal::Handle<v8::internal::JSFunction>, v8::internal::PretenureFlag) [node]
 6: v8::internal::JsonParser<true>::ParseJsonObject() [node]
 7: v8::internal::JsonParser<true>::ParseJsonArray() [node]
 8: v8::internal::JsonParser<true>::ParseJsonObject() [node]
 9: v8::internal::JsonParser<true>::ParseJsonArray() [node]
10: v8::internal::JsonParser<true>::ParseJsonObject() [node]
11: v8::internal::JsonParser<true>::ParseJson() [node]
12: 0x55e6e287802b [node]
13: 0x12bdd98196dd
Aborted (core dumped)

Here is how to reproduce this crash.
You'd need to pull from master in solution-spider repo to get the latest version of the makefiles.

make run-ft daml_profile_dir=/tmp/prof ftags=@SPD-8658 (/tmp/prof must exist)

It should emit around 3.2Gb json.

Then try to combine all the files.