Reproducible Playwright benchmark for large JSON + Base64 in browser-based JSON editors (CSV output).

run.mjs

/**
 * JSON Editor Benchmark — run.mjs
 *
 * Measures Load / Format / Minify performance across multiple online JSON editors
 * using Playwright to simulate real user interactions with Base64-heavy payloads.
 *
 * INSTALL
 *   npm install playwright
 *   npx playwright install chromium
 *
 * RUN
 *   node run.mjs
 *
 * Results are written to ./results/benchmark-<timestamp>.csv
 *
 * ENVIRONMENT VARIABLES (all optional)
 *   BENCH_RUNS=1            Number of runs per scenario per site  (default: 3)
 *   BENCH_HEADLESS=0        Set to "0" to watch the browser live  (default: headless)
 *   BENCH_VERBOSE=0         Set to "0" to suppress step logs      (default: verbose)
 *   BENCH_TIMEOUT_MS=45000  Per-action timeout in ms              (default: 45000)
 *
 * EXAMPLE
 *   BENCH_RUNS=1 BENCH_HEADLESS=0 node run.mjs
 */

import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { chromium } from "playwright";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const OUTPUT_DIR = path.join(__dirname, "results");
const OUTPUT_CSV = path.join(OUTPUT_DIR, `benchmark-${Date.now()}.csv`);

const SITES = [
  {
    key: "jsonformatter",
    name: "jsonformatter.org",
    url: "https://jsonformatter.org",
    buttons: {
      format: /Format\s*\/\s*Beautify/i,
      minify: /Minify\s*\/\s*Compact/i,
    },
  },
  {
    key: "codebeautify",
    name: "codebeautify.org/jsonviewer",
    url: "https://codebeautify.org/jsonviewer",
    buttons: {
      format: /Beautify|Format JSON/i,
      minify: /Minify|Compact JSON/i,
    },
  },
  {
    key: "viewjson",
    name: "viewjson.net",
    url: "https://viewjson.net",
    buttons: {
      format: /Prettify/i,
      minify: /Minify/i,
    },
  },
];

const SCENARIOS = [
  { key: "s1-1mb", sizeMB: 1, base64Fields: 0 },
  { key: "s2-5mb", sizeMB: 5, base64Fields: 1 },
  { key: "s3-10mb", sizeMB: 10, base64Fields: 1 },
  { key: "s4-20mb", sizeMB: 20, base64Fields: 1 },
];

const RUNS = Number(process.env.BENCH_RUNS || 3);
const HEADLESS = process.env.BENCH_HEADLESS !== "0";
const ACTION_TIMEOUT_MS = Number(process.env.BENCH_TIMEOUT_MS || 45000);
const VERBOSE = process.env.BENCH_VERBOSE !== "0";

function logStep(message) {
  if (!VERBOSE) return;
  const now = new Date().toISOString().replace("T", " ").slice(0, 19);
  console.log(`[${now}] ${message}`);
}

function createJsonPayload(sizeMB, base64Fields) {
  const targetBytes = sizeMB * 1024 * 1024;
  const prefix = "data:image/png;base64,";
  const current = {
    meta: {
      source: "benchmark",
      targetMB: sizeMB,
      createdAt: new Date().toISOString(),
    },
    data: {},
  };

  const fieldCount = Math.max(base64Fields, 1);
  for (let i = 0; i < fieldCount; i++) {
    current.data[`payload_${i + 1}`] = prefix;
  }

  // Build target size directly to avoid repeated stringify loops for large scenarios.
  const baseJson = JSON.stringify(current);
  const baseBytes = Buffer.byteLength(baseJson, "utf8");
  const prefixBytes = Buffer.byteLength(prefix, "utf8");
  const availableBytes = Math.max(0, targetBytes - baseBytes);
  const perFieldExtra = Math.floor(availableBytes / fieldCount);

  for (let i = 0; i < fieldCount; i++) {
    current.data[`payload_${i + 1}`] = prefix + "A".repeat(perFieldExtra);
  }

  let json = JSON.stringify(current);
  let currentBytes = Buffer.byteLength(json, "utf8");
  if (currentBytes < targetBytes) {
    const diff = targetBytes - currentBytes;
    current.data.payload_1 += "A".repeat(diff);
    json = JSON.stringify(current);
    currentBytes = Buffer.byteLength(json, "utf8");
  }
  if (currentBytes > targetBytes) {
    const over = currentBytes - targetBytes;
    if (over > 0 && current.data.payload_1.length > prefixBytes + over) {
      current.data.payload_1 = current.data.payload_1.slice(0, -over);
      json = JSON.stringify(current);
    }
  }
  return json;
}

async function dismissCommonPopups(page) {
  // Use broad text locator (not role-based) to handle consent dialogs that may
  // use divs/spans styled as buttons rather than actual <button> elements.
  const popupTexts = [
    "Close Ad",
    "Close",
    "Accept",
    "I agree",
    "同意",
    "接受",
    "OK",
  ];
  for (const text of popupTexts) {
    // Try both exact-text and role-button locators.
    const byText = page.locator(`button:has-text("${text}"), [role=button]:has-text("${text}")`).first();
    if (await byText.isVisible().catch(() => false)) {
      await byText.click({ timeout: 2000, force: true }).catch(() => {});
      await page.waitForTimeout(400);
      continue;
    }
    const byRole = page.getByRole("button", { name: text, exact: true }).first();
    if (await byRole.isVisible().catch(() => false)) {
      await byRole.click({ timeout: 2000, force: true }).catch(() => {});
      await page.waitForTimeout(400);
    }
  }
}

async function clearEditorValue(page) {
  // 1) Visible textarea/input clear
  const textArea = page.locator("textarea:visible, input[type='text']:visible").first();
  if (await textArea.isVisible().catch(() => false)) {
    await textArea.fill("", { timeout: ACTION_TIMEOUT_MS, force: true }).catch(() => {});
  }

  // 2) Contenteditable clear (single-shot, no typing)
  const editable = page.locator("[contenteditable='true']:visible, .cm-content:visible").first();
  if (await editable.isVisible().catch(() => false)) {
    await editable.evaluate((el) => {
      el.textContent = "";
      el.dispatchEvent(new InputEvent("input", { bubbles: true, inputType: "deleteContentBackward", data: "" }));
    }).catch(() => {});
  }

  // 3) Common editor APIs clear
  await page
    .evaluate(() => {
      try {
        if (window.monaco?.editor?.getModels?.()[0]) {
          window.monaco.editor.getModels()[0].setValue("");
        }
        if (window.editor?.setValue) {
          window.editor.setValue("");
        }
        if (window.ace?.edit) {
          const nodes = document.querySelectorAll(".ace_editor");
          if (nodes.length > 0) {
            window.ace.edit(nodes[0]).setValue("", -1);
          }
        }
      } catch {
        // no-op
      }
    })
    .catch(() => {});
}

async function setEditorValue(page, json) {
  // 1) Normal textarea/input
  const textArea = page.locator("textarea:visible, input[type='text']:visible").first();
  if (await textArea.isVisible().catch(() => false)) {
    await textArea.fill(json, { timeout: ACTION_TIMEOUT_MS, force: true }).catch(() => {});
    const ok = await page.evaluate(() => {
      const el = document.querySelector("textarea");
      return !!(el && el.value && el.value.length > 50);
    });
    if (ok) return true;
  }

  // 2) Contenteditable editors (single-shot assignment, no keyboard typing)
  const editable = page.locator("[contenteditable='true']:visible, .cm-content:visible").first();
  if (await editable.isVisible().catch(() => false)) {
    const ok = await editable
      .evaluate((el, value) => {
        el.textContent = value;
        el.dispatchEvent(new InputEvent("input", { bubbles: true, inputType: "insertText", data: value }));
        return true;
      }, json)
      .catch(() => false);
    if (ok) return true;
  }

  // 3) Fallback: try common global editors through evaluate (single-shot setValue)
  const injected = await page
    .evaluate((value) => {
      try {
        if (window.monaco?.editor?.getModels?.()[0]) {
          window.monaco.editor.getModels()[0].setValue(value);
          return true;
        }
        if (window.editor?.setValue) {
          window.editor.setValue(value);
          return true;
        }
        if (window.ace?.edit) {
          const nodes = document.querySelectorAll(".ace_editor");
          if (nodes.length > 0) {
            const instance = window.ace.edit(nodes[0]);
            instance.setValue(value, -1);
            return true;
          }
        }
      } catch {
        return false;
      }
      return false;
    }, json)
    .catch(() => false);

  return injected;
}

async function pasteToEditor(page, json) {
  // 1. Write content to clipboard via browser evaluate
  const clipOk = await page
    .evaluate(async (text) => {
      try {
        await navigator.clipboard.writeText(text);
        return true;
      } catch {
        return false;
      }
    }, json)
    .catch(() => false);

  if (!clipOk) return false;

  // 2. Focus the editor area (textarea first, then CM, then contenteditable)
  const targets = [
    "textarea:visible",
    ".cm-content:visible",
    "[contenteditable='true']:visible",
  ];
  let focused = false;
  for (const sel of targets) {
    const el = page.locator(sel).first();
    if (await el.isVisible().catch(() => false)) {
      await el.click({ timeout: 3000, force: true }).catch(() => {});
      focused = true;
      break;
    }
  }
  if (!focused) return false;

  // 3. Simulate Ctrl+V
  await page.keyboard.press("Control+v");
  return true;
}

async function clickAction(page, regex) {
  const button = page.getByRole("button", { name: regex }).first();
  if (!(await button.isVisible().catch(() => false))) return false;
  // force: true avoids Playwright scrolling the page to "reveal" the control;
  // that scroll is unrelated to the benchmarked format/minify work.
  await button.click({ timeout: ACTION_TIMEOUT_MS, force: true });
  return true;
}

async function getAllEditorContents(page) {
  return page
    .evaluate(() => {
      const results = [];
      // 1. Textareas
      document.querySelectorAll("textarea").forEach((el) => {
        if (el.value && el.value.length > 0) results.push(el.value);
      });
      // 2. CodeMirror: join .cm-line elements with \n to preserve line structure.
      document.querySelectorAll(".cm-content").forEach((el) => {
        const lines = Array.from(el.querySelectorAll(".cm-line")).map(
          (l) => l.textContent || ""
        );
        if (lines.length > 0) results.push(lines.join("\n"));
      });
      // 2b. Generic contenteditable (non-CodeMirror)
      document
        .querySelectorAll("[contenteditable='true']:not(.cm-content)")
        .forEach((el) => {
          const text = el.textContent || el.innerText || "";
          if (text.length > 0) results.push(text);
        });
      // 3. Monaco
      try {
        const models = window.monaco?.editor?.getModels?.() || [];
        models.forEach((m) => {
          const v = m.getValue();
          if (v && v.length > 0) results.push(v);
        });
      } catch {}
      // 4. Global editor
      try {
        if (window.editor?.getValue) {
          const v = window.editor.getValue();
          if (v && v.length > 0) results.push(v);
        }
      } catch {}
      // 5. Ace
      try {
        document.querySelectorAll(".ace_editor").forEach((el) => {
          const v = window.ace?.edit?.(el)?.getValue?.();
          if (v && v.length > 0) results.push(v);
        });
      } catch {}
      return results;
    })
    .catch(() => []);
}

async function waitForContentStable(page, validateFn, timeoutMs, pollMs = 300, stableMs = 600) {
  const deadline = Date.now() + timeoutMs;
  let lastSnapshot = "";
  let lastChangeTime = Date.now();

  while (Date.now() < deadline) {
    const contents = await getAllEditorContents(page);
    const snapshot = contents.map((c) => c.length).join(",");

    if (snapshot !== lastSnapshot) {
      lastSnapshot = snapshot;
      lastChangeTime = Date.now();
    }

    const isStable = Date.now() - lastChangeTime >= stableMs;
    if (isStable && contents.some(validateFn)) {
      return { success: true, contents };
    }

    await page.waitForTimeout(pollMs);
  }

  return { success: false };
}

function isFormatted(content) {
  const t = content.trim();
  return t.length > 50 && t.includes("\n") && (t.startsWith("{") || t.startsWith("["));
}

function isMinified(content) {
  const t = content.trim();
  return t.length > 50 && !t.includes("\n") && (t.startsWith("{") || t.startsWith("["));
}

function isLoaded(content) {
  return content.length > 100 && content.includes("benchmark");
}

async function measureAction(page, actionFn) {
  const start = performance.now();
  let success = true;
  let error = "";
  try {
    await actionFn();
  } catch (e) {
    success = false;
    error = e instanceof Error ? e.message : String(e);
  }
  const end = performance.now();
  return { durationMs: Math.round(end - start), success, error };
}

function csvLine(cols) {
  return (
    cols
      .map((c) => {
        const s = String(c ?? "");
        return `"${s.replaceAll('"', '""')}"`;
      })
      .join(",") + "\n"
  );
}

async function run() {
  await fs.mkdir(OUTPUT_DIR, { recursive: true });
  await fs.writeFile(
    OUTPUT_CSV,
    "tool,scenario,action,run,duration_ms,success,error\n",
    "utf8",
  );

  logStep(`Benchmark start | headless=${HEADLESS} | runs=${RUNS} | timeoutMs=${ACTION_TIMEOUT_MS}`);
  logStep(`CSV output: ${OUTPUT_CSV}`);

  // Pre-generate all scenario payloads once to avoid long pauses between scenarios.
  const payloadMap = new Map();
  for (const scenario of SCENARIOS) {
    const t0 = Date.now();
    const payload = createJsonPayload(scenario.sizeMB, scenario.base64Fields);
    payloadMap.set(scenario.key, payload);
    const size = Buffer.byteLength(payload, "utf8");
    logStep(`Payload cached: ${scenario.key}, bytes=${size}, buildMs=${Date.now() - t0}`);
  }

  const browser = await chromium.launch({
    headless: HEADLESS,
    args: HEADLESS ? [] : ["--start-maximized"],
  });
  const context = await browser.newContext({
    // Allow the window to use its real screen size (required for --start-maximized to work)
    viewport: HEADLESS ? { width: 1280, height: 720 } : null,
  });
  await context.grantPermissions(["clipboard-read", "clipboard-write"]);
  const page = await context.newPage();
  page.setDefaultTimeout(ACTION_TIMEOUT_MS);

  for (const site of SITES) {
    logStep(`Site start: ${site.name} (${site.url})`);
    for (const scenario of SCENARIOS) {
      logStep(`  Scenario start: ${scenario.key} (${scenario.sizeMB}MB)`);
      const payload = payloadMap.get(scenario.key);
      logStep(`  Payload ready: ${scenario.key}, bytes=${Buffer.byteLength(payload, "utf8")}`);

      for (let runIndex = 1; runIndex <= RUNS; runIndex++) {
        logStep(`    Run ${runIndex}/${RUNS} -> navigate`);
        await page.goto(site.url, { waitUntil: "domcontentloaded" });
        await page.waitForTimeout(1500);
        await dismissCommonPopups(page);
        await clearEditorValue(page);
        await page.waitForTimeout(150);

        logStep(`    Run ${runIndex}/${RUNS} -> load`);
        const loadResult = await measureAction(page, async () => {
          const ok = await pasteToEditor(page, payload);
          if (!ok) throw new Error("cannot locate or write editor via paste");
          const stable = await waitForContentStable(page, isLoaded, ACTION_TIMEOUT_MS);
          if (!stable.success) throw new Error("load content verification failed");
        });
        logStep(
          `    Run ${runIndex}/${RUNS} -> load done: ${loadResult.durationMs}ms, success=${loadResult.success}`,
        );
        await fs.appendFile(
          OUTPUT_CSV,
          csvLine([site.name, scenario.key, "load", runIndex, loadResult.durationMs, loadResult.success, loadResult.error]),
          "utf8",
        );

        logStep(`    Run ${runIndex}/${RUNS} -> format`);
        await dismissCommonPopups(page);
        const formatResult = await measureAction(page, async () => {
          const ok = await clickAction(page, site.buttons.format);
          if (!ok) throw new Error("format button not found");
          const stable = await waitForContentStable(page, isFormatted, ACTION_TIMEOUT_MS);
          if (!stable.success) throw new Error("format content verification failed");
        });
        logStep(
          `    Run ${runIndex}/${RUNS} -> format done: ${formatResult.durationMs}ms, success=${formatResult.success}`,
        );
        await fs.appendFile(
          OUTPUT_CSV,
          csvLine([site.name, scenario.key, "format", runIndex, formatResult.durationMs, formatResult.success, formatResult.error]),
          "utf8",
        );

        logStep(`    Run ${runIndex}/${RUNS} -> minify`);
        await dismissCommonPopups(page);
        const minifyResult = await measureAction(page, async () => {
          const ok = await clickAction(page, site.buttons.minify);
          if (!ok) throw new Error("minify button not found");
          const stable = await waitForContentStable(page, isMinified, ACTION_TIMEOUT_MS);
          if (!stable.success) throw new Error("minify content verification failed");
        });
        logStep(
          `    Run ${runIndex}/${RUNS} -> minify done: ${minifyResult.durationMs}ms, success=${minifyResult.success}`,
        );
        await fs.appendFile(
          OUTPUT_CSV,
          csvLine([site.name, scenario.key, "minify", runIndex, minifyResult.durationMs, minifyResult.success, minifyResult.error]),
          "utf8",
        );
      }
      logStep(`  Scenario end: ${scenario.key}`);
    }
    logStep(`Site end: ${site.name}`);
  }

  await browser.close();
  logStep(`Benchmark finished. CSV: ${OUTPUT_CSV}`);
}

run().catch((err) => {
  console.error("Benchmark failed:", err);
  process.exitCode = 1;
});