site speed measurements + CMS breakdown

check-speed.js

#!/usr/bin/env node

const { chromium } = require('playwright');
const ExcelJS = require('exceljs');
const fs = require('fs');
const path = require('path');

const TTFB_THRESHOLD_MS = 1000;
const ATTEMPTS = 5;
const CONCURRENCY = 10;
const TIMEOUT_MS = 15000;
const REPORT_DIR = path.join(__dirname, 'reports');

function parseArgs() {
  const args = process.argv.slice(2);
  const opts = {
    years: [],
    recheck: null,
    limit: 0,
    output: null,
  };

  for (let i = 0; i < args.length; i++) {
    switch (args[i]) {
      case '--year':
      case '-y':
        opts.years.push(args[++i]);
        break;
      case '--recheck':
      case '-r':
        opts.recheck = args[++i];
        break;
      case '--limit':
      case '-l':
        opts.limit = parseInt(args[++i], 10);
        break;
      case '--output':
      case '-o':
        opts.output = args[++i];
        break;
      case '--help':
      case '-h':
        printUsage();
        process.exit(0);
    }
  }

  return opts;
}

function printUsage() {
  console.log(`
Usage: node check-speed.js [options]

Options:
  -y, --year <year>      Year file(s) to scan (can repeat, e.g. -y 2024 -y 2025)
  -r, --recheck <file>   Re-check only slow domains from a previous XLSX report
  -l, --limit <n>        Limit to first N domains (useful for testing)
  -o, --output <file>    Output XLSX filename (default: auto-generated in reports/)
  -h, --help             Show this help

Examples:
  node check-speed.js -y 2025                    # Scan all 2025 domains
  node check-speed.js -y 2024 -y 2025 -l 100     # First 100 domains from 2024+2025
  node check-speed.js -r reports/slow-2025.xlsx   # Re-check slow domains from report
`);
}

function loadDomainsFromYears(years) {
  const domains = [];
  for (const year of years) {
    const file = path.join(__dirname, `${year}.txt`);
    if (!fs.existsSync(file)) {
      console.error(`File not found: ${file}`);
      continue;
    }
    const lines = fs.readFileSync(file, 'utf8').trim().split('\n');
    // Skip header line "domain date"
    for (let i = 1; i < lines.length; i++) {
      const parts = lines[i].trim().split(/\s+/);
      if (parts[0]) domains.push(parts[0]);
    }
  }
  return [...new Set(domains)];
}

async function loadDomainsFromXlsx(filePath) {
  const workbook = new ExcelJS.Workbook();
  await workbook.xlsx.readFile(filePath);
  const sheet = workbook.getWorksheet('Slow Sites');
  if (!sheet) {
    console.error('No "Slow Sites" sheet found in the report.');
    process.exit(1);
  }
  const domains = [];
  sheet.eachRow((row, rowNumber) => {
    if (rowNumber === 1) return; // skip header
    const domain = row.getCell(1).value;
    if (domain) domains.push(String(domain));
  });
  return domains;
}

async function measureTTFB(page, url) {
  try {
    const response = await page.goto(url, {
      waitUntil: 'commit',
      timeout: TIMEOUT_MS,
    });

    if (!response) return null;

    const timing = await page.evaluate(() => {
      const nav = performance.getEntriesByType('navigation')[0];
      if (!nav) return null;
      return {
        ttfb: nav.responseStart - nav.requestStart,
        dns: nav.domainLookupEnd - nav.domainLookupStart,
        tcp: nav.connectEnd - nav.connectStart,
        tls: nav.secureConnectionStart > 0 ? nav.connectEnd - nav.secureConnectionStart : 0,
        serverProcessing: nav.responseStart - nav.requestStart,
        fullTTFB: nav.responseStart - nav.startTime,
      };
    });

    return {
      status: response.status(),
      ...timing,
    };
  } catch {
    return null;
  }
}

async function testDomain(browser, domain) {
  // First, determine which protocol works (try HTTPS, fallback to HTTP)
  let url = null;
  for (const proto of ['https://', 'http://']) {
    const tryUrl = `${proto}${domain}`;
    const ctx = await browser.newContext({
      userAgent:
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
      locale: 'hu-HU',
      timezoneId: 'Europe/Budapest',
      viewport: { width: 1920, height: 1080 },
    });
    const page = await ctx.newPage();
    const result = await measureTTFB(page, tryUrl);
    await ctx.close();
    if (result) {
      url = tryUrl;
      break;
    }
  }

  if (!url) {
    return { domain, status: 'down' };
  }

  // Now run the actual measurements
  const results = [];

  for (let i = 0; i < ATTEMPTS; i++) {
    const context = await browser.newContext({
      userAgent:
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
      locale: 'hu-HU',
      timezoneId: 'Europe/Budapest',
      viewport: { width: 1920, height: 1080 },
    });

    const page = await context.newPage();
    const result = await measureTTFB(page, url);
    await context.close();

    if (result) {
      results.push(result);
    }
  }

  if (results.length === 0) {
    return { domain, status: 'down' };
  }

  const ttfbs = results.map((r) => r.fullTTFB).filter((t) => t != null);
  if (ttfbs.length === 0) {
    return { domain, status: 'error' };
  }

  const avgTTFB = ttfbs.reduce((a, b) => a + b, 0) / ttfbs.length;
  const minTTFB = Math.min(...ttfbs);
  const maxTTFB = Math.max(...ttfbs);
  const medianTTFB = ttfbs.sort((a, b) => a - b)[Math.floor(ttfbs.length / 2)];

  const serverTimes = results.map((r) => r.serverProcessing).filter((t) => t != null);
  const avgServer = serverTimes.length
    ? serverTimes.reduce((a, b) => a + b, 0) / serverTimes.length
    : null;

  return {
    domain,
    status: 'ok',
    httpStatus: results[0].status,
    attempts: results.length,
    avgTTFB: Math.round(avgTTFB),
    medianTTFB: Math.round(medianTTFB),
    minTTFB: Math.round(minTTFB),
    maxTTFB: Math.round(maxTTFB),
    avgServerProcessing: avgServer != null ? Math.round(avgServer) : null,
    slow: medianTTFB > TTFB_THRESHOLD_MS,
  };
}

async function runBatch(browser, domains, concurrency) {
  const results = [];
  let processed = 0;
  const total = domains.length;

  async function worker(domainQueue) {
    while (domainQueue.length > 0) {
      const domain = domainQueue.shift();
      const result = await testDomain(browser, domain);
      results.push(result);
      processed++;
      const pct = ((processed / total) * 100).toFixed(1);
      const tag =
        result.status === 'down'
          ? 'DOWN'
          : result.slow
            ? `SLOW ${result.medianTTFB}ms`
            : `OK ${result.medianTTFB}ms`;
      process.stdout.write(`\r[${pct}%] ${processed}/${total} - ${domain} → ${tag}     `);
    }
  }

  const queue = [...domains];
  const workers = [];
  for (let i = 0; i < concurrency; i++) {
    workers.push(worker(queue));
  }
  await Promise.all(workers);
  console.log('\n');

  return results;
}

async function writeReport(results, outputPath) {
  const workbook = new ExcelJS.Workbook();

  // Sheet 1: Slow sites
  const slowResults = results.filter((r) => r.slow);
  const slowSheet = workbook.addWorksheet('Slow Sites');
  slowSheet.columns = [
    { header: 'Domain', key: 'domain', width: 40 },
    { header: 'HTTP Status', key: 'httpStatus', width: 12 },
    { header: 'Median TTFB (ms)', key: 'medianTTFB', width: 18 },
    { header: 'Avg TTFB (ms)', key: 'avgTTFB', width: 15 },
    { header: 'Min TTFB (ms)', key: 'minTTFB', width: 15 },
    { header: 'Max TTFB (ms)', key: 'maxTTFB', width: 15 },
    { header: 'Avg Server Processing (ms)', key: 'avgServerProcessing', width: 28 },
    { header: 'Attempts', key: 'attempts', width: 10 },
  ];

  // Style header
  slowSheet.getRow(1).font = { bold: true };
  slowSheet.getRow(1).fill = {
    type: 'pattern',
    pattern: 'solid',
    fgColor: { argb: 'FFD32F2F' },
  };
  slowSheet.getRow(1).font = { bold: true, color: { argb: 'FFFFFFFF' } };

  slowResults
    .sort((a, b) => b.medianTTFB - a.medianTTFB)
    .forEach((r) => slowSheet.addRow(r));

  // Conditional formatting: color TTFB cells
  slowSheet.eachRow((row, rowNumber) => {
    if (rowNumber === 1) return;
    const ttfbCell = row.getCell(3);
    const val = ttfbCell.value;
    if (val > 3000) {
      ttfbCell.fill = { type: 'pattern', pattern: 'solid', fgColor: { argb: 'FFFF0000' } };
      ttfbCell.font = { color: { argb: 'FFFFFFFF' }, bold: true };
    } else if (val > 2000) {
      ttfbCell.fill = { type: 'pattern', pattern: 'solid', fgColor: { argb: 'FFFF9800' } };
    } else {
      ttfbCell.fill = { type: 'pattern', pattern: 'solid', fgColor: { argb: 'FFFFF176' } };
    }
  });

  // Sheet 2: All results
  const allSheet = workbook.addWorksheet('All Results');
  allSheet.columns = [
    { header: 'Domain', key: 'domain', width: 40 },
    { header: 'Status', key: 'status', width: 10 },
    { header: 'HTTP Status', key: 'httpStatus', width: 12 },
    { header: 'Median TTFB (ms)', key: 'medianTTFB', width: 18 },
    { header: 'Avg TTFB (ms)', key: 'avgTTFB', width: 15 },
    { header: 'Min TTFB (ms)', key: 'minTTFB', width: 15 },
    { header: 'Max TTFB (ms)', key: 'maxTTFB', width: 15 },
    { header: 'Slow?', key: 'slow', width: 8 },
  ];
  allSheet.getRow(1).font = { bold: true };
  allSheet.getRow(1).fill = {
    type: 'pattern',
    pattern: 'solid',
    fgColor: { argb: 'FF1565C0' },
  };
  allSheet.getRow(1).font = { bold: true, color: { argb: 'FFFFFFFF' } };

  results
    .sort((a, b) => {
      if (a.status === 'down' && b.status !== 'down') return 1;
      if (a.status !== 'down' && b.status === 'down') return -1;
      return (b.medianTTFB || 0) - (a.medianTTFB || 0);
    })
    .forEach((r) =>
      allSheet.addRow({
        ...r,
        slow: r.slow ? 'YES' : r.status === 'down' ? '' : 'NO',
      })
    );

  // Sheet 3: Summary
  const summarySheet = workbook.addWorksheet('Summary');
  const total = results.length;
  const down = results.filter((r) => r.status === 'down').length;
  const ok = results.filter((r) => r.status === 'ok').length;
  const slow = slowResults.length;
  const fast = ok - slow;

  summarySheet.columns = [
    { header: 'Metric', key: 'metric', width: 30 },
    { header: 'Value', key: 'value', width: 15 },
  ];
  summarySheet.getRow(1).font = { bold: true };

  summarySheet.addRow({ metric: 'Total Domains Tested', value: total });
  summarySheet.addRow({ metric: 'Domains Up', value: ok });
  summarySheet.addRow({ metric: 'Domains Down/Unreachable', value: down });
  summarySheet.addRow({
    metric: `Slow (TTFB > ${TTFB_THRESHOLD_MS}ms)`,
    value: slow,
  });
  summarySheet.addRow({
    metric: `Fast (TTFB <= ${TTFB_THRESHOLD_MS}ms)`,
    value: fast,
  });
  summarySheet.addRow({
    metric: 'Slow % (of reachable)',
    value: ok > 0 ? `${((slow / ok) * 100).toFixed(1)}%` : 'N/A',
  });
  summarySheet.addRow({ metric: 'Scan Date', value: new Date().toISOString() });
  summarySheet.addRow({ metric: 'TTFB Threshold (ms)', value: TTFB_THRESHOLD_MS });
  summarySheet.addRow({ metric: 'Attempts per Domain', value: ATTEMPTS });

  await workbook.xlsx.writeFile(outputPath);
  return { total, ok, down, slow, fast };
}

async function main() {
  const opts = parseArgs();

  if (!opts.recheck && opts.years.length === 0) {
    console.error('Error: Specify --year or --recheck. Use --help for usage.');
    process.exit(1);
  }

  // Load domains
  let domains;
  if (opts.recheck) {
    console.log(`Re-checking slow domains from: ${opts.recheck}`);
    domains = await loadDomainsFromXlsx(opts.recheck);
  } else {
    console.log(`Loading domains from year(s): ${opts.years.join(', ')}`);
    domains = loadDomainsFromYears(opts.years);
  }

  if (opts.limit > 0) {
    domains = domains.slice(0, opts.limit);
  }

  console.log(`Domains to test: ${domains.length}`);
  console.log(`Concurrency: ${CONCURRENCY}, Attempts per domain: ${ATTEMPTS}`);
  console.log(`TTFB threshold: ${TTFB_THRESHOLD_MS}ms\n`);

  if (!fs.existsSync(REPORT_DIR)) {
    fs.mkdirSync(REPORT_DIR, { recursive: true });
  }

  // Launch browser
  const browser = await chromium.launch({ headless: true });

  const startTime = Date.now();
  const results = await runBatch(browser, domains, CONCURRENCY);
  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);

  await browser.close();

  // Determine output filename
  const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
  const tag = opts.recheck ? 'recheck' : opts.years.join('+');
  const outputPath = opts.output || path.join(REPORT_DIR, `speed-${tag}-${timestamp}.xlsx`);

  const stats = await writeReport(results, outputPath);

  console.log('=== SCAN COMPLETE ===');
  console.log(`Time: ${elapsed}s`);
  console.log(`Total: ${stats.total} | Up: ${stats.ok} | Down: ${stats.down}`);
  console.log(`Slow: ${stats.slow} | Fast: ${stats.fast}`);
  console.log(`Report: ${outputPath}`);

  if (stats.slow > 0) {
    console.log(`\nTo re-check slow sites:`);
    console.log(`  node check-speed.js --recheck ${outputPath}`);
  }
}

main().catch((err) => {
  console.error('Fatal error:', err);
  process.exit(1);
});

detect_cms.py

#!/usr/bin/env python3
"""Deterministic CMS / framework detection for the top 200 slow Hungarian sites.

Reads reports/speed-recheck-2026-03-16-filtered.xlsx, takes the first 200
domains from the "Slow Sites (filtered)" sheet, fetches each homepage
(HTTPS first, then HTTP) and classifies it using header + HTML
fingerprints.  No AI, just regex/string match.

Categories (first matching pattern wins, order matters):
  Parking / Suspended  – ELIN.hu, cPanel "Account Suspended", Apache
                          default, generic domain registrar parking.
  Joomla, Webnode, UNAS, Shoprenter, Shopify, Magento, PrestaShop,
  OpenCart, Wix, Squarespace, Webflow, Tilda, Jimdo, TYPO3, Umbraco,
  Kentico, Next.js, Gatsby, Nuxt, webtrees
  Drupal, WordPress, Sitecore   – the four originally requested buckets.
  Unknown               – anything else.

WordPress sites get an extra `woocommerce` flag in the CSV.
"""

import concurrent.futures
import csv
import json
import re
import sys
import urllib3
from pathlib import Path

import openpyxl
import requests

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

ROOT = Path('/mnt/work/gizra/slow-hungarian-sites')
XLSX = ROOT / 'reports' / 'speed-recheck-2026-03-16-filtered.xlsx'
OUT_DIR = Path('/tmp/cms-scan')
HTML_DIR = OUT_DIR / 'html'
HTML_DIR.mkdir(parents=True, exist_ok=True)
RESULT_CSV = OUT_DIR / 'results.csv'

TOP_N = 200
TIMEOUT = 25
WORKERS = 16
UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'


def rx(pattern, flags=re.I):
    return re.compile(pattern.encode() if isinstance(pattern, str) else pattern, flags)


# ---------------------------------------------------------------------------
# Parking / suspended fingerprints – checked FIRST so an ELIN parking page
# never accidentally gets attributed to the underlying domain owner.
# ---------------------------------------------------------------------------
PARKING_PATTERNS = [
    # ELIN.hu hosting / domain parking. Match on the distinctive ASCII bits
    # ("ELIN.hu Kft" inside a <title>, the @font-face "elinfont" they ship,
    # or their domain-services URL fragment) – avoids UTF-8 multibyte issues.
    (rx(r'<title>[^<]{0,200}ELIN\.hu\s+Kft'), 'ELIN.hu parking page'),
    (rx(r'font-family:\s*elinfont\b'), 'ELIN.hu parking page'),
    (rx(r'domain-szolgaltatasok/domain-regisztracio'), 'ELIN.hu parking page'),
    # cPanel "Account Suspended"
    (rx(r'<title>\s*Account\s+Suspended\s*</title>'), 'cPanel "Account Suspended"'),
    # Apache default landing page (Fedora/Debian/CentOS default index)
    (rx(r'Apache\s+is\s+functioning\s+normally'), 'Apache default page'),
    (rx(r'<title>\s*Apache2\s+(Debian|Ubuntu|Default)\s+Page'), 'Apache default page'),
    (rx(r'<title>\s*Test\s+Page\s+for\s+the\s+Apache'), 'Apache default page'),
    # Domain Regisztráció Kft parked page (e.g. klima-tesla.hu)
    (rx(r'<title>[^<]{0,200}Domain\s+Regisztr'), 'Domain Regisztráció Kft parking'),
    (rx(r'Domain\s+regisztr[^<]{0,40}\s+1\s*Ft'), 'Domain Regisztráció Kft parking'),
    (rx(r'domreg_favicon'), 'Domain Regisztráció Kft parking'),
    # Generic domain-for-sale / parking-as-a-service markers
    (rx(r'this\s+domain\s+(?:is|may\s+be)\s+for\s+sale'), 'Generic parking'),
    (rx(r'sedoparking\.com|cashparking\.com|parking\.bodis\.com'), 'Domain parking service'),
    # nginx default
    (rx(r'<title>\s*Welcome\s+to\s+nginx'), 'nginx default page'),
]


# ---------------------------------------------------------------------------
# CMS / framework fingerprints – first match wins.
# Each entry: (cms_label, [list of regex])
# ---------------------------------------------------------------------------
CMS_PATTERNS = [
    ('Joomla', [
        rx(r'<meta[^>]+name="generator"[^>]+content="Joomla'),
        rx(r'HELIX_ULTIMATE_GENERATOR_TEXT'),
        rx(r'/media/jui/'),
        rx(r'/components/com_'),
        rx(r'/modules/mod_'),
        rx(r'/templates/[a-z0-9_-]+/(html|css|js)/'),
        rx(r'option=com_'),
    ]),
    ('Webnode', [
        rx(r'<meta[^>]+name="generator"[^>]+content="Webnode'),
        rx(r'assets\.webnode\.com'),
        rx(r'cdn\.webnode\.com'),
    ]),
    ('UNAS', [
        rx(r'<meta[^>]+name="generator"[^>]+content="UNAS'),
        # UNAS-hosted webshops resolve assets via "<shop>.unas.hu/element/"
        # or use *.unas.hu CDN / shop hosts.
        rx(r'[a-z0-9-]+\.unas\.hu/element/'),
        rx(r'(shop|cdn|static|kep|img)\.unas\.hu'),
        rx(r'\bunas-shop\b'),
    ]),
    ('Shoprenter', [
        rx(r'<meta[^>]+name="generator"[^>]+content="Shoprenter'),
        rx(r'shoprenter\.hu'),
        rx(r'cdn\.myshoprenter\.hu'),
        rx(r'shp_(?:lang|currency|cookie)'),
    ]),
    ('Shopify', [
        rx(r'cdn\.shopify\.com'),
        rx(r'Shopify\.theme'),
        rx(r'shopify-section'),
        rx(r'<meta[^>]+name="shopify-'),
    ]),
    ('Magento', [
        rx(r'/skin/frontend/'),
        rx(r'Mage\.Cookies'),
        rx(r'/static/version\d'),
        rx(r'data-mage-init'),
        rx(r'Magento_Ui/js'),
    ]),
    ('PrestaShop', [
        rx(r'<meta[^>]+name="generator"[^>]+content="PrestaShop'),
        rx(r'var\s+prestashop\b'),
        rx(r'/themes/[^/]+/assets/cache/'),
        rx(r'/modules/ps_'),
    ]),
    ('OpenCart', [
        rx(r'<meta[^>]+name="generator"[^>]+content="OpenCart'),
        rx(r'/catalog/view/theme/'),
        rx(r'route=common/home'),
    ]),
    ('Wix', [
        rx(r'static\.wixstatic\.com'),
        rx(r'wix\.com/_partials'),
        rx(r'<meta[^>]+name="generator"[^>]+content="Wix\.com'),
    ]),
    ('Squarespace', [
        rx(r'static1\.squarespace\.com'),
        rx(r'<!--\s*This\s+is\s+Squarespace'),
        rx(r'squarespace-cdn\.com'),
    ]),
    ('Webflow', [
        rx(r'<html[^>]+data-wf-site='),
        rx(r'webflow\.js'),
        rx(r'<meta[^>]+name="generator"[^>]+content="Webflow'),
    ]),
    ('Tilda', [
        rx(r'tildacdn\.com'),
        rx(r'tilda\.ws'),
        rx(r'<meta[^>]+name="generator"[^>]+content="Tilda'),
    ]),
    ('Jimdo', [
        rx(r'jimdo-static\.com'),
        rx(r'<meta[^>]+name="generator"[^>]+content="Jimdo'),
    ]),
    ('TYPO3', [
        rx(r'<meta[^>]+name="generator"[^>]+content="TYPO3'),
        rx(r'/typo3conf/'),
        rx(r'/typo3temp/'),
        rx(r'/fileadmin/(templates|user_upload)'),
    ]),
    ('Umbraco', [
        rx(r'<meta[^>]+name="generator"[^>]+content="Umbraco'),
        rx(r'/umbraco/(surface|api|render)/'),
    ]),
    ('Kentico', [
        rx(r'<meta[^>]+name="generator"[^>]+content="Kentico'),
        rx(r'/CMSPages/'),
        rx(r'/CMSScripts/'),
    ]),
    ('webtrees', [
        rx(r'<meta[^>]+name="generator"[^>]+content="webtrees'),
        rx(r'/vendor/fisharebest/webtrees'),
    ]),
    ('Next.js', [
        rx(r'/_next/static/'),
        rx(r'__NEXT_DATA__'),
        rx(r'<script[^>]+id="__NEXT_DATA__"'),
    ]),
    ('Gatsby', [
        rx(r'<div[^>]+id="___gatsby"'),
        rx(r'window\.___gatsby'),
        rx(r'gatsby-link'),
    ]),
    ('Nuxt', [
        rx(r'window\.__NUXT__'),
        rx(r'/_nuxt/'),
    ]),
    # Originally requested buckets
    ('Drupal', [
        rx(r'<meta[^>]+name="generator"[^>]+content="Drupal'),
        rx(r'/sites/default/files/'),
        rx(r'/sites/all/(modules|themes)/'),
        rx(r'/core/misc/drupal\.'),
        rx(r'data-drupal-'),
        rx(r'Drupal\.settings'),
        rx(r'drupalSettings'),
        rx(r'jQuery\.extend\(Drupal'),
    ]),
    ('WordPress', [
        rx(r'<meta[^>]+name="generator"[^>]+content="WordPress'),
        rx(r'/wp-content/'),
        rx(r'/wp-includes/'),
        rx(r'/wp-json/'),
        rx(r'wp-emoji-release'),
        rx(r'wpemojiSettings'),
    ]),
    ('Sitecore', [
        rx(r'/sitecore/'),
        rx(r'<meta[^>]+name="generator"[^>]+content="Sitecore'),
        rx(r'sc_site='),
        rx(r'sitecore\.shell'),
        rx(r'xmlns:sc='),
    ]),
]

# Sub-flag patterns – computed in addition to the main CMS bucket.
WOOCOMMERCE_PATTERNS = [
    rx(r'/wp-content/plugins/woocommerce/'),
    rx(r'\bwoocommerce(?:-js)?\b'),
    rx(r'woocommerce_params'),
]

# HTTP-header fingerprints (checked before HTML to leverage server hints).
HEADER_PATTERNS = [
    ('Drupal',  'x-generator',          re.compile(r'drupal', re.I)),
    ('Drupal',  'x-drupal-cache',       re.compile(r'.', re.I)),
    ('Drupal',  'x-drupal-dynamic-cache', re.compile(r'.', re.I)),
    ('WordPress', 'x-powered-by',       re.compile(r'wordpress', re.I)),
    ('WordPress', 'link',               re.compile(r'wp-json', re.I)),
    ('Sitecore', 'x-powered-by',        re.compile(r'sitecore', re.I)),
    ('Shopify', 'x-shopify-stage',      re.compile(r'.', re.I)),
    ('Shopify', 'x-shopid',             re.compile(r'.', re.I)),
    ('Shopify', 'powered-by',           re.compile(r'shopify', re.I)),
    ('Magento', 'x-magento-cache-debug', re.compile(r'.', re.I)),
    ('TYPO3',   'x-typo3-parsetime',    re.compile(r'.', re.I)),
]


def load_top_domains(n):
    wb = openpyxl.load_workbook(XLSX, read_only=True)
    ws = wb['Slow Sites (filtered)']
    out = []
    for row in ws.iter_rows(min_row=2, values_only=True):
        domain = row[0]
        if not domain:
            continue
        out.append({
            'domain': domain.strip(),
            'http_status': row[1],
            'median_ttfb': row[2],
        })
        if len(out) >= n:
            break
    return out


def safe_name(domain):
    return re.sub(r'[^A-Za-z0-9._-]', '_', domain)


def fetch(domain):
    session = requests.Session()
    session.headers.update({'User-Agent': UA, 'Accept': 'text/html,*/*'})
    last_err = None
    for scheme in ('https', 'http'):
        try:
            r = session.get(f'{scheme}://{domain}/', timeout=TIMEOUT,
                            allow_redirects=True, verify=False)
            return r.url, r.status_code, dict(r.headers), r.content[:400_000]
        except requests.RequestException as e:
            last_err = f'{scheme}: {type(e).__name__}'
            continue
    return None, last_err, None, None


def get_header(headers, name):
    if not headers:
        return None
    for k, v in headers.items():
        if k.lower() == name.lower():
            return v
    return None


def classify(headers, body):
    if body:
        for pattern, label in PARKING_PATTERNS:
            if pattern.search(body):
                return 'Parking / Suspended', f'parking:{label}'
    if headers:
        for cms, hname, regex in HEADER_PATTERNS:
            v = get_header(headers, hname)
            if v and regex.search(v):
                return cms, f'header:{hname}={v[:80]}'
    if body:
        for cms, patterns in CMS_PATTERNS:
            for p in patterns:
                if p.search(body):
                    return cms, f'html:{p.pattern.decode("utf-8", "replace")[:80]}'
    return 'Unknown', ''


def detect_woocommerce(body):
    if not body:
        return False
    return any(p.search(body) for p in WOOCOMMERCE_PATTERNS)


def process(entry):
    domain = entry['domain']
    final_url, status, headers, body = fetch(domain)
    if body is None:
        return {
            **entry,
            'final_url': '',
            'http_fetch_status': status if isinstance(status, int) else 0,
            'fetch_error': '' if isinstance(status, int) else str(status),
            'cms': 'Unknown',
            'evidence': 'fetch-failed',
            'woocommerce': '',
            'body_len': 0,
        }
    out_file = HTML_DIR / f'{safe_name(domain)}.html'
    try:
        out_file.write_bytes(body)
    except OSError:
        pass
    cms, evidence = classify(headers, body)
    woo = ''
    if cms == 'WordPress':
        woo = 'yes' if detect_woocommerce(body) else 'no'
    return {
        **entry,
        'final_url': final_url or '',
        'http_fetch_status': status if isinstance(status, int) else 0,
        'fetch_error': '',
        'cms': cms,
        'evidence': evidence,
        'woocommerce': woo,
        'body_len': len(body),
    }


def main():
    domains = load_top_domains(TOP_N)
    print(f'Loaded {len(domains)} domains from {XLSX.name}', file=sys.stderr)
    results = []
    with concurrent.futures.ThreadPoolExecutor(max_workers=WORKERS) as pool:
        futures = {pool.submit(process, d): d['domain'] for d in domains}
        for i, fut in enumerate(concurrent.futures.as_completed(futures), 1):
            results.append(fut.result())
            if i % 10 == 0 or i == len(domains):
                print(f'  {i}/{len(domains)} done', file=sys.stderr)
    order = {d['domain']: i for i, d in enumerate(domains)}
    results.sort(key=lambda r: order.get(r['domain'], 1e9))

    with RESULT_CSV.open('w', newline='') as f:
        w = csv.DictWriter(f, fieldnames=[
            'domain', 'median_ttfb', 'http_status', 'http_fetch_status',
            'cms', 'woocommerce', 'evidence', 'final_url', 'fetch_error',
            'body_len',
        ])
        w.writeheader()
        for r in results:
            w.writerow({k: r.get(k, '') for k in w.fieldnames})
    counts = {}
    for r in results:
        counts[r['cms']] = counts.get(r['cms'], 0) + 1
    print(json.dumps(counts, indent=2, sort_keys=True))


if __name__ == '__main__':
    main()