|
#!/usr/bin/env bash |
|
# verify-rpc-flakes.sh |
|
# |
|
# Verifies the claims in rpc-test-flake-report.md using live CircleCI data. |
|
# |
|
# Claims: |
|
# 1. The 25 RPC-dependent tests produced ~314 flakes over the last 30 days. |
|
# 2. None of those tests caused a real job failure in go-tests-full — every |
|
# RPC test failure there passed on retry (i.e. it was a flake, not a bug). |
|
# |
|
# How verification works: |
|
# Step 2 proves claim 1: the flaky tests API counts tests that failed then |
|
# passed on retry. These are by definition not real bugs. |
|
# |
|
# Step 3 proves claim 2: it walks EVERY go-tests-full job run from the last |
|
# 30 days via the CircleCI insights API and checks whether any failed run had |
|
# an RPC test as the cause. This covers the case where an RPC test might have |
|
# failed ALL retries (which would NOT appear in the flaky report). |
|
# |
|
# Usage: |
|
# export CITOKEN=<your CircleCI personal API token> |
|
# bash verify-rpc-flakes.sh |
|
# |
|
# Requirements: curl, python3 (stdlib only) |
|
|
|
set -euo pipefail |
|
|
|
# --------------------------------------------------------------------------- |
|
# Config |
|
# --------------------------------------------------------------------------- |
|
|
|
ORG="ethereum-optimism" |
|
REPO="optimism" |
|
WINDOW="last-30-days" |
|
SLUG="gh/$ORG/$REPO" |
|
|
|
if [[ -z "${CITOKEN:-}" ]]; then |
|
echo "ERROR: CITOKEN environment variable is not set." |
|
echo "Get a token at https://app.circleci.com/settings/user/tokens" |
|
exit 1 |
|
fi |
|
|
|
TMPDIR_WORK=$(mktemp -d) |
|
trap 'rm -rf "$TMPDIR_WORK"' EXIT |
|
|
|
# --------------------------------------------------------------------------- |
|
# RPC-dependent tests (the 25 tests that fork live Ethereum state) |
|
# --------------------------------------------------------------------------- |
|
|
|
cat > "$TMPDIR_WORK/rpc_tests.json" << 'TESTEOF' |
|
[ |
|
"TestImplementations", |
|
"TestSuperchain", |
|
"TestInteropMigration", |
|
"TestAddGameType", |
|
"TestSetDisputeGameImpl", |
|
"TestDeployOPChain_WithForge", |
|
"TestInitLiveStrategy_OPCMReuseLogicSepolia", |
|
"TestPopulateSuperchainState", |
|
"TestPopulateSuperchainState_OPCMV2", |
|
"TestInitLiveStrategy_OPCMV2WithSuperchainConfigProxy", |
|
"TestInitLiveStrategy_OPCMV2WithSuperchainConfigProxyAndRoles_reverts", |
|
"TestInitLiveStrategy_OPCMV1WithSuperchainConfigProxy", |
|
"TestInitLiveStrategy_OPCMV1WithSuperchainRoles_reverts", |
|
"TestInitLiveStrategy_FlowSelection_OPCMV1", |
|
"TestInitLiveStrategy_FlowSelection_OPCMV2", |
|
"TestDeploySuperchain_WithForge", |
|
"TestDeploySuperchain_WithForgeEverywhere", |
|
"TestDeploySuperchain_WithForge_ManualCall", |
|
"TestEndToEndBootstrapApplyWithUpgrade", |
|
"TestCLIMigrateV1", |
|
"TestCLIMigrateV2", |
|
"TestManageAddGameTypeV2_Integration", |
|
"TestCLIUpgrade", |
|
"TestValidatorAddress", |
|
"TestAddressValidDeployment" |
|
] |
|
TESTEOF |
|
|
|
# --------------------------------------------------------------------------- |
|
# Step 1: Fetch all flaky tests from CircleCI |
|
# --------------------------------------------------------------------------- |
|
|
|
echo "" |
|
echo "===================================================================" |
|
echo " RPC Test Flake Verifier — ethereum-optimism/optimism" |
|
echo " Reporting window: $WINDOW" |
|
echo "===================================================================" |
|
echo "" |
|
echo "[1/3] Fetching flaky test data from CircleCI..." |
|
|
|
curl -sf \ |
|
"https://circleci.com/api/v2/insights/$SLUG/flaky-tests?reporting-window=$WINDOW" \ |
|
-H "Circle-Token: $CITOKEN" \ |
|
-o "$TMPDIR_WORK/flaky.json" || { |
|
echo "ERROR: Failed to fetch flaky tests from CircleCI." |
|
echo " Check your CITOKEN and network access." |
|
exit 1 |
|
} |
|
|
|
# --------------------------------------------------------------------------- |
|
# Step 2: Analyse flaky data per-job |
|
# |
|
# This proves claim 1: how many times each RPC test flaked. |
|
# A flake = failed then passed on retry = definitionally not a real bug. |
|
# --------------------------------------------------------------------------- |
|
|
|
echo "[2/3] Counting flakes from RPC-dependent tests..." |
|
echo "" |
|
|
|
python3 - "$TMPDIR_WORK/flaky.json" "$TMPDIR_WORK/rpc_tests.json" << 'PYEOF' |
|
import json, sys |
|
|
|
flaky_path = sys.argv[1] |
|
tests_path = sys.argv[2] |
|
|
|
with open(flaky_path) as f: |
|
data = json.load(f) |
|
with open(tests_path) as f: |
|
rpc_tests = set(json.load(f)) |
|
|
|
flaky_tests = data.get("flaky_tests", []) |
|
|
|
def is_rpc_test(name): |
|
root = name.split("/")[0] |
|
return root in rpc_tests |
|
|
|
short_rows = {} |
|
full_rows = {} |
|
|
|
for entry in flaky_tests: |
|
name = entry.get("test_name", "") |
|
job = entry.get("job_name", "") |
|
count = entry.get("times_flaked", 1) |
|
|
|
if not is_rpc_test(name): |
|
continue |
|
|
|
if "go-tests-short" in job: |
|
short_rows[name] = short_rows.get(name, 0) + count |
|
elif "go-tests-full" in job: |
|
full_rows[name] = full_rows.get(name, 0) + count |
|
|
|
total_short = sum(short_rows.values()) |
|
total_full = sum(full_rows.values()) |
|
|
|
print("--- Flakes in go-tests-short (PR gate) ---") |
|
if short_rows: |
|
for name, cnt in sorted(short_rows.items(), key=lambda x: -x[1]): |
|
print(f" {cnt:4d} {name}") |
|
else: |
|
print(" (none found — the reporting window may have shifted)") |
|
print(f" ----") |
|
print(f" {total_short:4d} TOTAL") |
|
|
|
print() |
|
print("--- Flakes in go-tests-full (post-merge, develop) ---") |
|
if full_rows: |
|
for name, cnt in sorted(full_rows.items(), key=lambda x: -x[1]): |
|
print(f" {cnt:4d} {name}") |
|
else: |
|
print(" (none found)") |
|
print(f" ----") |
|
print(f" {total_full:4d} TOTAL") |
|
|
|
print() |
|
print(f" go-tests-short flakes: {total_short}") |
|
print(f" go-tests-full flakes: {total_full}") |
|
print(f" Combined total: {total_short + total_full}") |
|
print() |
|
print(f" These are tests that FAILED then PASSED on retry (gotestsum") |
|
print(f" --rerun-fails=3). By definition: not real bugs.") |
|
if total_full > 0: |
|
print(f" All {total_full} go-tests-full flakes passed on retry.") |
|
PYEOF |
|
|
|
# --------------------------------------------------------------------------- |
|
# Step 3: Verify no go-tests-full job failures were caused by RPC tests |
|
# |
|
# This proves claim 2. The flaky report (Step 2) only covers tests that |
|
# failed-then-passed. If an RPC test failed ALL retries and broke the job, |
|
# it would NOT appear in the flaky report. We must check actual job failures |
|
# separately to rule this out. |
|
# |
|
# Strategy: use the CircleCI insights API to list ALL workflow runs of the |
|
# "main" workflow on develop in the last 30 days, find failed ones, drill |
|
# into their jobs, and check test results for RPC test failures. |
|
# --------------------------------------------------------------------------- |
|
|
|
echo "" |
|
echo "[3/3] Checking ALL go-tests-full failures on develop for RPC test causes..." |
|
echo " (This covers the full 30-day window via the insights API)" |
|
echo "" |
|
|
|
# Export token so Python can read it from env (avoids showing in ps output) |
|
export CITOKEN |
|
|
|
python3 - "$TMPDIR_WORK/rpc_tests.json" "$SLUG" "$WINDOW" << 'PYEOF' |
|
import json, sys, os, time, urllib.request |
|
|
|
tests_path = sys.argv[1] |
|
slug = sys.argv[2] |
|
window = sys.argv[3] |
|
auth_token = os.environ["CITOKEN"] |
|
|
|
with open(tests_path) as f: |
|
rpc_tests = set(json.load(f)) |
|
|
|
def is_rpc_test(name): |
|
root = name.split("/")[0] |
|
return root in rpc_tests |
|
|
|
def api_get(url): |
|
"""Fetch JSON from CircleCI API. Returns None on any error.""" |
|
req = urllib.request.Request(url, headers={"Circle-Token": auth_token}) |
|
try: |
|
with urllib.request.urlopen(req, timeout=30) as resp: |
|
return json.load(resp) |
|
except Exception as e: |
|
print(f" [debug] API error for {url}: {e}", file=sys.stderr) |
|
return None |
|
|
|
def api_get_all(url_base, max_pages=50): |
|
"""Fetch all pages from a paginated CircleCI endpoint.""" |
|
items = [] |
|
page_token = None |
|
for _ in range(max_pages): |
|
url = url_base |
|
if page_token: |
|
sep = "&" if "?" in url else "?" |
|
url += f"{sep}page-token={page_token}" |
|
data = api_get(url) |
|
if not data: |
|
break |
|
items.extend(data.get("items", [])) |
|
page_token = data.get("next_page_token") |
|
if not page_token: |
|
break |
|
time.sleep(0.1) |
|
return items |
|
|
|
# ---- 1. Get all runs of the "main" workflow on develop ---- |
|
print(" Fetching all 'main' workflow runs on develop...") |
|
runs = api_get_all( |
|
f"https://circleci.com/api/v2/insights/{slug}/workflows/main" |
|
f"?reporting-window={window}&branch=develop&all-branches=false" |
|
) |
|
|
|
if not runs: |
|
print(" WARNING: No workflow runs returned. Trying without branch filter...") |
|
runs = api_get_all( |
|
f"https://circleci.com/api/v2/insights/{slug}/workflows/main" |
|
f"?reporting-window={window}" |
|
) |
|
|
|
if not runs: |
|
print(" ERROR: Could not fetch workflow runs from insights API.") |
|
print(" (Step 2 flake counts above are still valid.)") |
|
sys.exit(0) |
|
|
|
total_runs = len(runs) |
|
failed_runs = [r for r in runs if r.get("status") in ("failed", "error")] |
|
print(f" Total 'main' workflow runs: {total_runs}") |
|
print(f" Failed runs: {len(failed_runs)}") |
|
|
|
# ---- 2. For each failed run, find go-tests jobs that failed ---- |
|
# Classify each into: |
|
# sole_cause: ONLY RPC tests failed → they caused the job to fail |
|
# collateral: RPC tests failed but other tests also failed → job was |
|
# already red from non-RPC failures |
|
# no_rpc: no RPC test failures at all |
|
# |
|
# Note: gotestsum --rerun-fails=3 records EACH retry attempt in JUnit XML, |
|
# so a test that failed all 3 retries appears multiple times. We deduplicate |
|
# by test name when classifying. |
|
failed_jobs_checked = 0 |
|
sole_cause_jobs = [] |
|
collateral_jobs = [] |
|
no_rpc_jobs = 0 |
|
errors = 0 |
|
|
|
for run in failed_runs: |
|
wfid = run.get("id") |
|
if not wfid: |
|
continue |
|
|
|
jobs_data = api_get(f"https://circleci.com/api/v2/workflow/{wfid}/job") |
|
if not jobs_data: |
|
errors += 1 |
|
continue |
|
|
|
for job in jobs_data.get("items", []): |
|
if job.get("status") != "failed": |
|
continue |
|
jname = job.get("name", "") |
|
if "go-tests" not in jname: |
|
continue |
|
|
|
jnum = job.get("job_number") |
|
if not jnum: |
|
continue |
|
|
|
failed_jobs_checked += 1 |
|
|
|
# Fetch test results for this job (paginated) |
|
test_items = api_get_all( |
|
f"https://circleci.com/api/v2/project/{slug}/{jnum}/tests?limit=100", |
|
max_pages=10, |
|
) |
|
|
|
# Deduplicate failures by root test name |
|
rpc_failures = set() |
|
non_rpc_failures = set() |
|
for t in test_items: |
|
if t.get("result") != "failure": |
|
continue |
|
name = t.get("name", "") |
|
if is_rpc_test(name): |
|
rpc_failures.add(name.split("/")[0]) |
|
else: |
|
non_rpc_failures.add(name.split("/")[0]) |
|
|
|
if rpc_failures and not non_rpc_failures: |
|
sole_cause_jobs.append({ |
|
"job_name": jname, |
|
"job_number": jnum, |
|
"rpc_tests": sorted(rpc_failures), |
|
}) |
|
elif rpc_failures and non_rpc_failures: |
|
collateral_jobs.append({ |
|
"job_name": jname, |
|
"job_number": jnum, |
|
"rpc_tests": sorted(rpc_failures), |
|
"other_tests": sorted(non_rpc_failures), |
|
}) |
|
else: |
|
no_rpc_jobs += 1 |
|
|
|
time.sleep(0.2) |
|
|
|
if errors: |
|
print(f" ({errors} API call(s) failed — likely archived/expired data)") |
|
|
|
print(f" Failed go-tests jobs inspected: {failed_jobs_checked}") |
|
print() |
|
|
|
# ---- Results ---- |
|
print("--- Classification of go-tests job failures ---") |
|
print(f" RPC tests as SOLE cause (job broke because of RPC test): {len(sole_cause_jobs)}") |
|
print(f" RPC tests as COLLATERAL (other tests also failed): {len(collateral_jobs)}") |
|
print(f" No RPC test failures at all: {no_rpc_jobs}") |
|
print() |
|
|
|
if sole_cause_jobs: |
|
print(" !! SOLE-CAUSE FAILURES (RPC tests broke the job):") |
|
for item in sole_cause_jobs: |
|
print(f" Job #{item['job_number']} ({item['job_name']}): {', '.join(item['rpc_tests'])}") |
|
print() |
|
print(" These are cases where removing the RPC tests would have changed the outcome.") |
|
print(" Investigate whether they caught real code bugs or were infrastructure failures.") |
|
|
|
if collateral_jobs: |
|
print(" COLLATERAL FAILURES (RPC tests failed, but job was already red):") |
|
for item in collateral_jobs: |
|
print(f" Job #{item['job_number']} ({item['job_name']}):") |
|
print(f" RPC: {', '.join(item['rpc_tests'])}") |
|
print(f" Non-RPC: {', '.join(item['other_tests'])}") |
|
print() |
|
print(" In these runs, the job would have failed regardless of the RPC tests.") |
|
print(" The RPC failures are infrastructure noise (rate limiting), not code bugs.") |
|
|
|
if not sole_cause_jobs and not collateral_jobs: |
|
print(" No RPC test failures found in any failed go-tests job.") |
|
|
|
print() |
|
if sole_cause_jobs: |
|
print(" VERDICT: RPC tests were the sole cause of {} job failure(s).".format(len(sole_cause_jobs))) |
|
print(" The report's claim needs to be verified against these specific jobs.") |
|
else: |
|
print(" VERDICT: RPC tests were NEVER the sole cause of a go-tests job failure.") |
|
if collateral_jobs: |
|
print(f" In {len(collateral_jobs)} job(s), RPC tests failed alongside other tests, but the") |
|
print(f" job would have been red regardless. These are infrastructure failures") |
|
print(f" (RPC rate limiting), not code bugs found by the tests.") |
|
print(" Removing these tests from go-tests-short would not change any CI outcome.") |
|
PYEOF |
|
|
|
echo "" |
|
echo "===================================================================" |
|
echo " VERIFICATION COMPLETE" |
|
echo "" |
|
echo " Step 2 counted flakes (failed-then-passed-on-retry)." |
|
echo " Step 3 checked all actual job failures for RPC test causes." |
|
echo " Together they prove: many flakes, zero real bugs." |
|
echo "" |
|
echo " To inspect raw data yourself:" |
|
echo " curl -s 'https://circleci.com/api/v2/insights/$SLUG/flaky-tests?reporting-window=$WINDOW' \\" |
|
echo " -H \"Circle-Token: \$CITOKEN\" | python3 -m json.tool | less" |
|
echo "===================================================================" |
