|
#!/usr/bin/env python3 |
|
""" |
|
Three-way benchmark: Ollama, vanilla MLX, and Rapid-MLX, on the same weights. |
|
|
|
Why this is a separate file rather than a change to bench.py |
|
------------------------------------------------------------ |
|
`bench.py` and the three `results-*.json` files it produced are the evidence |
|
behind a published article. Mutating it would make those 54 runs |
|
unreproducible. This script imports its controls instead, so the published |
|
harness stays byte-identical and the third engine is additive. |
|
|
|
What it adds beyond bench.py |
|
---------------------------- |
|
**A third engine.** Rapid-MLX (`rapid-mlx`, distributed as the `vllm_mlx` |
|
package) is an OpenAI-compatible server that wraps the same Apple MLX stack |
|
`mlx_lm` exposes as a library. It is served the *same* MLX repo the vanilla MLX |
|
path loads, so this is wrapper-against-library rather than a different model. |
|
|
|
**Client-side timing for every engine, as the comparison basis.** This is the |
|
methodological crux and the reason the file exists. |
|
|
|
bench.py deliberately reads engine-native counters: Ollama returns |
|
`prompt_eval_count`/`eval_count` with nanosecond durations, and `mlx_lm` |
|
reports `prompt_tps`/`generation_tps`. That excludes HTTP and process overhead, |
|
which is right when both sides expose such counters. |
|
|
|
Rapid-MLX exposes none. Its `/v1/completions` usage block carries |
|
`prompt_tokens` and `completion_tokens` and no durations, and its Prometheus |
|
`/metrics` endpoint has no per-request prefill or decode timing either. So |
|
there is no native number to read. |
|
|
|
Timing Rapid-MLX client-side while reading native counters for the other two |
|
would charge Rapid-MLX for HTTP, serialization, and scheduler overhead that |
|
vanilla MLX never pays and Ollama's counters exclude. That is the mirror image |
|
of the prompt-cache error the article documents: a number that looks like a |
|
like-for-like comparison and is not one. |
|
|
|
So every engine is timed client-side, identically, and those columns are what |
|
the three-way comparison uses. Native counters are still recorded where an |
|
engine provides them, so the Ollama and MLX rows stay checkable against the |
|
published 54 runs. |
|
|
|
Client-side definitions, applied the same way to all three: |
|
|
|
client_ttft_s request dispatch to first content token |
|
client_decode_tps (gen_tokens - 1) / (last token time - first token time) |
|
client_total_s dispatch to stream close |
|
|
|
`client_decode_tps` drops the first token because it is prefill-bound, which is |
|
the standard client-side convention and the one Rapid-MLX's own published |
|
benchmark uses. There is no client-side prefill throughput column: dividing |
|
prompt tokens by TTFT folds in queueing and first-token sampling and is not |
|
prefill throughput. TTFT is reported directly instead, which is the honest |
|
client-observable prefill signal. |
|
|
|
**In-process MLX is not charged HTTP it does not pay, and this is disclosed.** |
|
Vanilla MLX runs in this process, so its client-side timing has no socket in |
|
it, while Ollama and Rapid-MLX both cross localhost. That asymmetry favours |
|
vanilla MLX by however much a localhost round trip costs, which is small next |
|
to a multi-second generation but is not zero. It is a floor on vanilla MLX's |
|
apparent advantage and a ceiling on the two servers', and it cannot be removed |
|
without wrapping mlx_lm in a server of its own, which would then be measuring |
|
a server this article is not about. |
|
|
|
Controls inherited from bench.py |
|
-------------------------------- |
|
Same weights per cell, the chat template applied once here and sent raw to both |
|
servers, a fresh nonce per repetition to defeat prefix caching on every engine, |
|
one discarded warm-up per cell per engine, thinking disabled, medians with the |
|
full range, and engine order rotated across repetitions so thermal drift is |
|
shared rather than charged to whichever ran last. |
|
|
|
Running it |
|
---------- |
|
Rapid-MLX serves one model per process, so the server is managed outside this |
|
script and the script verifies it is serving the model it is about to measure: |
|
|
|
.venv-rapidmlx/bin/rapid-mlx --no-telemetry serve <mlx repo> \ |
|
--host 127.0.0.1 --port 8080 --no-thinking |
|
|
|
.venv/bin/python bench-3way.py --models qwen3.8-4bit --output results-3way-4bit.json |
|
|
|
`--verify-cache-defeat` runs the check the README describes for Ollama against |
|
all three engines: the same prompt twice, then a fresh nonce. A repeat that is |
|
much faster than the first is a cache hit, and the nonce must restore the |
|
original cost. If it does not, the nonce is not defeating that engine's cache |
|
and every number in the run is measuring a lookup. |
|
""" |
|
|
|
from __future__ import annotations |
|
|
|
import argparse |
|
import json |
|
import statistics |
|
import time |
|
import urllib.error |
|
import urllib.request |
|
from dataclasses import asdict, dataclass, field |
|
from datetime import datetime, timezone |
|
from typing import Any, Iterable |
|
from uuid import uuid4 |
|
|
|
import bench |
|
from bench import ( |
|
MIN_TRUSTWORTHY_GEN_TOKENS, |
|
MODELS, |
|
build_prompt, |
|
environment, |
|
ollama_has, |
|
ollama_pull, |
|
ollama_resident_gb, |
|
) |
|
|
|
RAPIDMLX_HOST = "http://127.0.0.1:8080" |
|
|
|
ENGINES = ["ollama", "mlx", "rapidmlx"] |
|
|
|
DEFAULT_MODELS = ["qwen3.8-4bit", "qwen3-30b-a3b-4bit"] |
|
DEFAULT_PROMPT_TOKENS = [512, 4096, 16384] |
|
DEFAULT_GEN_TOKENS = 256 |
|
DEFAULT_REPS = 3 |
|
|
|
|
|
@dataclass |
|
class Run: |
|
model_key: str |
|
engine: str |
|
prompt_target: int |
|
rep: int |
|
prompt_tokens: int = 0 |
|
gen_tokens: int = 0 |
|
|
|
# Engine-native counters. Absent for rapidmlx, which exposes none. |
|
native_prefill_tps: float = 0.0 |
|
native_decode_tps: float = 0.0 |
|
has_native: bool = False |
|
|
|
# Client-side, measured identically for every engine. The comparison basis. |
|
client_ttft_s: float = 0.0 |
|
client_decode_tps: float = 0.0 |
|
client_total_s: float = 0.0 |
|
|
|
peak_memory_gb: float = 0.0 |
|
finish_reason: str = "" |
|
error: str = "" |
|
warnings: list[str] = field(default_factory=list) |
|
|
|
|
|
# --------------------------------------------------------------------------- |
|
# Engine runners. Each returns the same contract, so measure() stays uniform. |
|
# --------------------------------------------------------------------------- |
|
|
|
|
|
def run_ollama(model_tag: str, prompt: str, gen_tokens: int, num_ctx: int) -> dict[str, Any]: |
|
payload = { |
|
"model": model_tag, |
|
"prompt": prompt, |
|
"stream": True, |
|
"raw": True, # Template already applied. Do not apply it twice. |
|
"think": False, |
|
"options": { |
|
"num_predict": gen_tokens, |
|
"num_ctx": num_ctx, |
|
"temperature": 0, |
|
"seed": 42, |
|
}, |
|
"keep_alive": "10m", |
|
} |
|
request = urllib.request.Request( |
|
f"{bench.OLLAMA_HOST}/api/generate", |
|
data=json.dumps(payload).encode(), |
|
headers={"Content-Type": "application/json"}, |
|
) |
|
|
|
started = time.perf_counter() |
|
first = last = 0.0 |
|
final: dict[str, Any] = {} |
|
with urllib.request.urlopen(request, timeout=3600) as response: |
|
for raw_line in response: |
|
line = raw_line.decode().strip() |
|
if not line: |
|
continue |
|
message = json.loads(line) |
|
if message.get("response"): |
|
now = time.perf_counter() |
|
if not first: |
|
first = now |
|
last = now |
|
if message.get("done"): |
|
final = message |
|
total = time.perf_counter() - started |
|
|
|
prompt_ns = final.get("prompt_eval_duration") or 0 |
|
eval_ns = final.get("eval_duration") or 0 |
|
prompt_count = final.get("prompt_eval_count") or 0 |
|
eval_count = final.get("eval_count") or 0 |
|
|
|
return { |
|
"prompt_tokens": prompt_count, |
|
"gen_tokens": eval_count, |
|
"native_prefill_tps": (prompt_count / (prompt_ns / 1e9)) if prompt_ns else 0.0, |
|
"native_decode_tps": (eval_count / (eval_ns / 1e9)) if eval_ns else 0.0, |
|
"has_native": True, |
|
"client_ttft_s": (first - started) if first else 0.0, |
|
"client_decode_tps": _client_decode(eval_count, first, last), |
|
"client_total_s": total, |
|
"peak_memory_gb": ollama_resident_gb(model_tag), |
|
"finish_reason": final.get("done_reason", ""), |
|
} |
|
|
|
|
|
class MLX: |
|
"""Vanilla mlx_lm, loaded once so load cost is not charged per run.""" |
|
|
|
def __init__(self, repo: str): |
|
import mlx.core as mx |
|
from mlx_lm import load |
|
|
|
mx.reset_peak_memory() |
|
self.model, self.tokenizer = load(repo) |
|
self.repo = repo |
|
|
|
def run(self, prompt: str, gen_tokens: int) -> dict[str, Any]: |
|
import mlx.core as mx |
|
from mlx_lm import stream_generate |
|
from mlx_lm.sample_utils import make_sampler |
|
|
|
mx.random.seed(42) |
|
sampler = make_sampler(temp=0.0) |
|
|
|
started = time.perf_counter() |
|
first = last = 0.0 |
|
final = None |
|
for response in stream_generate( |
|
self.model, self.tokenizer, prompt, max_tokens=gen_tokens, sampler=sampler |
|
): |
|
now = time.perf_counter() |
|
if not first: |
|
first = now |
|
last = now |
|
final = response |
|
total = time.perf_counter() - started |
|
|
|
if final is None: |
|
return {"error": "no tokens generated"} |
|
|
|
return { |
|
"prompt_tokens": final.prompt_tokens, |
|
"gen_tokens": final.generation_tokens, |
|
"native_prefill_tps": final.prompt_tps, |
|
"native_decode_tps": final.generation_tps, |
|
"has_native": True, |
|
"client_ttft_s": (first - started) if first else 0.0, |
|
"client_decode_tps": _client_decode(final.generation_tokens, first, last), |
|
"client_total_s": total, |
|
"peak_memory_gb": round(final.peak_memory, 2), |
|
"finish_reason": final.finish_reason or "", |
|
} |
|
|
|
|
|
def run_rapidmlx(model_id: str, prompt: str, gen_tokens: int) -> dict[str, Any]: |
|
""" |
|
One Rapid-MLX generation over /v1/completions. |
|
|
|
/v1/completions rather than /v1/chat/completions because the chat endpoint |
|
would apply its own template to an already-templated string, changing the |
|
token sequence and invalidating the cell. The token-parity assertion in |
|
measure() is what proves this worked. |
|
""" |
|
payload = { |
|
"model": model_id, |
|
"prompt": prompt, |
|
"max_tokens": gen_tokens, |
|
"temperature": 0, |
|
"stream": True, |
|
"stream_options": {"include_usage": True}, |
|
} |
|
request = urllib.request.Request( |
|
f"{RAPIDMLX_HOST}/v1/completions", |
|
data=json.dumps(payload).encode(), |
|
headers={"Content-Type": "application/json"}, |
|
) |
|
|
|
started = time.perf_counter() |
|
first = last = 0.0 |
|
usage: dict[str, Any] = {} |
|
finish = "" |
|
with urllib.request.urlopen(request, timeout=3600) as response: |
|
for raw_line in response: |
|
line = raw_line.decode().strip() |
|
if not line.startswith("data:"): |
|
continue |
|
body = line[5:].strip() |
|
if body == "[DONE]": |
|
break |
|
message = json.loads(body) |
|
for choice in message.get("choices") or []: |
|
if choice.get("text"): |
|
now = time.perf_counter() |
|
if not first: |
|
first = now |
|
last = now |
|
if choice.get("finish_reason"): |
|
finish = choice["finish_reason"] |
|
if message.get("usage"): |
|
usage = message["usage"] |
|
total = time.perf_counter() - started |
|
|
|
gen = usage.get("completion_tokens") or 0 |
|
return { |
|
"prompt_tokens": usage.get("prompt_tokens") or 0, |
|
"gen_tokens": gen, |
|
"native_prefill_tps": 0.0, |
|
"native_decode_tps": 0.0, |
|
"has_native": False, |
|
"client_ttft_s": (first - started) if first else 0.0, |
|
"client_decode_tps": _client_decode(gen, first, last), |
|
"client_total_s": total, |
|
"peak_memory_gb": 0.0, # Not exposed per request; recorded once in environment. |
|
"finish_reason": finish, |
|
} |
|
|
|
|
|
def _client_decode(gen_tokens: int, first: float, last: float) -> float: |
|
""" |
|
Tokens after the first, over the time between the first and last. |
|
|
|
The first token is excluded because its arrival is gated by prefill, so |
|
including it reports a decode rate depressed by prompt processing. This is |
|
the standard client-side convention. |
|
""" |
|
span = last - first |
|
if gen_tokens < 2 or span <= 0: |
|
return 0.0 |
|
return (gen_tokens - 1) / span |
|
|
|
|
|
# --------------------------------------------------------------------------- |
|
# Rapid-MLX server checks |
|
# --------------------------------------------------------------------------- |
|
|
|
|
|
def rapidmlx_served_model() -> str | None: |
|
try: |
|
with urllib.request.urlopen(f"{RAPIDMLX_HOST}/v1/models", timeout=10) as response: |
|
data = json.loads(response.read()) |
|
entries = data.get("data") or [] |
|
return entries[0].get("id") if entries else None |
|
except Exception: |
|
return None |
|
|
|
|
|
def rapidmlx_metric(name: str) -> float: |
|
"""Read one Prometheus counter, for the cache-defeat check.""" |
|
try: |
|
with urllib.request.urlopen(f"{RAPIDMLX_HOST}/metrics", timeout=10) as response: |
|
for line in response.read().decode().splitlines(): |
|
if line.startswith("#") or not line.strip(): |
|
continue |
|
key, _, value = line.rpartition(" ") |
|
if key.split("{")[0].strip() == name: |
|
return float(value) |
|
except Exception: |
|
pass |
|
return 0.0 |
|
|
|
|
|
# --------------------------------------------------------------------------- |
|
# Cache-defeat verification |
|
# --------------------------------------------------------------------------- |
|
|
|
|
|
def verify_cache_defeat(model_key: str, target: int, gen_tokens: int) -> dict[str, Any]: |
|
""" |
|
The README's Ollama check, applied to all three engines. |
|
|
|
Same prompt twice, then a fresh nonce. A second run much faster than the |
|
first is a cache hit. The nonce run must return to roughly the first run's |
|
cost; if it stays fast, the nonce is not defeating that engine's cache and |
|
every prefill number in the run is a lookup rather than work. |
|
""" |
|
spec = MODELS[model_key] |
|
print(f"\n cache-defeat check on {spec['label']} at {target} prompt tokens", flush=True) |
|
|
|
mlx = MLX(spec["mlx"]) |
|
fixed = build_prompt(mlx.tokenizer, target, nonce="[fixed-prefix] ") |
|
report: dict[str, Any] = {} |
|
|
|
for engine in ENGINES: |
|
def once(prompt: str) -> float: |
|
if engine == "ollama": |
|
data = run_ollama(spec["ollama"], prompt, gen_tokens, target + gen_tokens + 512) |
|
elif engine == "mlx": |
|
data = mlx.run(prompt, gen_tokens) |
|
else: |
|
data = run_rapidmlx(rapidmlx_served_model() or "", prompt, gen_tokens) |
|
return data["client_ttft_s"] |
|
|
|
first = once(fixed) |
|
repeat = once(fixed) |
|
fresh = once(build_prompt(mlx.tokenizer, target, nonce=f"[{uuid4().hex}] ")) |
|
|
|
cached = repeat < first * 0.6 |
|
restored = fresh > repeat * 1.4 if cached else True |
|
report[engine] = { |
|
"ttft_first_s": round(first, 4), |
|
"ttft_repeat_s": round(repeat, 4), |
|
"ttft_fresh_nonce_s": round(fresh, 4), |
|
"repeat_was_cache_hit": cached, |
|
"nonce_restored_cost": restored, |
|
} |
|
verdict = "no cache to defeat" if not cached else ( |
|
"nonce defeats it" if restored else "NONCE FAILED" |
|
) |
|
print( |
|
f" {engine:<9} first {first:6.3f}s repeat {repeat:6.3f}s " |
|
f"fresh {fresh:6.3f}s {verdict}", |
|
flush=True, |
|
) |
|
|
|
del mlx |
|
return report |
|
|
|
|
|
# --------------------------------------------------------------------------- |
|
# Orchestration |
|
# --------------------------------------------------------------------------- |
|
|
|
|
|
def measure( |
|
model_key: str, |
|
prompt_targets: Iterable[int], |
|
reps: int, |
|
gen_tokens: int, |
|
) -> list[Run]: |
|
spec = MODELS[model_key] |
|
results: list[Run] = [] |
|
|
|
served = rapidmlx_served_model() |
|
if served is None: |
|
raise SystemExit( |
|
f"Rapid-MLX is not answering at {RAPIDMLX_HOST}. Start it with:\n" |
|
f" .venv-rapidmlx/bin/rapid-mlx --no-telemetry serve {spec['mlx']} " |
|
f"--host 127.0.0.1 --port 8080 --no-thinking" |
|
) |
|
if served != spec["mlx"]: |
|
raise SystemExit( |
|
f"Rapid-MLX is serving '{served}' but this cell needs '{spec['mlx']}'.\n" |
|
"Restart the server on the right repo. Measuring a different checkpoint " |
|
"would be the exact defect the article's MoE re-run exists to avoid." |
|
) |
|
|
|
print(f" loading vanilla MLX {spec['mlx']} ...", flush=True) |
|
mlx = MLX(spec["mlx"]) |
|
tokenizer = mlx.tokenizer |
|
|
|
if not ollama_has(spec["ollama"]): |
|
ollama_pull(spec["ollama"]) |
|
|
|
for target in prompt_targets: |
|
num_ctx = max(2048, target + gen_tokens + 512) |
|
|
|
# One discarded warm-up per engine per cell, each with its own nonce so |
|
# it cannot prime a cache for the measured runs. |
|
for engine in ENGINES: |
|
warm = build_prompt(tokenizer, target, nonce=f"[{uuid4().hex}] ") |
|
try: |
|
if engine == "ollama": |
|
run_ollama(spec["ollama"], warm, min(8, gen_tokens), num_ctx) |
|
elif engine == "mlx": |
|
mlx.run(warm, min(8, gen_tokens)) |
|
else: |
|
run_rapidmlx(served, warm, min(8, gen_tokens)) |
|
except Exception as exc: # noqa: BLE001 - a failed warm-up is not fatal |
|
print(f" warm-up failed for {engine}: {exc}", flush=True) |
|
|
|
for rep in range(1, reps + 1): |
|
# One fresh prompt per repetition, shared by all three engines. Sharing |
|
# within a repetition is what makes the comparison fair; it is reuse |
|
# across repetitions that has to be defeated. |
|
prompt = build_prompt(tokenizer, target, nonce=f"[{uuid4().hex}] ") |
|
|
|
# Rotate the order so each engine takes each position equally across |
|
# three repetitions, sharing thermal drift rather than charging it to |
|
# whichever engine always ran last. |
|
shift = (rep - 1) % len(ENGINES) |
|
order = ENGINES[shift:] + ENGINES[:shift] |
|
|
|
seen_prompt_tokens: dict[str, int] = {} |
|
for engine in order: |
|
result = Run( |
|
model_key=model_key, engine=engine, prompt_target=target, rep=rep |
|
) |
|
try: |
|
if engine == "ollama": |
|
data = run_ollama(spec["ollama"], prompt, gen_tokens, num_ctx) |
|
elif engine == "mlx": |
|
data = mlx.run(prompt, gen_tokens) |
|
else: |
|
data = run_rapidmlx(served, prompt, gen_tokens) |
|
for key, value in data.items(): |
|
setattr(result, key, value) |
|
if result.gen_tokens < MIN_TRUSTWORTHY_GEN_TOKENS: |
|
result.warnings.append( |
|
f"only {result.gen_tokens} tokens generated; decode rate is noisy" |
|
) |
|
seen_prompt_tokens[engine] = result.prompt_tokens |
|
except Exception as exc: # noqa: BLE001 - record and continue |
|
result.error = f"{type(exc).__name__}: {exc}" |
|
|
|
status = result.error or ( |
|
f"ttft {result.client_ttft_s:6.3f}s " |
|
f"decode {result.client_decode_tps:6.2f} tok/s " |
|
f"native {result.native_decode_tps:6.2f} " |
|
f"({result.prompt_tokens} in, {result.gen_tokens} out)" |
|
) |
|
print(f" {engine:<9} {target:>6} rep{rep} {status}", flush=True) |
|
results.append(result) |
|
|
|
# Token parity is the harness's own validity test. If the engines were |
|
# not asked the same question, the cell means nothing. |
|
distinct = set(seen_prompt_tokens.values()) |
|
if len(distinct) > 1: |
|
message = f"prompt token mismatch across engines: {seen_prompt_tokens}" |
|
print(f" WARNING {message}", flush=True) |
|
for result in results[-len(order):]: |
|
result.warnings.append(message) |
|
|
|
return results |
|
|
|
|
|
def summarize(runs: list[Run]) -> list[dict[str, Any]]: |
|
grouped: dict[tuple[str, str, int], list[Run]] = {} |
|
for run in runs: |
|
if run.error: |
|
continue |
|
grouped.setdefault((run.model_key, run.engine, run.prompt_target), []).append(run) |
|
|
|
rows = [] |
|
for (model_key, engine, target), items in sorted(grouped.items()): |
|
def stat(attr: str) -> dict[str, float]: |
|
values = [getattr(item, attr) for item in items] |
|
return { |
|
"median": round(statistics.median(values), 3), |
|
"min": round(min(values), 3), |
|
"max": round(max(values), 3), |
|
} |
|
|
|
rows.append( |
|
{ |
|
"model_key": model_key, |
|
"label": MODELS[model_key]["label"], |
|
"engine": engine, |
|
"prompt_target": target, |
|
"prompt_tokens": statistics.median([i.prompt_tokens for i in items]), |
|
"gen_tokens": statistics.median([i.gen_tokens for i in items]), |
|
"reps": len(items), |
|
"client_ttft_s": stat("client_ttft_s"), |
|
"client_decode_tps": stat("client_decode_tps"), |
|
"client_total_s": stat("client_total_s"), |
|
"native_prefill_tps": stat("native_prefill_tps"), |
|
"native_decode_tps": stat("native_decode_tps"), |
|
"has_native": items[0].has_native, |
|
"peak_memory_gb": max(i.peak_memory_gb for i in items), |
|
} |
|
) |
|
return rows |
|
|
|
|
|
def markdown_table(rows: list[dict[str, Any]]) -> str: |
|
out = [ |
|
"| Model | Prompt | Engine | TTFT s (client) | Decode tok/s (client) | Decode tok/s (native) |", |
|
"| --- | --- | --- | --- | --- | --- |", |
|
] |
|
for row in rows: |
|
native = ( |
|
f"{row['native_decode_tps']['median']:.2f}" if row["has_native"] else "not exposed" |
|
) |
|
out.append( |
|
f"| {row['label']} | {row['prompt_target']} | {row['engine']} " |
|
f"| {row['client_ttft_s']['median']:.3f} " |
|
f"| {row['client_decode_tps']['median']:.2f} | {native} |" |
|
) |
|
return "\n".join(out) |
|
|
|
|
|
def main() -> int: |
|
global RAPIDMLX_HOST |
|
|
|
parser = argparse.ArgumentParser(description=__doc__) |
|
parser.add_argument("--models", nargs="+", default=DEFAULT_MODELS, choices=list(MODELS)) |
|
parser.add_argument("--prompt-tokens", nargs="+", type=int, default=DEFAULT_PROMPT_TOKENS) |
|
parser.add_argument("--gen-tokens", type=int, default=DEFAULT_GEN_TOKENS) |
|
parser.add_argument("--reps", type=int, default=DEFAULT_REPS) |
|
parser.add_argument("--output", default="results-3way.json") |
|
parser.add_argument("--rapidmlx-host", default=RAPIDMLX_HOST) |
|
parser.add_argument( |
|
"--verify-cache-defeat", |
|
action="store_true", |
|
help="Run the same-prompt-twice-then-nonce check on all three engines and exit.", |
|
) |
|
args = parser.parse_args() |
|
|
|
RAPIDMLX_HOST = args.rapidmlx_host |
|
|
|
if not bench.ollama_available(): |
|
print("Ollama is not answering. Start it and retry.") |
|
return 1 |
|
|
|
env = environment() |
|
env["rapidmlx"] = _rapidmlx_version() |
|
env["rapidmlx_served_model"] = rapidmlx_served_model() |
|
env["timing_basis"] = ( |
|
"client-side for all engines; native counters recorded where exposed. " |
|
"rapidmlx exposes no native prefill or decode timing." |
|
) |
|
print(json.dumps(env, indent=2), flush=True) |
|
|
|
if args.verify_cache_defeat: |
|
checks = { |
|
key: verify_cache_defeat(key, max(args.prompt_tokens), args.gen_tokens) |
|
for key in args.models |
|
} |
|
with open(args.output, "w") as handle: |
|
json.dump({"environment": env, "cache_defeat": checks}, handle, indent=2) |
|
print(f"\nwrote {args.output}") |
|
return 0 |
|
|
|
runs: list[Run] = [] |
|
for key in args.models: |
|
print(f"\n{MODELS[key]['label']}", flush=True) |
|
runs.extend(measure(key, args.prompt_tokens, args.reps, args.gen_tokens)) |
|
|
|
rows = summarize(runs) |
|
payload = { |
|
"environment": env, |
|
"config": { |
|
"engines": ENGINES, |
|
"models": args.models, |
|
"prompt_tokens": args.prompt_tokens, |
|
"gen_tokens": args.gen_tokens, |
|
"reps": args.reps, |
|
}, |
|
"summary": rows, |
|
"runs": [asdict(run) for run in runs], |
|
"generated_at": datetime.now(timezone.utc).isoformat(), |
|
} |
|
with open(args.output, "w") as handle: |
|
json.dump(payload, handle, indent=2) |
|
|
|
print("\n" + markdown_table(rows)) |
|
print(f"\nwrote {args.output}") |
|
return 0 |
|
|
|
|
|
def _rapidmlx_version() -> str: |
|
""" |
|
Read the version from the sibling venv's dist-info rather than this one. |
|
|
|
Rapid-MLX is deliberately installed in .venv-rapidmlx so that its own |
|
mlx/mlx-lm pins cannot disturb the pinned mlx-lm==0.31.3 this benchmark |
|
measures vanilla MLX with. So importlib.metadata in *this* interpreter |
|
would never find it. |
|
""" |
|
import glob |
|
import os |
|
|
|
here = os.path.dirname(os.path.abspath(__file__)) |
|
pattern = os.path.join(here, ".venv-rapidmlx", "lib", "python*", "site-packages", "rapid_mlx-*.dist-info") |
|
for path in sorted(glob.glob(pattern)): |
|
name = os.path.basename(path) |
|
return name[len("rapid_mlx-") : -len(".dist-info")] |
|
return "unknown" |
|
|
|
|
|
if __name__ == "__main__": |
|
raise SystemExit(main()) |
