Skip to content

Instantly share code, notes, and snippets.

@Anthony-Hoo
Created June 12, 2026 15:26
Show Gist options
  • Select an option

  • Save Anthony-Hoo/ae5f20b58082ca18683f559e3a91d86f to your computer and use it in GitHub Desktop.

Select an option

Save Anthony-Hoo/ae5f20b58082ca18683f559e3a91d86f to your computer and use it in GitHub Desktop.
GPU_stress_test by pytorch
#!/usr/bin/env python3
# /// script
# dependencies = ["torch", "nvidia-ml-py"]
# ///
"""
PyTorch NVIDIA GPU stress test.
Default target: CUDA devices 2 and 3, 15 minutes, high GEMM load,
and near-full VRAM allocation with a configurable safety headroom.
"""
from __future__ import annotations
import argparse
import math
import multiprocessing as mp
import os
import queue
import sys
import time
import traceback
from statistics import mean
from typing import Any
MIB = 1024**2
GIB = 1024**3
def parse_gpu_list(value: str) -> list[int]:
try:
gpus = [int(x.strip()) for x in value.split(",") if x.strip() != ""]
except ValueError as exc:
raise argparse.ArgumentTypeError("--gpus must look like: 2,3") from exc
if not gpus:
raise argparse.ArgumentTypeError("--gpus cannot be empty")
if any(g < 0 for g in gpus):
raise argparse.ArgumentTypeError("GPU ids must be >= 0")
if len(set(gpus)) != len(gpus):
raise argparse.ArgumentTypeError("GPU ids must be unique")
return gpus
def gib(n: float | int) -> float:
return float(n) / GIB
def bar(value: float, max_value: float = 100.0, width: int = 34) -> str:
if max_value <= 0:
max_value = 1.0
value = max(0.0, min(float(value), max_value))
fill = int(round(width * value / max_value))
return "[" + ("#" * fill).ljust(width, "-") + "]"
def downsample(values: list[float], max_points: int) -> list[float]:
if len(values) <= max_points:
return values
bucket = len(values) / max_points
out: list[float] = []
for i in range(max_points):
start = int(i * bucket)
end = int((i + 1) * bucket)
part = values[start:max(end, start + 1)]
out.append(mean(part))
return out
def print_series(title: str, values: list[float], suffix: str, *, max_value: float | None = None) -> None:
if not values:
print(f" {title}: no samples")
return
vals = downsample(values, 24)
scale = max_value if max_value is not None else max(max(vals), 1e-9)
print(f" {title}:")
for i, v in enumerate(vals):
print(f" {i:02d} {bar(v, scale, 28)} {v:8.2f}{suffix}")
def nvml_physical_index(torch_gpu: int) -> int:
"""Map torch CUDA index to NVML physical index when CUDA_VISIBLE_DEVICES is numeric."""
visible = os.environ.get("CUDA_VISIBLE_DEVICES")
if not visible:
return torch_gpu
parts = [p.strip() for p in visible.split(",") if p.strip()]
if torch_gpu < len(parts) and parts[torch_gpu].isdigit():
return int(parts[torch_gpu])
return torch_gpu
def nvml_snapshot(torch_gpu: int) -> dict[str, float | int | None]:
try:
import pynvml # type: ignore
pynvml.nvmlInit()
handle = pynvml.nvmlDeviceGetHandleByIndex(nvml_physical_index(torch_gpu))
util = pynvml.nvmlDeviceGetUtilizationRates(handle)
mem = pynvml.nvmlDeviceGetMemoryInfo(handle)
try:
temp_c = pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU)
except Exception:
temp_c = None
try:
power_w = pynvml.nvmlDeviceGetPowerUsage(handle) / 1000.0
except Exception:
power_w = None
return {
"nvml_ok": 1,
"gpu_util_pct": float(util.gpu),
"mem_util_pct": float(mem.used) * 100.0 / float(mem.total),
"nvml_mem_used_gib": gib(mem.used),
"nvml_mem_total_gib": gib(mem.total),
"temp_c": temp_c,
"power_w": power_w,
}
except Exception:
return {
"nvml_ok": 0,
"gpu_util_pct": None,
"mem_util_pct": None,
"nvml_mem_used_gib": None,
"nvml_mem_total_gib": None,
"temp_c": None,
"power_w": None,
}
def choose_dtype(torch: Any, gpu: int, dtype_name: str) -> Any:
name = dtype_name.lower()
if name == "auto":
major, _minor = torch.cuda.get_device_capability(gpu)
return torch.float16 if major >= 7 else torch.float32
if name in {"fp16", "float16", "half"}:
return torch.float16
if name in {"fp32", "float32"}:
return torch.float32
if name in {"bf16", "bfloat16"}:
if hasattr(torch.cuda, "is_bf16_supported") and not torch.cuda.is_bf16_supported():
raise RuntimeError(f"GPU{gpu} / this PyTorch build does not report BF16 support")
return torch.bfloat16
raise RuntimeError(f"Unsupported dtype: {dtype_name}")
def dtype_name(torch: Any, dtype: Any) -> str:
if dtype is torch.float16:
return "float16"
if dtype is torch.float32:
return "float32"
if dtype is torch.bfloat16:
return "bfloat16"
return str(dtype)
def fit_matmul_size(n: int, dtype_bytes: int, free_bytes: int, headroom_bytes: int) -> int:
# Need A, B, C plus enough workspace/headroom. Round down to multiples of 256.
n = max(256, int(n // 256 * 256))
while n >= 512:
needed = 3 * n * n * dtype_bytes
if needed + headroom_bytes < int(free_bytes * 0.70):
return n
n = int((n * 0.80) // 256 * 256)
return max(256, n)
def allocate_compute_tensors(torch: Any, gpu: int, n: int, dtype: Any, headroom_bytes: int) -> tuple[Any, Any, Any, int]:
dtype_bytes = torch.empty((), dtype=dtype).element_size()
free_bytes, _total_bytes = torch.cuda.mem_get_info(gpu)
n = fit_matmul_size(n, dtype_bytes, free_bytes, headroom_bytes)
last_error = None
while n >= 256:
a = b = c = None
try:
scale = 1.0 / math.sqrt(max(n, 1))
a = torch.empty((n, n), device=f"cuda:{gpu}", dtype=dtype)
b = torch.empty((n, n), device=f"cuda:{gpu}", dtype=dtype)
c = torch.empty((n, n), device=f"cuda:{gpu}", dtype=dtype)
a.fill_(scale)
b.fill_(scale)
c.zero_()
torch.cuda.synchronize(gpu)
return a, b, c, n
except torch.cuda.OutOfMemoryError as exc:
last_error = exc
try:
del a, b, c
except Exception:
pass
torch.cuda.empty_cache()
n = int((n * 0.70) // 256 * 256)
raise RuntimeError(f"Could not allocate matmul tensors; last error: {last_error}")
def allocate_ballast(torch: Any, gpu: int, mem_fraction: float, headroom_mib: int, block_mib: int) -> tuple[list[Any], int]:
ballast: list[Any] = []
headroom_bytes = int(headroom_mib * MIB)
block_bytes = int(block_mib * MIB)
free_bytes, total_bytes = torch.cuda.mem_get_info(gpu)
target_free = max(int(total_bytes * (1.0 - mem_fraction)), headroom_bytes)
min_block = 16 * MIB
allocated = 0
while True:
free_bytes, _total_bytes = torch.cuda.mem_get_info(gpu)
remaining = free_bytes - target_free
if remaining <= min_block:
break
this_block = int(min(block_bytes, remaining))
this_block = max(min_block, this_block // MIB * MIB)
try:
t = torch.empty((this_block,), dtype=torch.uint8, device=f"cuda:{gpu}")
t.fill_(gpu & 0xFF)
ballast.append(t)
allocated += this_block
except torch.cuda.OutOfMemoryError:
torch.cuda.empty_cache()
block_bytes //= 2
if block_bytes < min_block:
break
torch.cuda.synchronize(gpu)
return ballast, allocated
def worker(gpu: int, args: dict[str, Any], q: mp.Queue) -> None:
try:
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import torch
torch.cuda.set_device(gpu)
torch.backends.cuda.matmul.allow_tf32 = True
try:
torch.set_float32_matmul_precision("high")
except Exception:
pass
if not torch.cuda.is_available():
raise RuntimeError("torch.cuda.is_available() is False")
count = torch.cuda.device_count()
if gpu >= count:
raise RuntimeError(f"Requested cuda:{gpu}, but PyTorch sees only {count} CUDA device(s)")
props = torch.cuda.get_device_properties(gpu)
dtype = choose_dtype(torch, gpu, args["dtype"])
dtype_str = dtype_name(torch, dtype)
torch.cuda.empty_cache()
torch.cuda.reset_peak_memory_stats(gpu)
headroom_bytes = int(args["headroom_mib"] * MIB)
a, b, c, n = allocate_compute_tensors(torch, gpu, args["matmul_size"], dtype, headroom_bytes)
# Warm up cuBLAS/PyTorch first, so internal workspaces are allocated before we fill VRAM.
for _ in range(max(1, args["warmup_iters"])):
torch.mm(a, b, out=c)
torch.cuda.synchronize(gpu)
ballast, ballast_bytes = allocate_ballast(
torch,
gpu,
args["mem_fraction"],
args["headroom_mib"],
args["block_mib"],
)
free_bytes, total_bytes = torch.cuda.mem_get_info(gpu)
q.put(
{
"type": "ready",
"gpu": gpu,
"name": props.name,
"dtype": dtype_str,
"matmul_size": n,
"ballast_gib": gib(ballast_bytes),
"free_gib": gib(free_bytes),
"total_gib": gib(total_bytes),
}
)
start = time.monotonic()
end = start + float(args["minutes"]) * 60.0
last = start
last_iters = 0
iters = 0
sample_interval = float(args["sample_interval"])
sync_every = max(1, int(args["sync_every"]))
flops_per_mm = 2.0 * float(n) ** 3
while True:
now = time.monotonic()
if now >= end:
break
for _ in range(sync_every):
torch.mm(a, b, out=c)
iters += sync_every
torch.cuda.synchronize(gpu)
now = time.monotonic()
if now - last >= sample_interval:
dt = now - last
di = iters - last_iters
free_bytes, total_bytes = torch.cuda.mem_get_info(gpu)
allocated = torch.cuda.memory_allocated(gpu)
reserved = torch.cuda.memory_reserved(gpu)
nvml = nvml_snapshot(gpu)
tflops = (di * flops_per_mm) / max(dt, 1e-9) / 1e12
q.put(
{
"type": "sample",
"gpu": gpu,
"elapsed": now - start,
"iter_s": di / max(dt, 1e-9),
"tflops": tflops,
"torch_alloc_gib": gib(allocated),
"torch_reserved_gib": gib(reserved),
"torch_free_gib": gib(free_bytes),
"torch_total_gib": gib(total_bytes),
**nvml,
}
)
last = now
last_iters = iters
torch.cuda.synchronize(gpu)
elapsed = time.monotonic() - start
peak_alloc = torch.cuda.max_memory_allocated(gpu)
peak_reserved = torch.cuda.max_memory_reserved(gpu)
q.put(
{
"type": "done",
"gpu": gpu,
"elapsed": elapsed,
"iters": iters,
"name": props.name,
"dtype": dtype_str,
"matmul_size": n,
"ballast_gib": gib(ballast_bytes),
"peak_alloc_gib": gib(peak_alloc),
"peak_reserved_gib": gib(peak_reserved),
}
)
# Keep references alive until after the final report.
_ = (a, b, c, ballast)
except Exception:
q.put({"type": "error", "gpu": gpu, "traceback": traceback.format_exc()})
sys.exit(1)
def print_final_report(gpus: list[int], samples: dict[int, list[dict[str, Any]]], done: dict[int, dict[str, Any]]) -> None:
print("\n" + "=" * 92)
print("FINAL CLI CHART")
print("=" * 92)
for gpu in gpus:
ss = samples.get(gpu, [])
d = done.get(gpu, {})
print(
f"\nGPU{gpu} | {d.get('name', 'unknown')} | "
f"dtype={d.get('dtype', '?')} | "
f"matmul={d.get('matmul_size', '?')}x{d.get('matmul_size', '?')}"
)
print(
f" elapsed={float(d.get('elapsed', 0.0)):.1f}s "
f"iterations={int(d.get('iters', 0))} "
f"ballast={float(d.get('ballast_gib', 0.0)):.2f} GiB"
)
print(f" peak torch allocated : {float(d.get('peak_alloc_gib', 0.0)):.2f} GiB")
print(f" peak torch reserved : {float(d.get('peak_reserved_gib', 0.0)):.2f} GiB")
if not ss:
print(" no runtime samples collected")
continue
mem_pcts = [float(s["mem_util_pct"]) for s in ss if s.get("mem_util_pct") is not None]
if not mem_pcts:
mem_pcts = [
100.0 * float(s["torch_reserved_gib"]) / max(float(s["torch_total_gib"]), 1e-9)
for s in ss
]
util_pcts = [float(s["gpu_util_pct"]) for s in ss if s.get("gpu_util_pct") is not None]
tflops = [float(s["tflops"]) for s in ss]
iter_s = [float(s["iter_s"]) for s in ss]
temps = [float(s["temp_c"]) for s in ss if s.get("temp_c") is not None]
powers = [float(s["power_w"]) for s in ss if s.get("power_w") is not None]
avg_mem = mean(mem_pcts)
print(f" avg memory usage : {bar(avg_mem)} {avg_mem:6.2f}%")
if util_pcts:
avg_util = mean(util_pcts)
print(f" avg GPU util : {bar(avg_util)} {avg_util:6.2f}%")
print(f" avg observed TFLOP/s : {mean(tflops):8.2f} peak sample: {max(tflops):8.2f}")
print(f" avg iter/s : {mean(iter_s):8.2f} peak sample: {max(iter_s):8.2f}")
if temps:
print(f" temp C avg/max : {mean(temps):8.1f} / {max(temps):.1f}")
if powers:
print(f" power W avg/max : {mean(powers):8.1f} / {max(powers):.1f}")
print_series("memory %", mem_pcts, "%", max_value=100.0)
if util_pcts:
print_series("GPU util %", util_pcts, "%", max_value=100.0)
print_series("observed TFLOP/s", tflops, "")
def positive_float(value: str) -> float:
v = float(value)
if v <= 0:
raise argparse.ArgumentTypeError("must be > 0")
return v
def fraction(value: str) -> float:
v = float(value)
if not (0.10 <= v < 1.0):
raise argparse.ArgumentTypeError("must be in [0.10, 1.0)")
return v
def main() -> int:
parser = argparse.ArgumentParser(description="Stress test selected NVIDIA GPUs with PyTorch GEMM + VRAM ballast.")
parser.add_argument("--gpus", type=parse_gpu_list, default="2,3", help="CUDA device ids, e.g. 2,3. Default: 2,3")
parser.add_argument("--minutes", type=positive_float, default=15.0, help="Run duration in minutes. Default: 15")
parser.add_argument("--mem-fraction", type=fraction, default=0.94, help="Target fraction of total VRAM to occupy. Default: 0.94")
parser.add_argument("--headroom-mib", type=int, default=768, help="Minimum free VRAM left per GPU. Default: 768")
parser.add_argument("--block-mib", type=int, default=512, help="VRAM ballast allocation block size. Default: 512")
parser.add_argument("--matmul-size", type=int, default=8192, help="Square GEMM size N for NxN @ NxN. Default: 8192")
parser.add_argument(
"--dtype",
default="auto",
choices=["auto", "fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],
help="GEMM dtype. Default: auto",
)
parser.add_argument("--sample-interval", type=positive_float, default=5.0, help="Seconds between samples. Default: 5")
parser.add_argument("--sync-every", type=int, default=8, help="Synchronize after this many matmuls. Default: 8")
parser.add_argument("--warmup-iters", type=int, default=8, help="Warmup matmuls before VRAM ballast. Default: 8")
args_ns = parser.parse_args()
args = vars(args_ns)
if args["headroom_mib"] < 128:
print("Refusing --headroom-mib < 128 because it makes OOM much more likely.", file=sys.stderr)
return 2
if args["block_mib"] < 16:
print("Refusing --block-mib < 16.", file=sys.stderr)
return 2
if os.environ.get("CUDA_VISIBLE_DEVICES"):
print(
"WARNING: CUDA_VISIBLE_DEVICES is set. PyTorch CUDA ids are relative to the visible device list. "
"Unset it if you want physical GPU ids 2 and 3."
)
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
try:
import torch
if not torch.cuda.is_available():
print("ERROR: PyTorch CUDA is not available. Check NVIDIA driver and CUDA-enabled torch wheel.", file=sys.stderr)
return 1
count = torch.cuda.device_count()
bad = [g for g in args["gpus"] if g >= count]
if bad:
print(f"ERROR: requested GPU(s) {bad}, but PyTorch sees only {count} CUDA device(s).", file=sys.stderr)
return 1
print(f"PyTorch {torch.__version__}; CUDA available; device_count={count}")
for g in args["gpus"]:
p = torch.cuda.get_device_properties(g)
print(f" GPU{g}: {p.name}, total VRAM={gib(p.total_memory):.2f} GiB")
except Exception as exc:
print(f"ERROR during PyTorch CUDA validation: {exc}", file=sys.stderr)
return 1
ctx = mp.get_context("spawn")
q: mp.Queue = ctx.Queue()
procs: list[mp.Process] = []
for gpu in args["gpus"]:
p = ctx.Process(target=worker, args=(gpu, args, q), daemon=False)
p.start()
procs.append(p)
samples: dict[int, list[dict[str, Any]]] = {g: [] for g in args["gpus"]}
done: dict[int, dict[str, Any]] = {}
errors: dict[int, str] = {}
def terminate_children() -> None:
for p in procs:
if p.is_alive():
p.terminate()
for p in procs:
p.join(timeout=5)
try:
while len(done) + len(errors) < len(args["gpus"]):
try:
msg = q.get(timeout=1.0)
except queue.Empty:
if all(not p.is_alive() for p in procs):
break
continue
typ = msg.get("type")
gpu = int(msg.get("gpu", -1))
if typ == "ready":
print(
f"READY GPU{gpu}: {msg['name']} dtype={msg['dtype']} matmul={msg['matmul_size']} "
f"ballast={msg['ballast_gib']:.2f}GiB free={msg['free_gib']:.2f}/{msg['total_gib']:.2f}GiB"
)
elif typ == "sample":
samples[gpu].append(msg)
util = msg.get("gpu_util_pct")
util_s = "n/a" if util is None else f"{float(util):5.1f}%"
mem = msg.get("mem_util_pct")
if mem is None:
mem = 100.0 * float(msg["torch_reserved_gib"]) / max(float(msg["torch_total_gib"]), 1e-9)
temp = msg.get("temp_c")
temp_s = "n/a" if temp is None else f"{float(temp):4.0f}C"
power = msg.get("power_w")
power_s = "n/a" if power is None else f"{float(power):5.0f}W"
print(
f"GPU{gpu} t={msg['elapsed']:7.1f}s "
f"util={util_s} mem={float(mem):5.1f}% "
f"torch={msg['torch_reserved_gib']:.2f}/{msg['torch_total_gib']:.2f}GiB "
f"iter/s={msg['iter_s']:7.2f} TFLOP/s={msg['tflops']:8.2f} "
f"temp={temp_s} power={power_s}"
)
elif typ == "done":
done[gpu] = msg
print(f"DONE GPU{gpu}: elapsed={msg['elapsed']:.1f}s iters={msg['iters']}")
elif typ == "error":
errors[gpu] = str(msg.get("traceback", "unknown error"))
print(f"ERROR GPU{gpu}:\n{errors[gpu]}", file=sys.stderr)
except KeyboardInterrupt:
print("\nInterrupted; terminating workers...", file=sys.stderr)
terminate_children()
return 130
finally:
for p in procs:
p.join(timeout=10)
if errors:
terminate_children()
print("\nAt least one GPU worker failed; no success chart generated for failed workers.", file=sys.stderr)
for gpu, tb in errors.items():
print(f"\n--- GPU{gpu} traceback ---\n{tb}", file=sys.stderr)
print_final_report(args["gpus"], samples, done)
return 1
print_final_report(args["gpus"], samples, done)
print("\nSUCCESS: all requested GPU workers completed without reporting an exception.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
uv run python ./gpu_stress_torch.py --gpus 2,3 --minutes 15
PyTorch 2.11.0+cu130; CUDA available; device_count=4
  GPU2: Tesla T10, total VRAM=15.56 GiB
  GPU3: Tesla T10, total VRAM=15.56 GiB
READY GPU2: Tesla T10 dtype=float16 matmul=8192 ballast=14.06GiB free=0.93/15.56GiB
READY GPU3: Tesla T10 dtype=float16 matmul=8192 ballast=14.06GiB free=0.93/15.56GiB
GPU3 t=    5.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  69.87 TFLOP/s=   76.82 temp=  46C power=  144W
GPU2 t=    5.1s util= 99.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  67.93 TFLOP/s=   74.69 temp=  47C power=  144W
GPU3 t=   10.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  69.26 TFLOP/s=   76.15 temp=  49C power=  142W
GPU2 t=   10.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  67.45 TFLOP/s=   74.16 temp=  51C power=  140W
GPU3 t=   15.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  69.12 TFLOP/s=   76.00 temp=  52C power=  145W
GPU2 t=   15.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  67.38 TFLOP/s=   74.08 temp=  54C power=  147W
GPU3 t=   20.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  68.62 TFLOP/s=   75.44 temp=  55C power=  146W
GPU2 t=   20.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  66.97 TFLOP/s=   73.63 temp=  57C power=  143W
GPU3 t=   25.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  67.95 TFLOP/s=   74.71 temp=  58C power=  147W
GPU2 t=   25.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  66.68 TFLOP/s=   73.32 temp=  60C power=  146W
GPU2 t=   30.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  66.72 TFLOP/s=   73.36 temp=  63C power=  146W
GPU3 t=   30.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  67.73 TFLOP/s=   74.47 temp=  61C power=  146W
GPU3 t=   35.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  67.05 TFLOP/s=   73.72 temp=  63C power=  143W
GPU2 t=   35.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  66.50 TFLOP/s=   73.12 temp=  66C power=  147W
GPU3 t=   40.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  66.68 TFLOP/s=   73.31 temp=  65C power=  149W
GPU2 t=   40.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  66.18 TFLOP/s=   72.77 temp=  68C power=  147W
GPU3 t=   45.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  66.00 TFLOP/s=   72.57 temp=  67C power=  144W
GPU2 t=   45.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  65.86 TFLOP/s=   72.42 temp=  70C power=  148W
GPU3 t=   50.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  65.95 TFLOP/s=   72.52 temp=  69C power=  148W
GPU2 t=   50.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  65.94 TFLOP/s=   72.50 temp=  72C power=  150W
GPU3 t=   55.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  65.04 TFLOP/s=   71.51 temp=  70C power=  143W
GPU2 t=   55.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  65.62 TFLOP/s=   72.15 temp=  73C power=  143W
GPU3 t=   60.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.86 TFLOP/s=   71.31 temp=  71C power=  142W
GPU2 t=   60.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  65.23 TFLOP/s=   71.72 temp=  75C power=  151W
GPU3 t=   65.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.68 TFLOP/s=   71.12 temp=  72C power=  144W
GPU2 t=   65.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.93 TFLOP/s=   71.39 temp=  76C power=  150W
GPU3 t=   70.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.44 TFLOP/s=   70.85 temp=  73C power=  147W
GPU2 t=   71.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.79 TFLOP/s=   71.23 temp=  77C power=  145W
GPU3 t=   76.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.04 TFLOP/s=   70.42 temp=  74C power=  140W
GPU2 t=   76.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.75 TFLOP/s=   71.20 temp=  77C power=  149W
GPU3 t=   81.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  63.97 TFLOP/s=   70.34 temp=  74C power=  139W
GPU2 t=   81.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.71 TFLOP/s=   71.15 temp=  78C power=  145W
GPU3 t=   86.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  63.54 TFLOP/s=   69.86 temp=  75C power=  140W
GPU2 t=   86.2s util= 99.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.33 TFLOP/s=   70.74 temp=  79C power=  147W
GPU3 t=   91.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  63.19 TFLOP/s=   69.48 temp=  75C power=  142W
GPU2 t=   91.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.15 TFLOP/s=   70.53 temp=  79C power=  150W
GPU3 t=   96.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  62.91 TFLOP/s=   69.18 temp=  76C power=  148W
GPU2 t=   96.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.18 TFLOP/s=   70.57 temp=  80C power=  147W
GPU3 t=  101.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  62.61 TFLOP/s=   68.84 temp=  77C power=  139W
GPU2 t=  101.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  64.21 TFLOP/s=   70.60 temp=  81C power=  149W
GPU3 t=  106.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  62.45 TFLOP/s=   68.67 temp=  77C power=  143W
GPU2 t=  106.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  63.98 TFLOP/s=   70.34 temp=  82C power=  149W
GPU2 t=  111.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  63.80 TFLOP/s=   70.15 temp=  82C power=  146W
GPU3 t=  111.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  62.43 TFLOP/s=   68.64 temp=  78C power=  141W
GPU2 t=  116.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  63.67 TFLOP/s=   70.00 temp=  83C power=  144W
GPU3 t=  116.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  62.18 TFLOP/s=   68.37 temp=  78C power=  141W
GPU2 t=  121.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  63.57 TFLOP/s=   69.90 temp=  83C power=  146W
GPU3 t=  121.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  61.90 TFLOP/s=   68.06 temp=  79C power=  142W
GPU2 t=  126.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  63.11 TFLOP/s=   69.39 temp=  83C power=  143W
GPU3 t=  126.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  61.62 TFLOP/s=   67.75 temp=  79C power=  141W
GPU2 t=  131.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  63.05 TFLOP/s=   69.32 temp=  84C power=  146W
GPU3 t=  131.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  61.08 TFLOP/s=   67.16 temp=  80C power=  145W
GPU2 t=  136.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  62.99 TFLOP/s=   69.26 temp=  85C power=  147W
GPU3 t=  136.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  61.43 TFLOP/s=   67.55 temp=  80C power=  127W
GPU2 t=  141.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  62.82 TFLOP/s=   69.07 temp=  85C power=  146W
GPU3 t=  141.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  61.38 TFLOP/s=   67.49 temp=  81C power=  142W
GPU3 t=  147.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  60.21 TFLOP/s=   66.20 temp=  81C power=  142W
GPU2 t=  147.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  62.78 TFLOP/s=   69.03 temp=  86C power=  147W
GPU3 t=  152.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  59.92 TFLOP/s=   65.88 temp=  81C power=  140W
GPU2 t=  152.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  62.79 TFLOP/s=   69.04 temp=  86C power=  145W
GPU3 t=  157.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  59.68 TFLOP/s=   65.62 temp=  82C power=  141W
GPU2 t=  157.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  62.53 TFLOP/s=   68.75 temp=  86C power=  146W
GPU2 t=  162.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  62.32 TFLOP/s=   68.52 temp=  87C power=  145W
GPU3 t=  162.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  59.57 TFLOP/s=   65.50 temp=  82C power=  139W
GPU2 t=  167.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  59.87 TFLOP/s=   65.83 temp=  86C power=  129W
GPU3 t=  167.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  59.36 TFLOP/s=   65.27 temp=  83C power=  139W
GPU2 t=  172.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.42 TFLOP/s=   64.24 temp=  86C power=  137W
GPU3 t=  172.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  59.23 TFLOP/s=   65.13 temp=  83C power=  140W
GPU2 t=  177.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.37 TFLOP/s=   64.17 temp=  86C power=  132W
GPU3 t=  177.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  59.08 TFLOP/s=   64.96 temp=  83C power=  104W
GPU2 t=  182.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.40 TFLOP/s=   64.21 temp=  86C power=  132W
GPU3 t=  182.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.86 TFLOP/s=   64.72 temp=  83C power=  140W
GPU3 t=  187.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.86 TFLOP/s=   64.72 temp=  84C power=  139W
GPU2 t=  187.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.40 TFLOP/s=   64.22 temp=  86C power=  124W
GPU3 t=  192.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.54 TFLOP/s=   64.37 temp=  84C power=  138W
GPU2 t=  192.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.23 TFLOP/s=   64.03 temp=  87C power=  132W
GPU2 t=  197.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.36 TFLOP/s=   64.17 temp=  87C power=  131W
GPU3 t=  197.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  57.98 TFLOP/s=   63.75 temp=  84C power=  139W
GPU2 t=  202.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.29 TFLOP/s=   64.09 temp=  87C power=  132W
GPU3 t=  202.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  57.78 TFLOP/s=   63.53 temp=  85C power=  137W
GPU2 t=  207.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.23 TFLOP/s=   64.02 temp=  87C power=  133W
GPU3 t=  208.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  57.76 TFLOP/s=   63.51 temp=  85C power=  139W
GPU2 t=  213.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  58.16 TFLOP/s=   63.95 temp=  87C power=  135W
GPU3 t=  213.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  57.51 TFLOP/s=   63.23 temp=  85C power=  138W
GPU3 t=  218.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  57.36 TFLOP/s=   63.06 temp=  86C power=  143W
GPU2 t=  218.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  55.64 TFLOP/s=   61.17 temp=  86C power=  121W
GPU3 t=  223.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  57.31 TFLOP/s=   63.01 temp=  86C power=  135W
GPU2 t=  223.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.62 TFLOP/s=   58.96 temp=  86C power=  127W
GPU3 t=  228.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  57.08 TFLOP/s=   62.76 temp=  86C power=  140W
GPU2 t=  228.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  55.28 TFLOP/s=   60.78 temp=  86C power=  124W
GPU3 t=  233.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  56.79 TFLOP/s=   62.44 temp=  86C power=  137W
GPU2 t=  233.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  55.49 TFLOP/s=   61.02 temp=  86C power=  127W
GPU2 t=  238.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  57.32 TFLOP/s=   63.02 temp=  87C power=  131W
GPU3 t=  238.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  56.64 TFLOP/s=   62.28 temp=  86C power=  137W
GPU2 t=  243.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  57.16 TFLOP/s=   62.85 temp=  87C power=  128W
GPU3 t=  243.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  56.56 TFLOP/s=   62.19 temp=  87C power=  138W
GPU2 t=  248.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  57.10 TFLOP/s=   62.78 temp=  87C power=  127W
GPU3 t=  248.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  56.37 TFLOP/s=   61.98 temp=  87C power=  136W
GPU2 t=  253.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  57.01 TFLOP/s=   62.69 temp=  87C power=  128W
GPU3 t=  253.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.04 TFLOP/s=   57.22 temp=  86C power=  118W
GPU2 t=  258.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.36 TFLOP/s=   58.67 temp=  86C power=  120W
GPU3 t=  258.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.41 TFLOP/s=   55.43 temp=  86C power=  125W
GPU2 t=  263.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.16 TFLOP/s=   58.45 temp=  86C power=  118W
GPU3 t=  263.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.85 TFLOP/s=   55.91 temp=  86C power=  128W
GPU2 t=  268.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.23 TFLOP/s=   58.53 temp=  86C power=  119W
GPU3 t=  268.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.10 TFLOP/s=   58.39 temp=  86C power=  129W
GPU2 t=  273.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.26 TFLOP/s=   58.56 temp=  86C power=  119W
GPU3 t=  273.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.99 TFLOP/s=   58.26 temp=  86C power=  134W
GPU2 t=  278.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.25 TFLOP/s=   58.55 temp=  86C power=  118W
GPU3 t=  279.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.03 TFLOP/s=   58.31 temp=  87C power=  132W
GPU2 t=  283.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  55.51 TFLOP/s=   61.03 temp=  86C power=  118W
GPU3 t=  284.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.99 TFLOP/s=   58.26 temp=  87C power=  129W
GPU2 t=  289.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  55.44 TFLOP/s=   60.96 temp=  87C power=  124W
GPU3 t=  289.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.89 TFLOP/s=   58.15 temp=  87C power=  125W
GPU2 t=  294.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  55.34 TFLOP/s=   60.85 temp=  87C power=  127W
GPU3 t=  294.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.26 TFLOP/s=   57.46 temp=  87C power=  111W
GPU2 t=  299.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  55.33 TFLOP/s=   60.84 temp=  87C power=  125W
GPU3 t=  299.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  45.04 TFLOP/s=   49.52 temp=  86C power=  126W
GPU2 t=  304.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.96 TFLOP/s=   58.23 temp=  86C power=  110W
GPU3 t=  304.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.33 TFLOP/s=   54.24 temp=  86C power=  123W
GPU2 t=  309.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.20 TFLOP/s=   55.20 temp=  86C power=  114W
GPU3 t=  309.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.30 TFLOP/s=   54.20 temp=  86C power=  124W
GPU2 t=  314.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.98 TFLOP/s=   57.15 temp=  86C power=  120W
GPU3 t=  314.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.31 TFLOP/s=   54.22 temp=  86C power=   94W
GPU2 t=  319.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.44 TFLOP/s=   58.76 temp=  86C power=  120W
GPU3 t=  319.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.33 TFLOP/s=   54.24 temp=  86C power=  123W
GPU2 t=  324.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.46 TFLOP/s=   58.78 temp=  87C power=  120W
GPU3 t=  324.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.34 TFLOP/s=   54.25 temp=  86C power=  125W
GPU3 t=  329.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.29 TFLOP/s=   54.19 temp=  86C power=  121W
GPU2 t=  329.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.41 TFLOP/s=   58.72 temp=  86C power=  120W
GPU3 t=  334.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.37 TFLOP/s=   54.28 temp=  86C power=  124W
GPU2 t=  334.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.37 TFLOP/s=   58.68 temp=  87C power=  120W
GPU3 t=  339.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.27 TFLOP/s=   54.18 temp=  86C power=  123W
GPU2 t=  339.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.34 TFLOP/s=   58.65 temp=  87C power=  121W
GPU3 t=  344.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.36 TFLOP/s=   54.27 temp=  86C power=  122W
GPU2 t=  345.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.35 TFLOP/s=   58.66 temp=  87C power=  120W
GPU3 t=  349.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.36 TFLOP/s=   54.27 temp=  86C power=  123W
GPU2 t=  350.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.39 TFLOP/s=   58.70 temp=  87C power=  120W
GPU3 t=  354.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.12 TFLOP/s=   55.11 temp=  86C power=  125W
GPU2 t=  355.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.36 TFLOP/s=   58.67 temp=  87C power=  120W
GPU3 t=  360.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.47 TFLOP/s=   56.59 temp=  86C power=  126W
GPU2 t=  360.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.25 TFLOP/s=   58.55 temp=  87C power=  114W
GPU3 t=  365.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.30 TFLOP/s=   56.40 temp=  87C power=  126W
GPU2 t=  365.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.27 TFLOP/s=   58.57 temp=  87C power=  120W
GPU3 t=  370.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.15 TFLOP/s=   56.24 temp=  87C power=  126W
GPU2 t=  370.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.24 TFLOP/s=   58.54 temp=  87C power=  120W
GPU3 t=  375.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.00 TFLOP/s=   56.07 temp=  87C power=  129W
GPU2 t=  375.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.25 TFLOP/s=   58.55 temp=  87C power=  118W
GPU3 t=  380.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.05 TFLOP/s=   56.13 temp=  87C power=  127W
GPU2 t=  380.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.23 TFLOP/s=   58.53 temp=  87C power=  121W
GPU3 t=  385.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.10 TFLOP/s=   52.88 temp=  86C power=   93W
GPU2 t=  385.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.26 TFLOP/s=   58.56 temp=  87C power=  120W
GPU3 t=  390.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  43.54 TFLOP/s=   47.88 temp=  86C power=  116W
GPU2 t=  390.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.85 TFLOP/s=   52.62 temp=  86C power=  107W
GPU3 t=  395.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.60 TFLOP/s=   53.44 temp=  86C power=  124W
GPU2 t=  396.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.14 TFLOP/s=   55.13 temp=  86C power=  116W
GPU3 t=  400.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.33 TFLOP/s=   54.24 temp=  86C power=  123W
GPU2 t=  401.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.81 TFLOP/s=   55.86 temp=  86C power=  115W
GPU3 t=  405.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.48 TFLOP/s=   54.40 temp=  86C power=  122W
GPU2 t=  406.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.05 TFLOP/s=   56.13 temp=  86C power=  120W
GPU3 t=  410.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.33 TFLOP/s=   54.24 temp=  86C power=  119W
GPU2 t=  411.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.29 TFLOP/s=   58.60 temp=  87C power=  120W
GPU3 t=  415.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.29 TFLOP/s=   54.20 temp=  87C power=  124W
GPU2 t=  416.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.29 TFLOP/s=   58.59 temp=  87C power=  121W
GPU3 t=  420.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.24 TFLOP/s=   54.14 temp=  86C power=  124W
GPU2 t=  421.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.32 TFLOP/s=   58.63 temp=  87C power=  121W
GPU3 t=  425.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.25 TFLOP/s=   54.15 temp=  86C power=  125W
GPU2 t=  426.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.28 TFLOP/s=   58.58 temp=  87C power=  119W
GPU3 t=  430.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.28 TFLOP/s=   54.19 temp=  86C power=  123W
GPU2 t=  431.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.34 TFLOP/s=   58.65 temp=  87C power=  123W
GPU3 t=  435.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.16 TFLOP/s=   54.05 temp=  87C power=  123W
GPU2 t=  436.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.11 TFLOP/s=   52.90 temp=  86C power=  111W
GPU3 t=  440.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.16 TFLOP/s=   54.05 temp=  87C power=  123W
GPU2 t=  441.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.68 TFLOP/s=   55.73 temp=  86C power=  116W
GPU3 t=  445.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.12 TFLOP/s=   54.01 temp=  87C power=  119W
GPU2 t=  446.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.78 TFLOP/s=   55.83 temp=  86C power=  114W
GPU3 t=  451.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.06 TFLOP/s=   53.95 temp=  87C power=  124W
GPU2 t=  451.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.76 TFLOP/s=   55.82 temp=  86C power=  115W
GPU3 t=  456.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.04 TFLOP/s=   53.92 temp=  87C power=  123W
GPU2 t=  456.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.71 TFLOP/s=   55.75 temp=  86C power=  111W
GPU3 t=  461.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.02 TFLOP/s=   53.90 temp=  87C power=  121W
GPU2 t=  461.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.71 TFLOP/s=   55.76 temp=  86C power=  113W
GPU3 t=  466.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.03 TFLOP/s=   53.91 temp=  87C power=  121W
GPU2 t=  467.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.77 TFLOP/s=   55.82 temp=  86C power=  112W
GPU3 t=  471.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.74 TFLOP/s=   53.59 temp=  87C power=  121W
GPU2 t=  472.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.78 TFLOP/s=   55.84 temp=  86C power=  113W
GPU3 t=  476.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.76 TFLOP/s=   53.62 temp=  87C power=  124W
GPU2 t=  477.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.80 TFLOP/s=   55.86 temp=  86C power=  116W
GPU3 t=  481.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.72 TFLOP/s=   53.57 temp=  87C power=  120W
GPU2 t=  482.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.11 TFLOP/s=   58.40 temp=  86C power=  125W
GPU3 t=  486.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.61 TFLOP/s=   53.45 temp=  87C power=  122W
GPU2 t=  487.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.31 TFLOP/s=   58.61 temp=  87C power=  120W
GPU3 t=  491.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.95 TFLOP/s=   53.83 temp=  87C power=  122W
GPU2 t=  492.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.16 TFLOP/s=   58.45 temp=  87C power=  120W
GPU3 t=  496.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.80 TFLOP/s=   53.65 temp=  87C power=  124W
GPU2 t=  497.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.21 TFLOP/s=   58.51 temp=  87C power=  121W
GPU3 t=  501.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.81 TFLOP/s=   53.67 temp=  87C power=  124W
GPU2 t=  502.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.26 TFLOP/s=   58.55 temp=  87C power=  119W
GPU3 t=  506.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.81 TFLOP/s=   53.66 temp=  87C power=  122W
GPU2 t=  507.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.72 TFLOP/s=   57.97 temp=  87C power=  109W
GPU3 t=  511.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.70 TFLOP/s=   53.54 temp=  87C power=  107W
GPU2 t=  512.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.37 TFLOP/s=   52.09 temp=  86C power=  109W
GPU3 t=  517.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.77 TFLOP/s=   53.62 temp=  87C power=  119W
GPU2 t=  517.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.20 TFLOP/s=   54.09 temp=  86C power=  117W
GPU3 t=  522.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.67 TFLOP/s=   53.52 temp=  87C power=  114W
GPU2 t=  523.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.85 TFLOP/s=   54.81 temp=  86C power=  114W
GPU3 t=  527.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.60 TFLOP/s=   53.44 temp=  87C power=  125W
GPU2 t=  528.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.71 TFLOP/s=   56.86 temp=  86C power=  119W
GPU3 t=  532.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.55 TFLOP/s=   53.38 temp=  87C power=  120W
GPU2 t=  533.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.65 TFLOP/s=   57.89 temp=  87C power=  122W
GPU3 t=  537.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.60 TFLOP/s=   53.44 temp=  87C power=  122W
GPU2 t=  538.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.49 TFLOP/s=   57.71 temp=  87C power=  117W
GPU3 t=  542.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.43 TFLOP/s=   53.24 temp=  87C power=  120W
GPU2 t=  543.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.46 TFLOP/s=   57.68 temp=  87C power=  119W
GPU3 t=  547.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.42 TFLOP/s=   53.24 temp=  87C power=  119W
GPU2 t=  548.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.36 TFLOP/s=   57.57 temp=  87C power=  117W
GPU3 t=  552.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.49 TFLOP/s=   53.31 temp=  87C power=  121W
GPU2 t=  553.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.41 TFLOP/s=   57.62 temp=  87C power=  117W
GPU3 t=  557.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  43.16 TFLOP/s=   47.46 temp=  86C power=   98W
GPU2 t=  558.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.35 TFLOP/s=   57.56 temp=  87C power=  118W
GPU3 t=  562.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  43.41 TFLOP/s=   47.73 temp=  86C power=  114W
GPU2 t=  563.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.37 TFLOP/s=   57.58 temp=  87C power=  118W
GPU3 t=  568.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.21 TFLOP/s=   51.91 temp=  86C power=  118W
GPU2 t=  568.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.32 TFLOP/s=   57.53 temp=  87C power=  120W
GPU3 t=  573.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.37 TFLOP/s=   52.08 temp=  86C power=  117W
GPU2 t=  573.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  46.70 TFLOP/s=   51.35 temp=  86C power=  107W
GPU3 t=  578.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.39 TFLOP/s=   52.10 temp=  86C power=  120W
GPU2 t=  578.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.05 TFLOP/s=   55.03 temp=  86C power=  113W
GPU3 t=  583.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.40 TFLOP/s=   52.11 temp=  86C power=  119W
GPU2 t=  583.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.15 TFLOP/s=   55.14 temp=  86C power=  114W
GPU3 t=  588.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.38 TFLOP/s=   52.09 temp=  86C power=  119W
GPU2 t=  588.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.18 TFLOP/s=   55.17 temp=  86C power=  113W
GPU3 t=  593.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.34 TFLOP/s=   52.06 temp=  86C power=  118W
GPU2 t=  594.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.10 TFLOP/s=   55.08 temp=  86C power=  115W
GPU3 t=  598.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.40 TFLOP/s=   52.11 temp=  86C power=  118W
GPU2 t=  599.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.11 TFLOP/s=   55.10 temp=  86C power=  113W
GPU3 t=  603.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.49 TFLOP/s=   52.21 temp=  86C power=  118W
GPU2 t=  604.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.19 TFLOP/s=   55.19 temp=  86C power=  112W
GPU3 t=  608.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.45 TFLOP/s=   52.17 temp=  86C power=  121W
GPU2 t=  609.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.24 TFLOP/s=   55.24 temp=  86C power=  113W
GPU3 t=  613.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.44 TFLOP/s=   52.16 temp=  86C power=  117W
GPU2 t=  614.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.16 TFLOP/s=   56.25 temp=  86C power=  121W
GPU3 t=  618.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.86 TFLOP/s=   54.82 temp=  86C power=  123W
GPU2 t=  619.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.98 TFLOP/s=   58.25 temp=  87C power=  111W
GPU3 t=  623.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.78 TFLOP/s=   54.73 temp=  87C power=  125W
GPU2 t=  624.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  53.00 TFLOP/s=   58.28 temp=  87C power=  120W
GPU3 t=  629.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.69 TFLOP/s=   54.64 temp=  87C power=  123W
GPU2 t=  629.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.95 TFLOP/s=   58.22 temp=  87C power=  122W
GPU3 t=  634.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.75 TFLOP/s=   54.70 temp=  87C power=  128W
GPU2 t=  634.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.93 TFLOP/s=   58.20 temp=  87C power=  117W
GPU3 t=  639.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.70 TFLOP/s=   54.64 temp=  87C power=  124W
GPU2 t=  639.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  52.88 TFLOP/s=   58.14 temp=  87C power=  117W
GPU3 t=  644.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.58 TFLOP/s=   54.51 temp=  87C power=  122W
GPU2 t=  645.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  47.62 TFLOP/s=   52.36 temp=  86C power=  115W
GPU3 t=  649.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  49.61 TFLOP/s=   54.55 temp=  87C power=  125W
GPU2 t=  650.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  50.06 TFLOP/s=   55.04 temp=  86C power=  113W
GPU3 t=  654.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  39.57 TFLOP/s=   43.51 temp=  86C power=  109W
GPU2 t=  655.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.30 TFLOP/s=   56.40 temp=  86C power=  114W
GPU3 t=  659.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  45.82 TFLOP/s=   50.38 temp=  86C power=  120W
GPU2 t=  660.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.28 TFLOP/s=   56.39 temp=  86C power=  115W
GPU3 t=  664.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.12 TFLOP/s=   52.90 temp=  86C power=  118W
GPU2 t=  665.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.16 TFLOP/s=   56.25 temp=  86C power=  117W
GPU3 t=  669.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.26 TFLOP/s=   53.06 temp=  86C power=  122W
GPU2 t=  670.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.16 TFLOP/s=   56.25 temp=  86C power=  114W
GPU3 t=  675.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.15 TFLOP/s=   52.94 temp=  86C power=  119W
GPU2 t=  675.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.19 TFLOP/s=   56.29 temp=  86C power=  114W
GPU3 t=  680.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.18 TFLOP/s=   52.97 temp=  86C power=  120W
GPU2 t=  680.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.14 TFLOP/s=   56.23 temp=  86C power=  118W
GPU3 t=  685.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.18 TFLOP/s=   52.98 temp=  86C power=  122W
GPU2 t=  685.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.19 TFLOP/s=   56.29 temp=  86C power=  114W
GPU3 t=  690.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.23 TFLOP/s=   53.03 temp=  86C power=  120W
GPU2 t=  690.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.05 TFLOP/s=   56.13 temp=  86C power=  118W
GPU3 t=  695.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.09 TFLOP/s=   52.88 temp=  86C power=  121W
GPU2 t=  695.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.28 TFLOP/s=   56.38 temp=  86C power=  113W
GPU3 t=  700.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.20 TFLOP/s=   52.99 temp=  86C power=  119W
GPU2 t=  700.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.25 TFLOP/s=   56.34 temp=  86C power=  117W
GPU3 t=  705.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.17 TFLOP/s=   52.96 temp=  86C power=  122W
GPU2 t=  706.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.29 TFLOP/s=   56.40 temp=  86C power=  115W
GPU3 t=  711.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.21 TFLOP/s=   53.01 temp=  86C power=  120W
GPU2 t=  711.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.24 TFLOP/s=   56.34 temp=  86C power=  117W
GPU2 t=  716.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.18 TFLOP/s=   56.28 temp=  86C power=  116W
GPU3 t=  716.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.10 TFLOP/s=   52.89 temp=  86C power=  126W
GPU2 t=  721.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.23 TFLOP/s=   56.33 temp=  87C power=  116W
GPU3 t=  721.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.20 TFLOP/s=   53.00 temp=  87C power=  121W
GPU2 t=  726.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.29 TFLOP/s=   56.40 temp=  87C power=  116W
GPU3 t=  726.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.18 TFLOP/s=   52.98 temp=  87C power=  121W
GPU2 t=  731.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.18 TFLOP/s=   56.27 temp=  87C power=  117W
GPU3 t=  731.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.19 TFLOP/s=   52.99 temp=  86C power=  116W
GPU2 t=  736.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.24 TFLOP/s=   56.33 temp=  86C power=  116W
GPU3 t=  736.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.09 TFLOP/s=   52.87 temp=  87C power=  120W
GPU2 t=  741.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.11 TFLOP/s=   56.20 temp=  86C power=  117W
GPU3 t=  742.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.12 TFLOP/s=   52.91 temp=  87C power=  121W
GPU2 t=  746.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.02 TFLOP/s=   56.10 temp=  87C power=  114W
GPU3 t=  747.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.13 TFLOP/s=   52.92 temp=  87C power=  121W
GPU2 t=  751.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.19 TFLOP/s=   56.29 temp=  87C power=  114W
GPU3 t=  752.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.19 TFLOP/s=   52.98 temp=  87C power=  122W
GPU2 t=  756.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.15 TFLOP/s=   56.24 temp=  86C power=  115W
GPU3 t=  757.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.08 TFLOP/s=   52.86 temp=  87C power=  122W
GPU2 t=  761.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.26 TFLOP/s=   56.36 temp=  86C power=  114W
GPU3 t=  762.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.04 TFLOP/s=   52.82 temp=  87C power=  122W
GPU2 t=  766.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.12 TFLOP/s=   56.21 temp=  87C power=  114W
GPU3 t=  767.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.11 TFLOP/s=   52.90 temp=  86C power=  120W
GPU2 t=  771.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.08 TFLOP/s=   56.16 temp=  87C power=  115W
GPU3 t=  772.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.05 TFLOP/s=   52.83 temp=  87C power=  121W
GPU2 t=  776.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.14 TFLOP/s=   56.23 temp=  87C power=  114W
GPU3 t=  778.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.09 TFLOP/s=   52.88 temp=  87C power=  126W
GPU2 t=  781.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.19 TFLOP/s=   56.28 temp=  87C power=  117W
GPU3 t=  783.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.05 TFLOP/s=   52.83 temp=  87C power=  120W
GPU2 t=  786.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.18 TFLOP/s=   56.27 temp=  87C power=  116W
GPU3 t=  788.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.07 TFLOP/s=   52.85 temp=  87C power=  125W
GPU2 t=  792.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.21 TFLOP/s=   56.30 temp=  87C power=  116W
GPU3 t=  793.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.12 TFLOP/s=   52.91 temp=  87C power=  120W
GPU2 t=  797.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.23 TFLOP/s=   56.33 temp=  87C power=  119W
GPU3 t=  798.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.13 TFLOP/s=   52.92 temp=  86C power=  120W
GPU2 t=  802.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.20 TFLOP/s=   56.29 temp=  86C power=  117W
GPU3 t=  803.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.13 TFLOP/s=   52.92 temp=  86C power=  125W
GPU2 t=  807.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.24 TFLOP/s=   56.34 temp=  86C power=  118W
GPU3 t=  809.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.05 TFLOP/s=   52.83 temp=  87C power=  120W
GPU2 t=  812.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.29 TFLOP/s=   56.39 temp=  86C power=  114W
GPU3 t=  814.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.14 TFLOP/s=   52.93 temp=  87C power=  122W
GPU2 t=  817.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.20 TFLOP/s=   56.29 temp=  86C power=  112W
GPU3 t=  819.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.10 TFLOP/s=   52.89 temp=  87C power=  120W
GPU2 t=  822.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.33 TFLOP/s=   56.43 temp=  87C power=  117W
GPU3 t=  824.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.06 TFLOP/s=   52.84 temp=  87C power=  120W
GPU2 t=  827.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.12 TFLOP/s=   56.21 temp=  86C power=  116W
GPU3 t=  829.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.07 TFLOP/s=   52.85 temp=  87C power=  122W
GPU2 t=  832.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.18 TFLOP/s=   56.27 temp=  86C power=  116W
GPU3 t=  834.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.14 TFLOP/s=   52.93 temp=  87C power=  122W
GPU2 t=  837.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.20 TFLOP/s=   56.30 temp=  86C power=  114W
GPU3 t=  839.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.19 TFLOP/s=   52.99 temp=  87C power=  107W
GPU2 t=  842.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.24 TFLOP/s=   56.34 temp=  86C power=  116W
GPU3 t=  845.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.14 TFLOP/s=   52.93 temp=  86C power=  126W
GPU2 t=  848.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.26 TFLOP/s=   56.37 temp=  87C power=  117W
GPU3 t=  850.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.16 TFLOP/s=   52.96 temp=  86C power=  119W
GPU2 t=  853.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.31 TFLOP/s=   56.42 temp=  87C power=  119W
GPU3 t=  855.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.17 TFLOP/s=   52.97 temp=  87C power=  122W
GPU2 t=  858.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.15 TFLOP/s=   56.24 temp=  87C power=  114W
GPU3 t=  860.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.18 TFLOP/s=   52.97 temp=  86C power=  122W
GPU2 t=  863.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.33 TFLOP/s=   56.43 temp=  86C power=  118W
GPU3 t=  865.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.16 TFLOP/s=   52.96 temp=  87C power=  121W
GPU2 t=  868.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.22 TFLOP/s=   56.31 temp=  86C power=  116W
GPU3 t=  870.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.18 TFLOP/s=   52.98 temp=  87C power=  123W
GPU2 t=  873.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.18 TFLOP/s=   56.27 temp=  86C power=  115W
GPU3 t=  876.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.15 TFLOP/s=   52.94 temp=  87C power=  119W
GPU2 t=  878.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.21 TFLOP/s=   56.31 temp=  86C power=  115W
GPU3 t=  881.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.16 TFLOP/s=   52.95 temp=  87C power=  120W
GPU2 t=  883.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.14 TFLOP/s=   56.23 temp=  86C power=  115W
GPU3 t=  886.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.14 TFLOP/s=   52.93 temp=  87C power=  120W
GPU2 t=  888.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.18 TFLOP/s=   56.28 temp=  86C power=  116W
GPU3 t=  891.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.16 TFLOP/s=   52.95 temp=  87C power=  120W
GPU2 t=  893.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.19 TFLOP/s=   56.28 temp=  86C power=  117W
GPU3 t=  896.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  48.14 TFLOP/s=   52.93 temp=  87C power=  125W
GPU2 t=  898.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s=  51.20 TFLOP/s=   56.30 temp=  86C power=  117W
DONE GPU3: elapsed=900.1s iters=47088
DONE GPU2: elapsed=900.1s iters=49176

============================================================================================
FINAL CLI CHART
============================================================================================

GPU2 | Tesla T10 | dtype=float16 | matmul=8192x8192
  elapsed=900.1s  iterations=49176  ballast=14.06 GiB
  peak torch allocated : 14.44 GiB
  peak torch reserved  : 14.45 GiB
  avg memory usage     : [################################--]  94.16%
  avg GPU util         : [##################################]  99.99%
  avg observed TFLOP/s :    60.08    peak sample:    74.69
  avg iter/s           :    54.64    peak sample:    67.93
  temp C avg/max       :     84.3 / 87.0
  power W avg/max      :    123.3 / 151.1
  memory %:
    00 [##########################--]    94.16%
    01 [##########################--]    94.16%
    02 [##########################--]    94.16%
    03 [##########################--]    94.16%
    04 [##########################--]    94.16%
    05 [##########################--]    94.16%
    06 [##########################--]    94.16%
    07 [##########################--]    94.16%
    08 [##########################--]    94.16%
    09 [##########################--]    94.16%
    10 [##########################--]    94.16%
    11 [##########################--]    94.16%
    12 [##########################--]    94.16%
    13 [##########################--]    94.16%
    14 [##########################--]    94.16%
    15 [##########################--]    94.16%
    16 [##########################--]    94.16%
    17 [##########################--]    94.16%
    18 [##########################--]    94.16%
    19 [##########################--]    94.16%
    20 [##########################--]    94.16%
    21 [##########################--]    94.16%
    22 [##########################--]    94.16%
    23 [##########################--]    94.16%
  GPU util %:
    00 [############################]    99.86%
    01 [############################]   100.00%
    02 [############################]    99.88%
    03 [############################]   100.00%
    04 [############################]   100.00%
    05 [############################]   100.00%
    06 [############################]   100.00%
    07 [############################]   100.00%
    08 [############################]   100.00%
    09 [############################]   100.00%
    10 [############################]   100.00%
    11 [############################]   100.00%
    12 [############################]   100.00%
    13 [############################]   100.00%
    14 [############################]   100.00%
    15 [############################]   100.00%
    16 [############################]   100.00%
    17 [############################]   100.00%
    18 [############################]   100.00%
    19 [############################]   100.00%
    20 [############################]   100.00%
    21 [############################]   100.00%
    22 [############################]   100.00%
    23 [############################]   100.00%
  observed TFLOP/s:
    00 [############################]    73.77
    01 [###########################-]    72.03
    02 [###########################-]    70.66
    03 [##########################--]    69.42
    04 [#########################---]    66.40
    05 [########################----]    63.08
    06 [#######################-----]    61.69
    07 [#######################-----]    59.72
    08 [######################------]    57.93
    09 [######################------]    58.62
    10 [######################------]    56.75
    11 [######################------]    56.99
    12 [#####################-------]    56.18
    13 [#####################-------]    56.63
    14 [######################------]    57.56
    15 [#####################-------]    55.25
    16 [######################------]    57.09
    17 [#####################-------]    55.83
    18 [#####################-------]    56.30
    19 [#####################-------]    56.27
    20 [#####################-------]    56.25
    21 [#####################-------]    56.33
    22 [#####################-------]    56.31
    23 [#####################-------]    56.30

GPU3 | Tesla T10 | dtype=float16 | matmul=8192x8192
  elapsed=900.1s  iterations=47088  ballast=14.06 GiB
  peak torch allocated : 14.44 GiB
  peak torch reserved  : 14.45 GiB
  avg memory usage     : [################################--]  94.16%
  avg GPU util         : [##################################] 100.00%
  avg observed TFLOP/s :    57.56    peak sample:    76.82
  avg iter/s           :    52.35    peak sample:    69.87
  temp C avg/max       :     83.6 / 87.0
  power W avg/max      :    126.5 / 148.6
  memory %:
    00 [##########################--]    94.16%
    01 [##########################--]    94.16%
    02 [##########################--]    94.16%
    03 [##########################--]    94.16%
    04 [##########################--]    94.16%
    05 [##########################--]    94.16%
    06 [##########################--]    94.16%
    07 [##########################--]    94.16%
    08 [##########################--]    94.16%
    09 [##########################--]    94.16%
    10 [##########################--]    94.16%
    11 [##########################--]    94.16%
    12 [##########################--]    94.16%
    13 [##########################--]    94.16%
    14 [##########################--]    94.16%
    15 [##########################--]    94.16%
    16 [##########################--]    94.16%
    17 [##########################--]    94.16%
    18 [##########################--]    94.16%
    19 [##########################--]    94.16%
    20 [##########################--]    94.16%
    21 [##########################--]    94.16%
    22 [##########################--]    94.16%
    23 [##########################--]    94.16%
  GPU util %:
    00 [############################]   100.00%
    01 [############################]   100.00%
    02 [############################]   100.00%
    03 [############################]   100.00%
    04 [############################]   100.00%
    05 [############################]   100.00%
    06 [############################]   100.00%
    07 [############################]   100.00%
    08 [############################]   100.00%
    09 [############################]   100.00%
    10 [############################]   100.00%
    11 [############################]   100.00%
    12 [############################]   100.00%
    13 [############################]   100.00%
    14 [############################]   100.00%
    15 [############################]   100.00%
    16 [############################]   100.00%
    17 [############################]   100.00%
    18 [############################]   100.00%
    19 [############################]   100.00%
    20 [############################]   100.00%
    21 [############################]   100.00%
    22 [############################]   100.00%
    23 [############################]   100.00%
  observed TFLOP/s:
    00 [############################]    75.33
    01 [###########################-]    71.89
    02 [##########################--]    69.43
    03 [#########################---]    67.51
    04 [########################----]    65.30
    05 [########################----]    63.65
    06 [#######################-----]    60.61
    07 [#####################-------]    57.82
    08 [####################--------]    53.64
    09 [#####################-------]    55.29
    10 [####################--------]    53.58
    11 [####################--------]    54.13
    12 [####################--------]    53.78
    13 [####################--------]    53.63
    14 [####################--------]    52.63
    15 [###################---------]    51.44
    16 [####################--------]    53.26
    17 [###################---------]    52.28
    18 [####################--------]    52.96
    19 [####################--------]    52.95
    20 [####################--------]    52.88
    21 [####################--------]    52.89
    22 [####################--------]    52.92
    23 [####################--------]    52.95

SUCCESS: all requested GPU workers completed without reporting an exception.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment