uv run python ./gpu_stress_torch.py --gpus 2,3 --minutes 15
PyTorch 2.11.0+cu130; CUDA available; device_count=4
GPU2: Tesla T10, total VRAM=15.56 GiB
GPU3: Tesla T10, total VRAM=15.56 GiB
READY GPU2: Tesla T10 dtype=float16 matmul=8192 ballast=14.06GiB free=0.93/15.56GiB
READY GPU3: Tesla T10 dtype=float16 matmul=8192 ballast=14.06GiB free=0.93/15.56GiB
GPU3 t= 5.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 69.87 TFLOP/s= 76.82 temp= 46C power= 144W
GPU2 t= 5.1s util= 99.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 67.93 TFLOP/s= 74.69 temp= 47C power= 144W
GPU3 t= 10.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 69.26 TFLOP/s= 76.15 temp= 49C power= 142W
GPU2 t= 10.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 67.45 TFLOP/s= 74.16 temp= 51C power= 140W
GPU3 t= 15.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 69.12 TFLOP/s= 76.00 temp= 52C power= 145W
GPU2 t= 15.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 67.38 TFLOP/s= 74.08 temp= 54C power= 147W
GPU3 t= 20.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 68.62 TFLOP/s= 75.44 temp= 55C power= 146W
GPU2 t= 20.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 66.97 TFLOP/s= 73.63 temp= 57C power= 143W
GPU3 t= 25.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 67.95 TFLOP/s= 74.71 temp= 58C power= 147W
GPU2 t= 25.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 66.68 TFLOP/s= 73.32 temp= 60C power= 146W
GPU2 t= 30.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 66.72 TFLOP/s= 73.36 temp= 63C power= 146W
GPU3 t= 30.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 67.73 TFLOP/s= 74.47 temp= 61C power= 146W
GPU3 t= 35.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 67.05 TFLOP/s= 73.72 temp= 63C power= 143W
GPU2 t= 35.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 66.50 TFLOP/s= 73.12 temp= 66C power= 147W
GPU3 t= 40.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 66.68 TFLOP/s= 73.31 temp= 65C power= 149W
GPU2 t= 40.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 66.18 TFLOP/s= 72.77 temp= 68C power= 147W
GPU3 t= 45.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 66.00 TFLOP/s= 72.57 temp= 67C power= 144W
GPU2 t= 45.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 65.86 TFLOP/s= 72.42 temp= 70C power= 148W
GPU3 t= 50.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 65.95 TFLOP/s= 72.52 temp= 69C power= 148W
GPU2 t= 50.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 65.94 TFLOP/s= 72.50 temp= 72C power= 150W
GPU3 t= 55.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 65.04 TFLOP/s= 71.51 temp= 70C power= 143W
GPU2 t= 55.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 65.62 TFLOP/s= 72.15 temp= 73C power= 143W
GPU3 t= 60.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.86 TFLOP/s= 71.31 temp= 71C power= 142W
GPU2 t= 60.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 65.23 TFLOP/s= 71.72 temp= 75C power= 151W
GPU3 t= 65.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.68 TFLOP/s= 71.12 temp= 72C power= 144W
GPU2 t= 65.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.93 TFLOP/s= 71.39 temp= 76C power= 150W
GPU3 t= 70.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.44 TFLOP/s= 70.85 temp= 73C power= 147W
GPU2 t= 71.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.79 TFLOP/s= 71.23 temp= 77C power= 145W
GPU3 t= 76.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.04 TFLOP/s= 70.42 temp= 74C power= 140W
GPU2 t= 76.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.75 TFLOP/s= 71.20 temp= 77C power= 149W
GPU3 t= 81.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 63.97 TFLOP/s= 70.34 temp= 74C power= 139W
GPU2 t= 81.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.71 TFLOP/s= 71.15 temp= 78C power= 145W
GPU3 t= 86.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 63.54 TFLOP/s= 69.86 temp= 75C power= 140W
GPU2 t= 86.2s util= 99.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.33 TFLOP/s= 70.74 temp= 79C power= 147W
GPU3 t= 91.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 63.19 TFLOP/s= 69.48 temp= 75C power= 142W
GPU2 t= 91.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.15 TFLOP/s= 70.53 temp= 79C power= 150W
GPU3 t= 96.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 62.91 TFLOP/s= 69.18 temp= 76C power= 148W
GPU2 t= 96.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.18 TFLOP/s= 70.57 temp= 80C power= 147W
GPU3 t= 101.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 62.61 TFLOP/s= 68.84 temp= 77C power= 139W
GPU2 t= 101.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 64.21 TFLOP/s= 70.60 temp= 81C power= 149W
GPU3 t= 106.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 62.45 TFLOP/s= 68.67 temp= 77C power= 143W
GPU2 t= 106.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 63.98 TFLOP/s= 70.34 temp= 82C power= 149W
GPU2 t= 111.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 63.80 TFLOP/s= 70.15 temp= 82C power= 146W
GPU3 t= 111.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 62.43 TFLOP/s= 68.64 temp= 78C power= 141W
GPU2 t= 116.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 63.67 TFLOP/s= 70.00 temp= 83C power= 144W
GPU3 t= 116.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 62.18 TFLOP/s= 68.37 temp= 78C power= 141W
GPU2 t= 121.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 63.57 TFLOP/s= 69.90 temp= 83C power= 146W
GPU3 t= 121.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 61.90 TFLOP/s= 68.06 temp= 79C power= 142W
GPU2 t= 126.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 63.11 TFLOP/s= 69.39 temp= 83C power= 143W
GPU3 t= 126.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 61.62 TFLOP/s= 67.75 temp= 79C power= 141W
GPU2 t= 131.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 63.05 TFLOP/s= 69.32 temp= 84C power= 146W
GPU3 t= 131.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 61.08 TFLOP/s= 67.16 temp= 80C power= 145W
GPU2 t= 136.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 62.99 TFLOP/s= 69.26 temp= 85C power= 147W
GPU3 t= 136.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 61.43 TFLOP/s= 67.55 temp= 80C power= 127W
GPU2 t= 141.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 62.82 TFLOP/s= 69.07 temp= 85C power= 146W
GPU3 t= 141.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 61.38 TFLOP/s= 67.49 temp= 81C power= 142W
GPU3 t= 147.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 60.21 TFLOP/s= 66.20 temp= 81C power= 142W
GPU2 t= 147.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 62.78 TFLOP/s= 69.03 temp= 86C power= 147W
GPU3 t= 152.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 59.92 TFLOP/s= 65.88 temp= 81C power= 140W
GPU2 t= 152.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 62.79 TFLOP/s= 69.04 temp= 86C power= 145W
GPU3 t= 157.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 59.68 TFLOP/s= 65.62 temp= 82C power= 141W
GPU2 t= 157.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 62.53 TFLOP/s= 68.75 temp= 86C power= 146W
GPU2 t= 162.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 62.32 TFLOP/s= 68.52 temp= 87C power= 145W
GPU3 t= 162.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 59.57 TFLOP/s= 65.50 temp= 82C power= 139W
GPU2 t= 167.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 59.87 TFLOP/s= 65.83 temp= 86C power= 129W
GPU3 t= 167.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 59.36 TFLOP/s= 65.27 temp= 83C power= 139W
GPU2 t= 172.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.42 TFLOP/s= 64.24 temp= 86C power= 137W
GPU3 t= 172.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 59.23 TFLOP/s= 65.13 temp= 83C power= 140W
GPU2 t= 177.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.37 TFLOP/s= 64.17 temp= 86C power= 132W
GPU3 t= 177.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 59.08 TFLOP/s= 64.96 temp= 83C power= 104W
GPU2 t= 182.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.40 TFLOP/s= 64.21 temp= 86C power= 132W
GPU3 t= 182.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.86 TFLOP/s= 64.72 temp= 83C power= 140W
GPU3 t= 187.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.86 TFLOP/s= 64.72 temp= 84C power= 139W
GPU2 t= 187.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.40 TFLOP/s= 64.22 temp= 86C power= 124W
GPU3 t= 192.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.54 TFLOP/s= 64.37 temp= 84C power= 138W
GPU2 t= 192.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.23 TFLOP/s= 64.03 temp= 87C power= 132W
GPU2 t= 197.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.36 TFLOP/s= 64.17 temp= 87C power= 131W
GPU3 t= 197.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 57.98 TFLOP/s= 63.75 temp= 84C power= 139W
GPU2 t= 202.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.29 TFLOP/s= 64.09 temp= 87C power= 132W
GPU3 t= 202.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 57.78 TFLOP/s= 63.53 temp= 85C power= 137W
GPU2 t= 207.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.23 TFLOP/s= 64.02 temp= 87C power= 133W
GPU3 t= 208.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 57.76 TFLOP/s= 63.51 temp= 85C power= 139W
GPU2 t= 213.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 58.16 TFLOP/s= 63.95 temp= 87C power= 135W
GPU3 t= 213.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 57.51 TFLOP/s= 63.23 temp= 85C power= 138W
GPU3 t= 218.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 57.36 TFLOP/s= 63.06 temp= 86C power= 143W
GPU2 t= 218.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 55.64 TFLOP/s= 61.17 temp= 86C power= 121W
GPU3 t= 223.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 57.31 TFLOP/s= 63.01 temp= 86C power= 135W
GPU2 t= 223.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.62 TFLOP/s= 58.96 temp= 86C power= 127W
GPU3 t= 228.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 57.08 TFLOP/s= 62.76 temp= 86C power= 140W
GPU2 t= 228.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 55.28 TFLOP/s= 60.78 temp= 86C power= 124W
GPU3 t= 233.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 56.79 TFLOP/s= 62.44 temp= 86C power= 137W
GPU2 t= 233.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 55.49 TFLOP/s= 61.02 temp= 86C power= 127W
GPU2 t= 238.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 57.32 TFLOP/s= 63.02 temp= 87C power= 131W
GPU3 t= 238.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 56.64 TFLOP/s= 62.28 temp= 86C power= 137W
GPU2 t= 243.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 57.16 TFLOP/s= 62.85 temp= 87C power= 128W
GPU3 t= 243.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 56.56 TFLOP/s= 62.19 temp= 87C power= 138W
GPU2 t= 248.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 57.10 TFLOP/s= 62.78 temp= 87C power= 127W
GPU3 t= 248.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 56.37 TFLOP/s= 61.98 temp= 87C power= 136W
GPU2 t= 253.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 57.01 TFLOP/s= 62.69 temp= 87C power= 128W
GPU3 t= 253.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.04 TFLOP/s= 57.22 temp= 86C power= 118W
GPU2 t= 258.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.36 TFLOP/s= 58.67 temp= 86C power= 120W
GPU3 t= 258.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.41 TFLOP/s= 55.43 temp= 86C power= 125W
GPU2 t= 263.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.16 TFLOP/s= 58.45 temp= 86C power= 118W
GPU3 t= 263.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.85 TFLOP/s= 55.91 temp= 86C power= 128W
GPU2 t= 268.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.23 TFLOP/s= 58.53 temp= 86C power= 119W
GPU3 t= 268.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.10 TFLOP/s= 58.39 temp= 86C power= 129W
GPU2 t= 273.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.26 TFLOP/s= 58.56 temp= 86C power= 119W
GPU3 t= 273.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.99 TFLOP/s= 58.26 temp= 86C power= 134W
GPU2 t= 278.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.25 TFLOP/s= 58.55 temp= 86C power= 118W
GPU3 t= 279.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.03 TFLOP/s= 58.31 temp= 87C power= 132W
GPU2 t= 283.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 55.51 TFLOP/s= 61.03 temp= 86C power= 118W
GPU3 t= 284.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.99 TFLOP/s= 58.26 temp= 87C power= 129W
GPU2 t= 289.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 55.44 TFLOP/s= 60.96 temp= 87C power= 124W
GPU3 t= 289.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.89 TFLOP/s= 58.15 temp= 87C power= 125W
GPU2 t= 294.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 55.34 TFLOP/s= 60.85 temp= 87C power= 127W
GPU3 t= 294.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.26 TFLOP/s= 57.46 temp= 87C power= 111W
GPU2 t= 299.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 55.33 TFLOP/s= 60.84 temp= 87C power= 125W
GPU3 t= 299.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 45.04 TFLOP/s= 49.52 temp= 86C power= 126W
GPU2 t= 304.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.96 TFLOP/s= 58.23 temp= 86C power= 110W
GPU3 t= 304.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.33 TFLOP/s= 54.24 temp= 86C power= 123W
GPU2 t= 309.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.20 TFLOP/s= 55.20 temp= 86C power= 114W
GPU3 t= 309.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.30 TFLOP/s= 54.20 temp= 86C power= 124W
GPU2 t= 314.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.98 TFLOP/s= 57.15 temp= 86C power= 120W
GPU3 t= 314.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.31 TFLOP/s= 54.22 temp= 86C power= 94W
GPU2 t= 319.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.44 TFLOP/s= 58.76 temp= 86C power= 120W
GPU3 t= 319.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.33 TFLOP/s= 54.24 temp= 86C power= 123W
GPU2 t= 324.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.46 TFLOP/s= 58.78 temp= 87C power= 120W
GPU3 t= 324.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.34 TFLOP/s= 54.25 temp= 86C power= 125W
GPU3 t= 329.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.29 TFLOP/s= 54.19 temp= 86C power= 121W
GPU2 t= 329.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.41 TFLOP/s= 58.72 temp= 86C power= 120W
GPU3 t= 334.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.37 TFLOP/s= 54.28 temp= 86C power= 124W
GPU2 t= 334.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.37 TFLOP/s= 58.68 temp= 87C power= 120W
GPU3 t= 339.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.27 TFLOP/s= 54.18 temp= 86C power= 123W
GPU2 t= 339.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.34 TFLOP/s= 58.65 temp= 87C power= 121W
GPU3 t= 344.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.36 TFLOP/s= 54.27 temp= 86C power= 122W
GPU2 t= 345.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.35 TFLOP/s= 58.66 temp= 87C power= 120W
GPU3 t= 349.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.36 TFLOP/s= 54.27 temp= 86C power= 123W
GPU2 t= 350.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.39 TFLOP/s= 58.70 temp= 87C power= 120W
GPU3 t= 354.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.12 TFLOP/s= 55.11 temp= 86C power= 125W
GPU2 t= 355.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.36 TFLOP/s= 58.67 temp= 87C power= 120W
GPU3 t= 360.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.47 TFLOP/s= 56.59 temp= 86C power= 126W
GPU2 t= 360.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.25 TFLOP/s= 58.55 temp= 87C power= 114W
GPU3 t= 365.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.30 TFLOP/s= 56.40 temp= 87C power= 126W
GPU2 t= 365.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.27 TFLOP/s= 58.57 temp= 87C power= 120W
GPU3 t= 370.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.15 TFLOP/s= 56.24 temp= 87C power= 126W
GPU2 t= 370.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.24 TFLOP/s= 58.54 temp= 87C power= 120W
GPU3 t= 375.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.00 TFLOP/s= 56.07 temp= 87C power= 129W
GPU2 t= 375.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.25 TFLOP/s= 58.55 temp= 87C power= 118W
GPU3 t= 380.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.05 TFLOP/s= 56.13 temp= 87C power= 127W
GPU2 t= 380.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.23 TFLOP/s= 58.53 temp= 87C power= 121W
GPU3 t= 385.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.10 TFLOP/s= 52.88 temp= 86C power= 93W
GPU2 t= 385.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.26 TFLOP/s= 58.56 temp= 87C power= 120W
GPU3 t= 390.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 43.54 TFLOP/s= 47.88 temp= 86C power= 116W
GPU2 t= 390.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.85 TFLOP/s= 52.62 temp= 86C power= 107W
GPU3 t= 395.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.60 TFLOP/s= 53.44 temp= 86C power= 124W
GPU2 t= 396.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.14 TFLOP/s= 55.13 temp= 86C power= 116W
GPU3 t= 400.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.33 TFLOP/s= 54.24 temp= 86C power= 123W
GPU2 t= 401.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.81 TFLOP/s= 55.86 temp= 86C power= 115W
GPU3 t= 405.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.48 TFLOP/s= 54.40 temp= 86C power= 122W
GPU2 t= 406.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.05 TFLOP/s= 56.13 temp= 86C power= 120W
GPU3 t= 410.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.33 TFLOP/s= 54.24 temp= 86C power= 119W
GPU2 t= 411.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.29 TFLOP/s= 58.60 temp= 87C power= 120W
GPU3 t= 415.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.29 TFLOP/s= 54.20 temp= 87C power= 124W
GPU2 t= 416.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.29 TFLOP/s= 58.59 temp= 87C power= 121W
GPU3 t= 420.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.24 TFLOP/s= 54.14 temp= 86C power= 124W
GPU2 t= 421.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.32 TFLOP/s= 58.63 temp= 87C power= 121W
GPU3 t= 425.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.25 TFLOP/s= 54.15 temp= 86C power= 125W
GPU2 t= 426.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.28 TFLOP/s= 58.58 temp= 87C power= 119W
GPU3 t= 430.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.28 TFLOP/s= 54.19 temp= 86C power= 123W
GPU2 t= 431.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.34 TFLOP/s= 58.65 temp= 87C power= 123W
GPU3 t= 435.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.16 TFLOP/s= 54.05 temp= 87C power= 123W
GPU2 t= 436.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.11 TFLOP/s= 52.90 temp= 86C power= 111W
GPU3 t= 440.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.16 TFLOP/s= 54.05 temp= 87C power= 123W
GPU2 t= 441.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.68 TFLOP/s= 55.73 temp= 86C power= 116W
GPU3 t= 445.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.12 TFLOP/s= 54.01 temp= 87C power= 119W
GPU2 t= 446.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.78 TFLOP/s= 55.83 temp= 86C power= 114W
GPU3 t= 451.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.06 TFLOP/s= 53.95 temp= 87C power= 124W
GPU2 t= 451.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.76 TFLOP/s= 55.82 temp= 86C power= 115W
GPU3 t= 456.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.04 TFLOP/s= 53.92 temp= 87C power= 123W
GPU2 t= 456.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.71 TFLOP/s= 55.75 temp= 86C power= 111W
GPU3 t= 461.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.02 TFLOP/s= 53.90 temp= 87C power= 121W
GPU2 t= 461.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.71 TFLOP/s= 55.76 temp= 86C power= 113W
GPU3 t= 466.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.03 TFLOP/s= 53.91 temp= 87C power= 121W
GPU2 t= 467.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.77 TFLOP/s= 55.82 temp= 86C power= 112W
GPU3 t= 471.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.74 TFLOP/s= 53.59 temp= 87C power= 121W
GPU2 t= 472.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.78 TFLOP/s= 55.84 temp= 86C power= 113W
GPU3 t= 476.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.76 TFLOP/s= 53.62 temp= 87C power= 124W
GPU2 t= 477.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.80 TFLOP/s= 55.86 temp= 86C power= 116W
GPU3 t= 481.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.72 TFLOP/s= 53.57 temp= 87C power= 120W
GPU2 t= 482.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.11 TFLOP/s= 58.40 temp= 86C power= 125W
GPU3 t= 486.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.61 TFLOP/s= 53.45 temp= 87C power= 122W
GPU2 t= 487.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.31 TFLOP/s= 58.61 temp= 87C power= 120W
GPU3 t= 491.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.95 TFLOP/s= 53.83 temp= 87C power= 122W
GPU2 t= 492.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.16 TFLOP/s= 58.45 temp= 87C power= 120W
GPU3 t= 496.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.80 TFLOP/s= 53.65 temp= 87C power= 124W
GPU2 t= 497.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.21 TFLOP/s= 58.51 temp= 87C power= 121W
GPU3 t= 501.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.81 TFLOP/s= 53.67 temp= 87C power= 124W
GPU2 t= 502.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.26 TFLOP/s= 58.55 temp= 87C power= 119W
GPU3 t= 506.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.81 TFLOP/s= 53.66 temp= 87C power= 122W
GPU2 t= 507.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.72 TFLOP/s= 57.97 temp= 87C power= 109W
GPU3 t= 511.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.70 TFLOP/s= 53.54 temp= 87C power= 107W
GPU2 t= 512.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.37 TFLOP/s= 52.09 temp= 86C power= 109W
GPU3 t= 517.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.77 TFLOP/s= 53.62 temp= 87C power= 119W
GPU2 t= 517.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.20 TFLOP/s= 54.09 temp= 86C power= 117W
GPU3 t= 522.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.67 TFLOP/s= 53.52 temp= 87C power= 114W
GPU2 t= 523.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.85 TFLOP/s= 54.81 temp= 86C power= 114W
GPU3 t= 527.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.60 TFLOP/s= 53.44 temp= 87C power= 125W
GPU2 t= 528.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.71 TFLOP/s= 56.86 temp= 86C power= 119W
GPU3 t= 532.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.55 TFLOP/s= 53.38 temp= 87C power= 120W
GPU2 t= 533.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.65 TFLOP/s= 57.89 temp= 87C power= 122W
GPU3 t= 537.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.60 TFLOP/s= 53.44 temp= 87C power= 122W
GPU2 t= 538.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.49 TFLOP/s= 57.71 temp= 87C power= 117W
GPU3 t= 542.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.43 TFLOP/s= 53.24 temp= 87C power= 120W
GPU2 t= 543.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.46 TFLOP/s= 57.68 temp= 87C power= 119W
GPU3 t= 547.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.42 TFLOP/s= 53.24 temp= 87C power= 119W
GPU2 t= 548.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.36 TFLOP/s= 57.57 temp= 87C power= 117W
GPU3 t= 552.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.49 TFLOP/s= 53.31 temp= 87C power= 121W
GPU2 t= 553.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.41 TFLOP/s= 57.62 temp= 87C power= 117W
GPU3 t= 557.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 43.16 TFLOP/s= 47.46 temp= 86C power= 98W
GPU2 t= 558.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.35 TFLOP/s= 57.56 temp= 87C power= 118W
GPU3 t= 562.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 43.41 TFLOP/s= 47.73 temp= 86C power= 114W
GPU2 t= 563.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.37 TFLOP/s= 57.58 temp= 87C power= 118W
GPU3 t= 568.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.21 TFLOP/s= 51.91 temp= 86C power= 118W
GPU2 t= 568.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.32 TFLOP/s= 57.53 temp= 87C power= 120W
GPU3 t= 573.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.37 TFLOP/s= 52.08 temp= 86C power= 117W
GPU2 t= 573.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 46.70 TFLOP/s= 51.35 temp= 86C power= 107W
GPU3 t= 578.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.39 TFLOP/s= 52.10 temp= 86C power= 120W
GPU2 t= 578.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.05 TFLOP/s= 55.03 temp= 86C power= 113W
GPU3 t= 583.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.40 TFLOP/s= 52.11 temp= 86C power= 119W
GPU2 t= 583.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.15 TFLOP/s= 55.14 temp= 86C power= 114W
GPU3 t= 588.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.38 TFLOP/s= 52.09 temp= 86C power= 119W
GPU2 t= 588.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.18 TFLOP/s= 55.17 temp= 86C power= 113W
GPU3 t= 593.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.34 TFLOP/s= 52.06 temp= 86C power= 118W
GPU2 t= 594.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.10 TFLOP/s= 55.08 temp= 86C power= 115W
GPU3 t= 598.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.40 TFLOP/s= 52.11 temp= 86C power= 118W
GPU2 t= 599.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.11 TFLOP/s= 55.10 temp= 86C power= 113W
GPU3 t= 603.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.49 TFLOP/s= 52.21 temp= 86C power= 118W
GPU2 t= 604.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.19 TFLOP/s= 55.19 temp= 86C power= 112W
GPU3 t= 608.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.45 TFLOP/s= 52.17 temp= 86C power= 121W
GPU2 t= 609.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.24 TFLOP/s= 55.24 temp= 86C power= 113W
GPU3 t= 613.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.44 TFLOP/s= 52.16 temp= 86C power= 117W
GPU2 t= 614.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.16 TFLOP/s= 56.25 temp= 86C power= 121W
GPU3 t= 618.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.86 TFLOP/s= 54.82 temp= 86C power= 123W
GPU2 t= 619.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.98 TFLOP/s= 58.25 temp= 87C power= 111W
GPU3 t= 623.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.78 TFLOP/s= 54.73 temp= 87C power= 125W
GPU2 t= 624.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 53.00 TFLOP/s= 58.28 temp= 87C power= 120W
GPU3 t= 629.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.69 TFLOP/s= 54.64 temp= 87C power= 123W
GPU2 t= 629.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.95 TFLOP/s= 58.22 temp= 87C power= 122W
GPU3 t= 634.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.75 TFLOP/s= 54.70 temp= 87C power= 128W
GPU2 t= 634.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.93 TFLOP/s= 58.20 temp= 87C power= 117W
GPU3 t= 639.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.70 TFLOP/s= 54.64 temp= 87C power= 124W
GPU2 t= 639.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 52.88 TFLOP/s= 58.14 temp= 87C power= 117W
GPU3 t= 644.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.58 TFLOP/s= 54.51 temp= 87C power= 122W
GPU2 t= 645.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 47.62 TFLOP/s= 52.36 temp= 86C power= 115W
GPU3 t= 649.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 49.61 TFLOP/s= 54.55 temp= 87C power= 125W
GPU2 t= 650.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 50.06 TFLOP/s= 55.04 temp= 86C power= 113W
GPU3 t= 654.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 39.57 TFLOP/s= 43.51 temp= 86C power= 109W
GPU2 t= 655.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.30 TFLOP/s= 56.40 temp= 86C power= 114W
GPU3 t= 659.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 45.82 TFLOP/s= 50.38 temp= 86C power= 120W
GPU2 t= 660.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.28 TFLOP/s= 56.39 temp= 86C power= 115W
GPU3 t= 664.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.12 TFLOP/s= 52.90 temp= 86C power= 118W
GPU2 t= 665.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.16 TFLOP/s= 56.25 temp= 86C power= 117W
GPU3 t= 669.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.26 TFLOP/s= 53.06 temp= 86C power= 122W
GPU2 t= 670.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.16 TFLOP/s= 56.25 temp= 86C power= 114W
GPU3 t= 675.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.15 TFLOP/s= 52.94 temp= 86C power= 119W
GPU2 t= 675.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.19 TFLOP/s= 56.29 temp= 86C power= 114W
GPU3 t= 680.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.18 TFLOP/s= 52.97 temp= 86C power= 120W
GPU2 t= 680.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.14 TFLOP/s= 56.23 temp= 86C power= 118W
GPU3 t= 685.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.18 TFLOP/s= 52.98 temp= 86C power= 122W
GPU2 t= 685.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.19 TFLOP/s= 56.29 temp= 86C power= 114W
GPU3 t= 690.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.23 TFLOP/s= 53.03 temp= 86C power= 120W
GPU2 t= 690.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.05 TFLOP/s= 56.13 temp= 86C power= 118W
GPU3 t= 695.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.09 TFLOP/s= 52.88 temp= 86C power= 121W
GPU2 t= 695.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.28 TFLOP/s= 56.38 temp= 86C power= 113W
GPU3 t= 700.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.20 TFLOP/s= 52.99 temp= 86C power= 119W
GPU2 t= 700.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.25 TFLOP/s= 56.34 temp= 86C power= 117W
GPU3 t= 705.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.17 TFLOP/s= 52.96 temp= 86C power= 122W
GPU2 t= 706.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.29 TFLOP/s= 56.40 temp= 86C power= 115W
GPU3 t= 711.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.21 TFLOP/s= 53.01 temp= 86C power= 120W
GPU2 t= 711.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.24 TFLOP/s= 56.34 temp= 86C power= 117W
GPU2 t= 716.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.18 TFLOP/s= 56.28 temp= 86C power= 116W
GPU3 t= 716.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.10 TFLOP/s= 52.89 temp= 86C power= 126W
GPU2 t= 721.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.23 TFLOP/s= 56.33 temp= 87C power= 116W
GPU3 t= 721.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.20 TFLOP/s= 53.00 temp= 87C power= 121W
GPU2 t= 726.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.29 TFLOP/s= 56.40 temp= 87C power= 116W
GPU3 t= 726.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.18 TFLOP/s= 52.98 temp= 87C power= 121W
GPU2 t= 731.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.18 TFLOP/s= 56.27 temp= 87C power= 117W
GPU3 t= 731.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.19 TFLOP/s= 52.99 temp= 86C power= 116W
GPU2 t= 736.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.24 TFLOP/s= 56.33 temp= 86C power= 116W
GPU3 t= 736.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.09 TFLOP/s= 52.87 temp= 87C power= 120W
GPU2 t= 741.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.11 TFLOP/s= 56.20 temp= 86C power= 117W
GPU3 t= 742.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.12 TFLOP/s= 52.91 temp= 87C power= 121W
GPU2 t= 746.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.02 TFLOP/s= 56.10 temp= 87C power= 114W
GPU3 t= 747.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.13 TFLOP/s= 52.92 temp= 87C power= 121W
GPU2 t= 751.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.19 TFLOP/s= 56.29 temp= 87C power= 114W
GPU3 t= 752.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.19 TFLOP/s= 52.98 temp= 87C power= 122W
GPU2 t= 756.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.15 TFLOP/s= 56.24 temp= 86C power= 115W
GPU3 t= 757.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.08 TFLOP/s= 52.86 temp= 87C power= 122W
GPU2 t= 761.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.26 TFLOP/s= 56.36 temp= 86C power= 114W
GPU3 t= 762.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.04 TFLOP/s= 52.82 temp= 87C power= 122W
GPU2 t= 766.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.12 TFLOP/s= 56.21 temp= 87C power= 114W
GPU3 t= 767.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.11 TFLOP/s= 52.90 temp= 86C power= 120W
GPU2 t= 771.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.08 TFLOP/s= 56.16 temp= 87C power= 115W
GPU3 t= 772.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.05 TFLOP/s= 52.83 temp= 87C power= 121W
GPU2 t= 776.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.14 TFLOP/s= 56.23 temp= 87C power= 114W
GPU3 t= 778.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.09 TFLOP/s= 52.88 temp= 87C power= 126W
GPU2 t= 781.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.19 TFLOP/s= 56.28 temp= 87C power= 117W
GPU3 t= 783.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.05 TFLOP/s= 52.83 temp= 87C power= 120W
GPU2 t= 786.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.18 TFLOP/s= 56.27 temp= 87C power= 116W
GPU3 t= 788.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.07 TFLOP/s= 52.85 temp= 87C power= 125W
GPU2 t= 792.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.21 TFLOP/s= 56.30 temp= 87C power= 116W
GPU3 t= 793.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.12 TFLOP/s= 52.91 temp= 87C power= 120W
GPU2 t= 797.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.23 TFLOP/s= 56.33 temp= 87C power= 119W
GPU3 t= 798.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.13 TFLOP/s= 52.92 temp= 86C power= 120W
GPU2 t= 802.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.20 TFLOP/s= 56.29 temp= 86C power= 117W
GPU3 t= 803.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.13 TFLOP/s= 52.92 temp= 86C power= 125W
GPU2 t= 807.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.24 TFLOP/s= 56.34 temp= 86C power= 118W
GPU3 t= 809.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.05 TFLOP/s= 52.83 temp= 87C power= 120W
GPU2 t= 812.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.29 TFLOP/s= 56.39 temp= 86C power= 114W
GPU3 t= 814.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.14 TFLOP/s= 52.93 temp= 87C power= 122W
GPU2 t= 817.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.20 TFLOP/s= 56.29 temp= 86C power= 112W
GPU3 t= 819.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.10 TFLOP/s= 52.89 temp= 87C power= 120W
GPU2 t= 822.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.33 TFLOP/s= 56.43 temp= 87C power= 117W
GPU3 t= 824.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.06 TFLOP/s= 52.84 temp= 87C power= 120W
GPU2 t= 827.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.12 TFLOP/s= 56.21 temp= 86C power= 116W
GPU3 t= 829.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.07 TFLOP/s= 52.85 temp= 87C power= 122W
GPU2 t= 832.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.18 TFLOP/s= 56.27 temp= 86C power= 116W
GPU3 t= 834.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.14 TFLOP/s= 52.93 temp= 87C power= 122W
GPU2 t= 837.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.20 TFLOP/s= 56.30 temp= 86C power= 114W
GPU3 t= 839.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.19 TFLOP/s= 52.99 temp= 87C power= 107W
GPU2 t= 842.9s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.24 TFLOP/s= 56.34 temp= 86C power= 116W
GPU3 t= 845.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.14 TFLOP/s= 52.93 temp= 86C power= 126W
GPU2 t= 848.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.26 TFLOP/s= 56.37 temp= 87C power= 117W
GPU3 t= 850.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.16 TFLOP/s= 52.96 temp= 86C power= 119W
GPU2 t= 853.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.31 TFLOP/s= 56.42 temp= 87C power= 119W
GPU3 t= 855.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.17 TFLOP/s= 52.97 temp= 87C power= 122W
GPU2 t= 858.2s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.15 TFLOP/s= 56.24 temp= 87C power= 114W
GPU3 t= 860.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.18 TFLOP/s= 52.97 temp= 86C power= 122W
GPU2 t= 863.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.33 TFLOP/s= 56.43 temp= 86C power= 118W
GPU3 t= 865.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.16 TFLOP/s= 52.96 temp= 87C power= 121W
GPU2 t= 868.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.22 TFLOP/s= 56.31 temp= 86C power= 116W
GPU3 t= 870.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.18 TFLOP/s= 52.98 temp= 87C power= 123W
GPU2 t= 873.5s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.18 TFLOP/s= 56.27 temp= 86C power= 115W
GPU3 t= 876.0s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.15 TFLOP/s= 52.94 temp= 87C power= 119W
GPU2 t= 878.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.21 TFLOP/s= 56.31 temp= 86C power= 115W
GPU3 t= 881.1s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.16 TFLOP/s= 52.95 temp= 87C power= 120W
GPU2 t= 883.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.14 TFLOP/s= 56.23 temp= 86C power= 115W
GPU3 t= 886.3s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.14 TFLOP/s= 52.93 temp= 87C power= 120W
GPU2 t= 888.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.18 TFLOP/s= 56.28 temp= 86C power= 116W
GPU3 t= 891.4s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.16 TFLOP/s= 52.95 temp= 87C power= 120W
GPU2 t= 893.7s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.19 TFLOP/s= 56.28 temp= 86C power= 117W
GPU3 t= 896.6s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 48.14 TFLOP/s= 52.93 temp= 87C power= 125W
GPU2 t= 898.8s util=100.0% mem= 94.2% torch=14.45/15.56GiB iter/s= 51.20 TFLOP/s= 56.30 temp= 86C power= 117W
DONE GPU3: elapsed=900.1s iters=47088
DONE GPU2: elapsed=900.1s iters=49176
============================================================================================
FINAL CLI CHART
============================================================================================
GPU2 | Tesla T10 | dtype=float16 | matmul=8192x8192
elapsed=900.1s iterations=49176 ballast=14.06 GiB
peak torch allocated : 14.44 GiB
peak torch reserved : 14.45 GiB
avg memory usage : [################################--] 94.16%
avg GPU util : [##################################] 99.99%
avg observed TFLOP/s : 60.08 peak sample: 74.69
avg iter/s : 54.64 peak sample: 67.93
temp C avg/max : 84.3 / 87.0
power W avg/max : 123.3 / 151.1
memory %:
00 [##########################--] 94.16%
01 [##########################--] 94.16%
02 [##########################--] 94.16%
03 [##########################--] 94.16%
04 [##########################--] 94.16%
05 [##########################--] 94.16%
06 [##########################--] 94.16%
07 [##########################--] 94.16%
08 [##########################--] 94.16%
09 [##########################--] 94.16%
10 [##########################--] 94.16%
11 [##########################--] 94.16%
12 [##########################--] 94.16%
13 [##########################--] 94.16%
14 [##########################--] 94.16%
15 [##########################--] 94.16%
16 [##########################--] 94.16%
17 [##########################--] 94.16%
18 [##########################--] 94.16%
19 [##########################--] 94.16%
20 [##########################--] 94.16%
21 [##########################--] 94.16%
22 [##########################--] 94.16%
23 [##########################--] 94.16%
GPU util %:
00 [############################] 99.86%
01 [############################] 100.00%
02 [############################] 99.88%
03 [############################] 100.00%
04 [############################] 100.00%
05 [############################] 100.00%
06 [############################] 100.00%
07 [############################] 100.00%
08 [############################] 100.00%
09 [############################] 100.00%
10 [############################] 100.00%
11 [############################] 100.00%
12 [############################] 100.00%
13 [############################] 100.00%
14 [############################] 100.00%
15 [############################] 100.00%
16 [############################] 100.00%
17 [############################] 100.00%
18 [############################] 100.00%
19 [############################] 100.00%
20 [############################] 100.00%
21 [############################] 100.00%
22 [############################] 100.00%
23 [############################] 100.00%
observed TFLOP/s:
00 [############################] 73.77
01 [###########################-] 72.03
02 [###########################-] 70.66
03 [##########################--] 69.42
04 [#########################---] 66.40
05 [########################----] 63.08
06 [#######################-----] 61.69
07 [#######################-----] 59.72
08 [######################------] 57.93
09 [######################------] 58.62
10 [######################------] 56.75
11 [######################------] 56.99
12 [#####################-------] 56.18
13 [#####################-------] 56.63
14 [######################------] 57.56
15 [#####################-------] 55.25
16 [######################------] 57.09
17 [#####################-------] 55.83
18 [#####################-------] 56.30
19 [#####################-------] 56.27
20 [#####################-------] 56.25
21 [#####################-------] 56.33
22 [#####################-------] 56.31
23 [#####################-------] 56.30
GPU3 | Tesla T10 | dtype=float16 | matmul=8192x8192
elapsed=900.1s iterations=47088 ballast=14.06 GiB
peak torch allocated : 14.44 GiB
peak torch reserved : 14.45 GiB
avg memory usage : [################################--] 94.16%
avg GPU util : [##################################] 100.00%
avg observed TFLOP/s : 57.56 peak sample: 76.82
avg iter/s : 52.35 peak sample: 69.87
temp C avg/max : 83.6 / 87.0
power W avg/max : 126.5 / 148.6
memory %:
00 [##########################--] 94.16%
01 [##########################--] 94.16%
02 [##########################--] 94.16%
03 [##########################--] 94.16%
04 [##########################--] 94.16%
05 [##########################--] 94.16%
06 [##########################--] 94.16%
07 [##########################--] 94.16%
08 [##########################--] 94.16%
09 [##########################--] 94.16%
10 [##########################--] 94.16%
11 [##########################--] 94.16%
12 [##########################--] 94.16%
13 [##########################--] 94.16%
14 [##########################--] 94.16%
15 [##########################--] 94.16%
16 [##########################--] 94.16%
17 [##########################--] 94.16%
18 [##########################--] 94.16%
19 [##########################--] 94.16%
20 [##########################--] 94.16%
21 [##########################--] 94.16%
22 [##########################--] 94.16%
23 [##########################--] 94.16%
GPU util %:
00 [############################] 100.00%
01 [############################] 100.00%
02 [############################] 100.00%
03 [############################] 100.00%
04 [############################] 100.00%
05 [############################] 100.00%
06 [############################] 100.00%
07 [############################] 100.00%
08 [############################] 100.00%
09 [############################] 100.00%
10 [############################] 100.00%
11 [############################] 100.00%
12 [############################] 100.00%
13 [############################] 100.00%
14 [############################] 100.00%
15 [############################] 100.00%
16 [############################] 100.00%
17 [############################] 100.00%
18 [############################] 100.00%
19 [############################] 100.00%
20 [############################] 100.00%
21 [############################] 100.00%
22 [############################] 100.00%
23 [############################] 100.00%
observed TFLOP/s:
00 [############################] 75.33
01 [###########################-] 71.89
02 [##########################--] 69.43
03 [#########################---] 67.51
04 [########################----] 65.30
05 [########################----] 63.65
06 [#######################-----] 60.61
07 [#####################-------] 57.82
08 [####################--------] 53.64
09 [#####################-------] 55.29
10 [####################--------] 53.58
11 [####################--------] 54.13
12 [####################--------] 53.78
13 [####################--------] 53.63
14 [####################--------] 52.63
15 [###################---------] 51.44
16 [####################--------] 53.26
17 [###################---------] 52.28
18 [####################--------] 52.96
19 [####################--------] 52.95
20 [####################--------] 52.88
21 [####################--------] 52.89
22 [####################--------] 52.92
23 [####################--------] 52.95
SUCCESS: all requested GPU workers completed without reporting an exception.
Created
June 12, 2026 15:26
-
-
Save Anthony-Hoo/ae5f20b58082ca18683f559e3a91d86f to your computer and use it in GitHub Desktop.
GPU_stress_test by pytorch
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| # /// script | |
| # dependencies = ["torch", "nvidia-ml-py"] | |
| # /// | |
| """ | |
| PyTorch NVIDIA GPU stress test. | |
| Default target: CUDA devices 2 and 3, 15 minutes, high GEMM load, | |
| and near-full VRAM allocation with a configurable safety headroom. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import math | |
| import multiprocessing as mp | |
| import os | |
| import queue | |
| import sys | |
| import time | |
| import traceback | |
| from statistics import mean | |
| from typing import Any | |
| MIB = 1024**2 | |
| GIB = 1024**3 | |
| def parse_gpu_list(value: str) -> list[int]: | |
| try: | |
| gpus = [int(x.strip()) for x in value.split(",") if x.strip() != ""] | |
| except ValueError as exc: | |
| raise argparse.ArgumentTypeError("--gpus must look like: 2,3") from exc | |
| if not gpus: | |
| raise argparse.ArgumentTypeError("--gpus cannot be empty") | |
| if any(g < 0 for g in gpus): | |
| raise argparse.ArgumentTypeError("GPU ids must be >= 0") | |
| if len(set(gpus)) != len(gpus): | |
| raise argparse.ArgumentTypeError("GPU ids must be unique") | |
| return gpus | |
| def gib(n: float | int) -> float: | |
| return float(n) / GIB | |
| def bar(value: float, max_value: float = 100.0, width: int = 34) -> str: | |
| if max_value <= 0: | |
| max_value = 1.0 | |
| value = max(0.0, min(float(value), max_value)) | |
| fill = int(round(width * value / max_value)) | |
| return "[" + ("#" * fill).ljust(width, "-") + "]" | |
| def downsample(values: list[float], max_points: int) -> list[float]: | |
| if len(values) <= max_points: | |
| return values | |
| bucket = len(values) / max_points | |
| out: list[float] = [] | |
| for i in range(max_points): | |
| start = int(i * bucket) | |
| end = int((i + 1) * bucket) | |
| part = values[start:max(end, start + 1)] | |
| out.append(mean(part)) | |
| return out | |
| def print_series(title: str, values: list[float], suffix: str, *, max_value: float | None = None) -> None: | |
| if not values: | |
| print(f" {title}: no samples") | |
| return | |
| vals = downsample(values, 24) | |
| scale = max_value if max_value is not None else max(max(vals), 1e-9) | |
| print(f" {title}:") | |
| for i, v in enumerate(vals): | |
| print(f" {i:02d} {bar(v, scale, 28)} {v:8.2f}{suffix}") | |
| def nvml_physical_index(torch_gpu: int) -> int: | |
| """Map torch CUDA index to NVML physical index when CUDA_VISIBLE_DEVICES is numeric.""" | |
| visible = os.environ.get("CUDA_VISIBLE_DEVICES") | |
| if not visible: | |
| return torch_gpu | |
| parts = [p.strip() for p in visible.split(",") if p.strip()] | |
| if torch_gpu < len(parts) and parts[torch_gpu].isdigit(): | |
| return int(parts[torch_gpu]) | |
| return torch_gpu | |
| def nvml_snapshot(torch_gpu: int) -> dict[str, float | int | None]: | |
| try: | |
| import pynvml # type: ignore | |
| pynvml.nvmlInit() | |
| handle = pynvml.nvmlDeviceGetHandleByIndex(nvml_physical_index(torch_gpu)) | |
| util = pynvml.nvmlDeviceGetUtilizationRates(handle) | |
| mem = pynvml.nvmlDeviceGetMemoryInfo(handle) | |
| try: | |
| temp_c = pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU) | |
| except Exception: | |
| temp_c = None | |
| try: | |
| power_w = pynvml.nvmlDeviceGetPowerUsage(handle) / 1000.0 | |
| except Exception: | |
| power_w = None | |
| return { | |
| "nvml_ok": 1, | |
| "gpu_util_pct": float(util.gpu), | |
| "mem_util_pct": float(mem.used) * 100.0 / float(mem.total), | |
| "nvml_mem_used_gib": gib(mem.used), | |
| "nvml_mem_total_gib": gib(mem.total), | |
| "temp_c": temp_c, | |
| "power_w": power_w, | |
| } | |
| except Exception: | |
| return { | |
| "nvml_ok": 0, | |
| "gpu_util_pct": None, | |
| "mem_util_pct": None, | |
| "nvml_mem_used_gib": None, | |
| "nvml_mem_total_gib": None, | |
| "temp_c": None, | |
| "power_w": None, | |
| } | |
| def choose_dtype(torch: Any, gpu: int, dtype_name: str) -> Any: | |
| name = dtype_name.lower() | |
| if name == "auto": | |
| major, _minor = torch.cuda.get_device_capability(gpu) | |
| return torch.float16 if major >= 7 else torch.float32 | |
| if name in {"fp16", "float16", "half"}: | |
| return torch.float16 | |
| if name in {"fp32", "float32"}: | |
| return torch.float32 | |
| if name in {"bf16", "bfloat16"}: | |
| if hasattr(torch.cuda, "is_bf16_supported") and not torch.cuda.is_bf16_supported(): | |
| raise RuntimeError(f"GPU{gpu} / this PyTorch build does not report BF16 support") | |
| return torch.bfloat16 | |
| raise RuntimeError(f"Unsupported dtype: {dtype_name}") | |
| def dtype_name(torch: Any, dtype: Any) -> str: | |
| if dtype is torch.float16: | |
| return "float16" | |
| if dtype is torch.float32: | |
| return "float32" | |
| if dtype is torch.bfloat16: | |
| return "bfloat16" | |
| return str(dtype) | |
| def fit_matmul_size(n: int, dtype_bytes: int, free_bytes: int, headroom_bytes: int) -> int: | |
| # Need A, B, C plus enough workspace/headroom. Round down to multiples of 256. | |
| n = max(256, int(n // 256 * 256)) | |
| while n >= 512: | |
| needed = 3 * n * n * dtype_bytes | |
| if needed + headroom_bytes < int(free_bytes * 0.70): | |
| return n | |
| n = int((n * 0.80) // 256 * 256) | |
| return max(256, n) | |
| def allocate_compute_tensors(torch: Any, gpu: int, n: int, dtype: Any, headroom_bytes: int) -> tuple[Any, Any, Any, int]: | |
| dtype_bytes = torch.empty((), dtype=dtype).element_size() | |
| free_bytes, _total_bytes = torch.cuda.mem_get_info(gpu) | |
| n = fit_matmul_size(n, dtype_bytes, free_bytes, headroom_bytes) | |
| last_error = None | |
| while n >= 256: | |
| a = b = c = None | |
| try: | |
| scale = 1.0 / math.sqrt(max(n, 1)) | |
| a = torch.empty((n, n), device=f"cuda:{gpu}", dtype=dtype) | |
| b = torch.empty((n, n), device=f"cuda:{gpu}", dtype=dtype) | |
| c = torch.empty((n, n), device=f"cuda:{gpu}", dtype=dtype) | |
| a.fill_(scale) | |
| b.fill_(scale) | |
| c.zero_() | |
| torch.cuda.synchronize(gpu) | |
| return a, b, c, n | |
| except torch.cuda.OutOfMemoryError as exc: | |
| last_error = exc | |
| try: | |
| del a, b, c | |
| except Exception: | |
| pass | |
| torch.cuda.empty_cache() | |
| n = int((n * 0.70) // 256 * 256) | |
| raise RuntimeError(f"Could not allocate matmul tensors; last error: {last_error}") | |
| def allocate_ballast(torch: Any, gpu: int, mem_fraction: float, headroom_mib: int, block_mib: int) -> tuple[list[Any], int]: | |
| ballast: list[Any] = [] | |
| headroom_bytes = int(headroom_mib * MIB) | |
| block_bytes = int(block_mib * MIB) | |
| free_bytes, total_bytes = torch.cuda.mem_get_info(gpu) | |
| target_free = max(int(total_bytes * (1.0 - mem_fraction)), headroom_bytes) | |
| min_block = 16 * MIB | |
| allocated = 0 | |
| while True: | |
| free_bytes, _total_bytes = torch.cuda.mem_get_info(gpu) | |
| remaining = free_bytes - target_free | |
| if remaining <= min_block: | |
| break | |
| this_block = int(min(block_bytes, remaining)) | |
| this_block = max(min_block, this_block // MIB * MIB) | |
| try: | |
| t = torch.empty((this_block,), dtype=torch.uint8, device=f"cuda:{gpu}") | |
| t.fill_(gpu & 0xFF) | |
| ballast.append(t) | |
| allocated += this_block | |
| except torch.cuda.OutOfMemoryError: | |
| torch.cuda.empty_cache() | |
| block_bytes //= 2 | |
| if block_bytes < min_block: | |
| break | |
| torch.cuda.synchronize(gpu) | |
| return ballast, allocated | |
| def worker(gpu: int, args: dict[str, Any], q: mp.Queue) -> None: | |
| try: | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import torch | |
| torch.cuda.set_device(gpu) | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| try: | |
| torch.set_float32_matmul_precision("high") | |
| except Exception: | |
| pass | |
| if not torch.cuda.is_available(): | |
| raise RuntimeError("torch.cuda.is_available() is False") | |
| count = torch.cuda.device_count() | |
| if gpu >= count: | |
| raise RuntimeError(f"Requested cuda:{gpu}, but PyTorch sees only {count} CUDA device(s)") | |
| props = torch.cuda.get_device_properties(gpu) | |
| dtype = choose_dtype(torch, gpu, args["dtype"]) | |
| dtype_str = dtype_name(torch, dtype) | |
| torch.cuda.empty_cache() | |
| torch.cuda.reset_peak_memory_stats(gpu) | |
| headroom_bytes = int(args["headroom_mib"] * MIB) | |
| a, b, c, n = allocate_compute_tensors(torch, gpu, args["matmul_size"], dtype, headroom_bytes) | |
| # Warm up cuBLAS/PyTorch first, so internal workspaces are allocated before we fill VRAM. | |
| for _ in range(max(1, args["warmup_iters"])): | |
| torch.mm(a, b, out=c) | |
| torch.cuda.synchronize(gpu) | |
| ballast, ballast_bytes = allocate_ballast( | |
| torch, | |
| gpu, | |
| args["mem_fraction"], | |
| args["headroom_mib"], | |
| args["block_mib"], | |
| ) | |
| free_bytes, total_bytes = torch.cuda.mem_get_info(gpu) | |
| q.put( | |
| { | |
| "type": "ready", | |
| "gpu": gpu, | |
| "name": props.name, | |
| "dtype": dtype_str, | |
| "matmul_size": n, | |
| "ballast_gib": gib(ballast_bytes), | |
| "free_gib": gib(free_bytes), | |
| "total_gib": gib(total_bytes), | |
| } | |
| ) | |
| start = time.monotonic() | |
| end = start + float(args["minutes"]) * 60.0 | |
| last = start | |
| last_iters = 0 | |
| iters = 0 | |
| sample_interval = float(args["sample_interval"]) | |
| sync_every = max(1, int(args["sync_every"])) | |
| flops_per_mm = 2.0 * float(n) ** 3 | |
| while True: | |
| now = time.monotonic() | |
| if now >= end: | |
| break | |
| for _ in range(sync_every): | |
| torch.mm(a, b, out=c) | |
| iters += sync_every | |
| torch.cuda.synchronize(gpu) | |
| now = time.monotonic() | |
| if now - last >= sample_interval: | |
| dt = now - last | |
| di = iters - last_iters | |
| free_bytes, total_bytes = torch.cuda.mem_get_info(gpu) | |
| allocated = torch.cuda.memory_allocated(gpu) | |
| reserved = torch.cuda.memory_reserved(gpu) | |
| nvml = nvml_snapshot(gpu) | |
| tflops = (di * flops_per_mm) / max(dt, 1e-9) / 1e12 | |
| q.put( | |
| { | |
| "type": "sample", | |
| "gpu": gpu, | |
| "elapsed": now - start, | |
| "iter_s": di / max(dt, 1e-9), | |
| "tflops": tflops, | |
| "torch_alloc_gib": gib(allocated), | |
| "torch_reserved_gib": gib(reserved), | |
| "torch_free_gib": gib(free_bytes), | |
| "torch_total_gib": gib(total_bytes), | |
| **nvml, | |
| } | |
| ) | |
| last = now | |
| last_iters = iters | |
| torch.cuda.synchronize(gpu) | |
| elapsed = time.monotonic() - start | |
| peak_alloc = torch.cuda.max_memory_allocated(gpu) | |
| peak_reserved = torch.cuda.max_memory_reserved(gpu) | |
| q.put( | |
| { | |
| "type": "done", | |
| "gpu": gpu, | |
| "elapsed": elapsed, | |
| "iters": iters, | |
| "name": props.name, | |
| "dtype": dtype_str, | |
| "matmul_size": n, | |
| "ballast_gib": gib(ballast_bytes), | |
| "peak_alloc_gib": gib(peak_alloc), | |
| "peak_reserved_gib": gib(peak_reserved), | |
| } | |
| ) | |
| # Keep references alive until after the final report. | |
| _ = (a, b, c, ballast) | |
| except Exception: | |
| q.put({"type": "error", "gpu": gpu, "traceback": traceback.format_exc()}) | |
| sys.exit(1) | |
| def print_final_report(gpus: list[int], samples: dict[int, list[dict[str, Any]]], done: dict[int, dict[str, Any]]) -> None: | |
| print("\n" + "=" * 92) | |
| print("FINAL CLI CHART") | |
| print("=" * 92) | |
| for gpu in gpus: | |
| ss = samples.get(gpu, []) | |
| d = done.get(gpu, {}) | |
| print( | |
| f"\nGPU{gpu} | {d.get('name', 'unknown')} | " | |
| f"dtype={d.get('dtype', '?')} | " | |
| f"matmul={d.get('matmul_size', '?')}x{d.get('matmul_size', '?')}" | |
| ) | |
| print( | |
| f" elapsed={float(d.get('elapsed', 0.0)):.1f}s " | |
| f"iterations={int(d.get('iters', 0))} " | |
| f"ballast={float(d.get('ballast_gib', 0.0)):.2f} GiB" | |
| ) | |
| print(f" peak torch allocated : {float(d.get('peak_alloc_gib', 0.0)):.2f} GiB") | |
| print(f" peak torch reserved : {float(d.get('peak_reserved_gib', 0.0)):.2f} GiB") | |
| if not ss: | |
| print(" no runtime samples collected") | |
| continue | |
| mem_pcts = [float(s["mem_util_pct"]) for s in ss if s.get("mem_util_pct") is not None] | |
| if not mem_pcts: | |
| mem_pcts = [ | |
| 100.0 * float(s["torch_reserved_gib"]) / max(float(s["torch_total_gib"]), 1e-9) | |
| for s in ss | |
| ] | |
| util_pcts = [float(s["gpu_util_pct"]) for s in ss if s.get("gpu_util_pct") is not None] | |
| tflops = [float(s["tflops"]) for s in ss] | |
| iter_s = [float(s["iter_s"]) for s in ss] | |
| temps = [float(s["temp_c"]) for s in ss if s.get("temp_c") is not None] | |
| powers = [float(s["power_w"]) for s in ss if s.get("power_w") is not None] | |
| avg_mem = mean(mem_pcts) | |
| print(f" avg memory usage : {bar(avg_mem)} {avg_mem:6.2f}%") | |
| if util_pcts: | |
| avg_util = mean(util_pcts) | |
| print(f" avg GPU util : {bar(avg_util)} {avg_util:6.2f}%") | |
| print(f" avg observed TFLOP/s : {mean(tflops):8.2f} peak sample: {max(tflops):8.2f}") | |
| print(f" avg iter/s : {mean(iter_s):8.2f} peak sample: {max(iter_s):8.2f}") | |
| if temps: | |
| print(f" temp C avg/max : {mean(temps):8.1f} / {max(temps):.1f}") | |
| if powers: | |
| print(f" power W avg/max : {mean(powers):8.1f} / {max(powers):.1f}") | |
| print_series("memory %", mem_pcts, "%", max_value=100.0) | |
| if util_pcts: | |
| print_series("GPU util %", util_pcts, "%", max_value=100.0) | |
| print_series("observed TFLOP/s", tflops, "") | |
| def positive_float(value: str) -> float: | |
| v = float(value) | |
| if v <= 0: | |
| raise argparse.ArgumentTypeError("must be > 0") | |
| return v | |
| def fraction(value: str) -> float: | |
| v = float(value) | |
| if not (0.10 <= v < 1.0): | |
| raise argparse.ArgumentTypeError("must be in [0.10, 1.0)") | |
| return v | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description="Stress test selected NVIDIA GPUs with PyTorch GEMM + VRAM ballast.") | |
| parser.add_argument("--gpus", type=parse_gpu_list, default="2,3", help="CUDA device ids, e.g. 2,3. Default: 2,3") | |
| parser.add_argument("--minutes", type=positive_float, default=15.0, help="Run duration in minutes. Default: 15") | |
| parser.add_argument("--mem-fraction", type=fraction, default=0.94, help="Target fraction of total VRAM to occupy. Default: 0.94") | |
| parser.add_argument("--headroom-mib", type=int, default=768, help="Minimum free VRAM left per GPU. Default: 768") | |
| parser.add_argument("--block-mib", type=int, default=512, help="VRAM ballast allocation block size. Default: 512") | |
| parser.add_argument("--matmul-size", type=int, default=8192, help="Square GEMM size N for NxN @ NxN. Default: 8192") | |
| parser.add_argument( | |
| "--dtype", | |
| default="auto", | |
| choices=["auto", "fp16", "float16", "bf16", "bfloat16", "fp32", "float32"], | |
| help="GEMM dtype. Default: auto", | |
| ) | |
| parser.add_argument("--sample-interval", type=positive_float, default=5.0, help="Seconds between samples. Default: 5") | |
| parser.add_argument("--sync-every", type=int, default=8, help="Synchronize after this many matmuls. Default: 8") | |
| parser.add_argument("--warmup-iters", type=int, default=8, help="Warmup matmuls before VRAM ballast. Default: 8") | |
| args_ns = parser.parse_args() | |
| args = vars(args_ns) | |
| if args["headroom_mib"] < 128: | |
| print("Refusing --headroom-mib < 128 because it makes OOM much more likely.", file=sys.stderr) | |
| return 2 | |
| if args["block_mib"] < 16: | |
| print("Refusing --block-mib < 16.", file=sys.stderr) | |
| return 2 | |
| if os.environ.get("CUDA_VISIBLE_DEVICES"): | |
| print( | |
| "WARNING: CUDA_VISIBLE_DEVICES is set. PyTorch CUDA ids are relative to the visible device list. " | |
| "Unset it if you want physical GPU ids 2 and 3." | |
| ) | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| try: | |
| import torch | |
| if not torch.cuda.is_available(): | |
| print("ERROR: PyTorch CUDA is not available. Check NVIDIA driver and CUDA-enabled torch wheel.", file=sys.stderr) | |
| return 1 | |
| count = torch.cuda.device_count() | |
| bad = [g for g in args["gpus"] if g >= count] | |
| if bad: | |
| print(f"ERROR: requested GPU(s) {bad}, but PyTorch sees only {count} CUDA device(s).", file=sys.stderr) | |
| return 1 | |
| print(f"PyTorch {torch.__version__}; CUDA available; device_count={count}") | |
| for g in args["gpus"]: | |
| p = torch.cuda.get_device_properties(g) | |
| print(f" GPU{g}: {p.name}, total VRAM={gib(p.total_memory):.2f} GiB") | |
| except Exception as exc: | |
| print(f"ERROR during PyTorch CUDA validation: {exc}", file=sys.stderr) | |
| return 1 | |
| ctx = mp.get_context("spawn") | |
| q: mp.Queue = ctx.Queue() | |
| procs: list[mp.Process] = [] | |
| for gpu in args["gpus"]: | |
| p = ctx.Process(target=worker, args=(gpu, args, q), daemon=False) | |
| p.start() | |
| procs.append(p) | |
| samples: dict[int, list[dict[str, Any]]] = {g: [] for g in args["gpus"]} | |
| done: dict[int, dict[str, Any]] = {} | |
| errors: dict[int, str] = {} | |
| def terminate_children() -> None: | |
| for p in procs: | |
| if p.is_alive(): | |
| p.terminate() | |
| for p in procs: | |
| p.join(timeout=5) | |
| try: | |
| while len(done) + len(errors) < len(args["gpus"]): | |
| try: | |
| msg = q.get(timeout=1.0) | |
| except queue.Empty: | |
| if all(not p.is_alive() for p in procs): | |
| break | |
| continue | |
| typ = msg.get("type") | |
| gpu = int(msg.get("gpu", -1)) | |
| if typ == "ready": | |
| print( | |
| f"READY GPU{gpu}: {msg['name']} dtype={msg['dtype']} matmul={msg['matmul_size']} " | |
| f"ballast={msg['ballast_gib']:.2f}GiB free={msg['free_gib']:.2f}/{msg['total_gib']:.2f}GiB" | |
| ) | |
| elif typ == "sample": | |
| samples[gpu].append(msg) | |
| util = msg.get("gpu_util_pct") | |
| util_s = "n/a" if util is None else f"{float(util):5.1f}%" | |
| mem = msg.get("mem_util_pct") | |
| if mem is None: | |
| mem = 100.0 * float(msg["torch_reserved_gib"]) / max(float(msg["torch_total_gib"]), 1e-9) | |
| temp = msg.get("temp_c") | |
| temp_s = "n/a" if temp is None else f"{float(temp):4.0f}C" | |
| power = msg.get("power_w") | |
| power_s = "n/a" if power is None else f"{float(power):5.0f}W" | |
| print( | |
| f"GPU{gpu} t={msg['elapsed']:7.1f}s " | |
| f"util={util_s} mem={float(mem):5.1f}% " | |
| f"torch={msg['torch_reserved_gib']:.2f}/{msg['torch_total_gib']:.2f}GiB " | |
| f"iter/s={msg['iter_s']:7.2f} TFLOP/s={msg['tflops']:8.2f} " | |
| f"temp={temp_s} power={power_s}" | |
| ) | |
| elif typ == "done": | |
| done[gpu] = msg | |
| print(f"DONE GPU{gpu}: elapsed={msg['elapsed']:.1f}s iters={msg['iters']}") | |
| elif typ == "error": | |
| errors[gpu] = str(msg.get("traceback", "unknown error")) | |
| print(f"ERROR GPU{gpu}:\n{errors[gpu]}", file=sys.stderr) | |
| except KeyboardInterrupt: | |
| print("\nInterrupted; terminating workers...", file=sys.stderr) | |
| terminate_children() | |
| return 130 | |
| finally: | |
| for p in procs: | |
| p.join(timeout=10) | |
| if errors: | |
| terminate_children() | |
| print("\nAt least one GPU worker failed; no success chart generated for failed workers.", file=sys.stderr) | |
| for gpu, tb in errors.items(): | |
| print(f"\n--- GPU{gpu} traceback ---\n{tb}", file=sys.stderr) | |
| print_final_report(args["gpus"], samples, done) | |
| return 1 | |
| print_final_report(args["gpus"], samples, done) | |
| print("\nSUCCESS: all requested GPU workers completed without reporting an exception.") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment