Skip to content

Instantly share code, notes, and snippets.

@jamesr66a

jamesr66a/gist:62f67c2077234773ce6168974d8f6a17

Created April 16, 2024 23:43
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jamesr66a/62f67c2077234773ce6168974d8f6a17 to your computer and use it in GitHub Desktop.
Save jamesr66a/62f67c2077234773ce6168974d8f6a17 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
================================================================================
Traceback (most recent call last):
File "/usr/local/lib/python3.10/dist-packages/fireworks/util/multiprocessing.py", line 70, in _wrap
fn(i, *args)
File "/usr/local/lib/python3.10/dist-packages/fireworks/serving/image_generation/coordinator.py", line 32, in _worker_fn
worker.init(args)
File "/usr/local/lib/python3.10/dist-packages/fireworks/serving/image_generation/worker.py", line 121, in init
self.model_manager.warmup_models()
File "/usr/local/lib/python3.10/dist-packages/fireworks/serving/image_generation/model_manager.py", line 333, in warmup_models
pipeline.text_to_image(
File "/usr/local/lib/python3.10/dist-packages/fireworks/serving/image_generation/pipeline.py", line 1375, in text_to_image
return self._run_pipe(
File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/fireworks/serving/image_generation/pipeline.py", line 1561, in _run_pipe
sampled_latent = self.do_sampling(
File "/usr/local/lib/python3.10/dist-packages/fireworks/serving/image_generation/pipeline.py", line 1775, in do_sampling
latent = sd3.sample_euler(
File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/torch/amp/autocast_mode.py", line 16, in decorate_autocast
return func(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/fireworks/models/sd3/sd3_impls.py", line 197, in sample_euler
denoised = model(x, sigma_hat * s_in, **extra_args)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/fireworks/models/sd3/sd3_impls.py", line 150, in forward
batched = self.model(
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/fireworks/models/sd3/sd3_impls.py", line 102, in forward
compiled_forward = firecuda.make_graphed_callable(
File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/fireworks/cuda/graph.py", line 73, in make_graphed_callable
callable(*inputs)
File "/usr/local/lib/python3.10/dist-packages/fireworks/models/sd3/sd3_impls.py", line 114, in _forward
model_output = self.diffusion_model(
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/fireworks/models/sd3/mmdit.py", line 672, in forward
x = self.forward_core_with_concat(x, c, context)
File "/usr/local/lib/python3.10/dist-packages/fireworks/models/sd3/mmdit.py", line 630, in forward_core_with_concat
context, x = block(context, x, c=c_mod)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/fireworks/models/sd3/mmdit.py", line 349, in forward
attn = self.self_attn(q, k, v) # (bs, seq_len, nheads, head_dim)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/fireworks/nn/self_attention.py", line 85, in forward
out = flash_attn_cuda.fwd(
TypeError: fwd(): incompatible function arguments. The following argument types are supported:
1. (arg0: torch.Tensor, arg1: torch.Tensor, arg2: torch.Tensor, arg3: Optional[torch.Tensor], arg4: Optional[torch.Tensor], arg5: float, arg6: float, arg7: bool, arg8: int, arg9: int, arg10: bool, arg11: Optional[torch.Generator]) -> List[torch.Tensor]
Invoked with: tensor([[[[-3.8477e-01, 1.1816e+00, 1.4570e+00, ..., 2.9224e-01,
-1.9766e+00, -9.5367e-03],
[ 2.0645e+00, 3.9551e-01, -4.5410e-01, ..., -1.6504e+00,
7.0923e-02, -8.6523e-01],
[-4.0833e-02, -9.7534e-02, 9.3896e-01, ..., -8.4619e-01,
1.8555e+00, -9.7656e-01],
...,
[ 4.6826e-01, -1.3770e-01, 1.7637e+00, ..., 1.0518e+00,
7.4365e-01, 5.0293e-01],
[-7.8271e-01, 4.8401e-02, 2.3816e-01, ..., -1.6436e+00,
-1.0986e+00, -2.0332e+00],
[-1.8127e-01, 9.6094e-01, -7.3242e-01, ..., -2.0703e-01,
2.0098e+00, -1.0410e+00]],
[[-6.7529e-01, 1.8604e+00, 1.5312e+00, ..., 9.9316e-01,
-2.2383e+00, -4.2627e-01],
[ 1.1836e+00, 3.3130e-01, -2.5146e-01, ..., -8.7549e-01,
4.7217e-01, -8.6865e-01],
[-2.4307e-02, -1.3330e-01, 8.6377e-01, ..., -1.0469e+00,
2.0645e+00, -1.0664e+00],
...,
[ 1.4355e+00, -5.9326e-01, 2.1719e+00, ..., 1.5781e+00,
1.9775e+00, 1.2900e+00],
[-6.4746e-01, 9.5410e-01, -1.0752e+00, ..., -1.1836e+00,
-1.2207e+00, -2.6934e+00],
[-2.7832e-01, 1.3311e+00, -3.3374e-01, ..., -1.0078e+00,
1.4414e+00, -7.3047e-01]],
[[-8.7598e-01, 9.1064e-01, 1.7451e+00, ..., 3.5889e-01,
-2.1680e+00, -4.5435e-01],
[ 1.7236e+00, 3.0737e-01, -4.6631e-01, ..., -1.6455e+00,
5.3662e-01, -9.3848e-01],
[ 5.0812e-03, -8.0872e-02, 8.7305e-01, ..., -7.5781e-01,
1.8799e+00, -8.4033e-01],
...,
[ 1.6611e+00, -5.5615e-01, 2.9844e+00, ..., 1.7559e+00,
2.4043e+00, 1.7119e+00],
[-5.8643e-01, 6.6602e-01, -9.4238e-01, ..., -1.5762e+00,
-1.0479e+00, -3.1367e+00],
[-1.6199e-01, 9.7656e-01, -4.3188e-01, ..., -3.4351e-01,
1.8389e+00, -9.7656e-01]],
...,
[[-5.4258e+00, 4.6758e+00, -2.5586e+00, ..., 2.4961e+00,
-5.9531e+00, 9.8516e+00],
[ 1.3223e+00, 3.2043e-02, -2.2205e-01, ..., -5.5859e-01,
-4.4995e-01, -2.3462e-01],
[-2.0361e-01, -4.7070e-01, -1.1963e-01, ..., -5.4980e-01,
4.7656e-01, -5.1636e-02],
...,
[ 3.9771e-01, -1.1804e-01, 2.6733e-01, ..., -6.4087e-02,
4.0466e-02, 1.1877e-01],
[-1.3501e-01, -3.8379e-01, 4.2285e-01, ..., -1.0664e+00,
3.2495e-01, -4.0039e-01],
[-1.0020e+00, 1.4441e-01, 5.0439e-01, ..., 4.5410e-01,
2.3193e-01, -8.1055e-01]],
[[-5.3477e+00, 4.3711e+00, -3.1582e+00, ..., 1.9189e+00,
-5.1289e+00, 7.8945e+00],
[ 1.2676e+00, -4.5532e-01, -8.7830e-02, ..., -4.2603e-01,
-1.3184e-01, -2.8638e-01],
[ 4.9023e-01, -3.5376e-01, 1.5210e-01, ..., -4.3506e-01,
1.1151e-01, 2.4329e-01],
...,
[ 1.9568e-01, -4.9731e-01, 3.3594e-01, ..., -2.7393e-01,
3.8892e-01, 1.6663e-01],
[ 1.9031e-01, -4.0430e-01, 2.8540e-01, ..., -1.0693e+00,
7.0312e-01, -1.0425e-01],
[-5.1270e-01, 7.5195e-02, 5.3711e-01, ..., -1.4929e-01,
-1.3525e-01, -2.0251e-01]],
[[-6.4219e+00, 5.8047e+00, -4.6914e+00, ..., 2.6992e+00,
-4.1875e+00, 7.2500e+00],
[ 1.1543e+00, -6.0150e-02, -1.4514e-01, ..., -6.0498e-01,
-3.8623e-01, -3.8452e-02],
[-1.0822e-01, -3.5498e-01, -4.4006e-02, ..., -5.1660e-01,
1.7346e-01, 3.1812e-01],
...,
[ 2.0593e-01, -5.9570e-01, 5.1025e-01, ..., -8.7646e-02,
1.5161e-01, 4.9957e-02],
[ 7.4585e-02, 1.6327e-02, 5.9082e-01, ..., -1.3115e+00,
2.2778e-01, -5.5225e-01],
[-6.3525e-01, 9.8816e-02, 5.7471e-01, ..., 3.8281e-01,
2.7490e-01, -4.8047e-01]]]], device='cuda:0', dtype=torch.float16), tensor([[[[-1.3633e+00, 1.8105e+00, 2.1758e+00, ..., 8.9941e-01,
-2.4395e+00, 8.1055e-02],
[ 1.4541e+00, 5.8899e-02, -4.8926e-01, ..., -8.9014e-01,
-2.8442e-01, -5.2881e-01],
[-5.3223e-01, 2.1484e-01, 2.0859e+00, ..., -1.9214e-01,
2.0391e+00, -1.4443e+00],
...,
[ 1.0059e+00, -4.1748e-01, 6.4795e-01, ..., 3.2031e-01,
9.8145e-01, 5.9424e-01],
[-3.0859e-01, 6.2451e-01, -1.3359e+00, ..., -4.9414e-01,
-5.8789e-01, -7.2510e-01],
[ 4.5410e-01, 1.8707e-02, -1.4954e-03, ..., -1.6284e-01,
8.7695e-01, 3.6523e-01]],
[[-1.4277e+00, 1.9297e+00, 2.1719e+00, ..., 8.8867e-01,
-2.0410e+00, 1.8860e-01],
[ 1.0059e+00, 1.3245e-01, -2.5952e-01, ..., -9.1553e-01,
1.2585e-01, -1.8323e-01],
[-3.2471e-01, 2.7905e-01, 1.1631e+00, ..., 2.5391e-01,
1.8682e+00, -6.1768e-01],
...,
[ 2.8687e-01, 3.6255e-02, 2.2009e-01, ..., 5.7861e-02,
3.7061e-01, 3.7451e-01],
[-6.4404e-01, 6.6309e-01, -3.5327e-01, ..., -7.1875e-01,
-1.4668e+00, -8.8525e-01],
[ 3.9941e-01, 3.1885e-01, -2.0691e-01, ..., -3.8892e-01,
1.2012e+00, 7.5317e-02]],
[[-1.2139e+00, 1.6719e+00, 2.0391e+00, ..., 9.9756e-01,
-2.0195e+00, -1.8579e-01],
[ 9.4092e-01, 1.1975e-01, -2.7905e-01, ..., -1.2705e+00,
-4.7607e-03, -4.1211e-01],
[-6.7188e-01, -3.7988e-01, 1.4893e+00, ..., -1.1194e-01,
1.4678e+00, -5.5713e-01],
...,
[ 6.4844e-01, -2.4219e-01, 3.7744e-01, ..., -2.2375e-01,
8.7939e-01, 7.3730e-01],
[-7.3584e-01, 4.4751e-01, -2.3340e-01, ..., -8.9551e-01,
-1.4443e+00, -7.1094e-01],
[ 3.5889e-01, 1.3037e-01, -1.8091e-01, ..., -1.3770e-01,
1.0078e+00, 5.7031e-01]],
...,
[[-4.5234e+00, -2.7734e-01, -2.3496e+00, ..., -4.0039e-01,
-2.3379e+00, 8.6641e+00],
[-8.1982e-01, -1.3330e+00, -2.9883e-01, ..., 6.7871e-01,
2.2827e-01, -1.0262e-02],
[ 2.3315e-01, 6.9458e-02, -1.3245e-01, ..., -3.0835e-01,
-7.4561e-01, 3.3154e-01],
...,
[ 3.5083e-01, -1.7224e-01, -5.3906e-01, ..., 5.0537e-01,
-1.0834e-01, 1.6089e-01],
[-3.2202e-01, 1.5781e+00, -8.0322e-01, ..., 2.2012e+00,
6.3232e-01, 1.8193e+00],
[-2.3743e-01, -1.2363e+00, 1.0840e+00, ..., -5.6000e-02,
-2.6387e+00, 1.0859e+00]],
[[-4.0430e+00, 9.5361e-01, -2.6973e+00, ..., -4.0723e-01,
-2.1035e+00, 7.7734e+00],
[-1.3501e-01, -1.8555e+00, -1.4575e-01, ..., 7.9736e-01,
2.4780e-01, 1.1530e-01],
[-9.1980e-02, 6.4746e-01, -8.2825e-02, ..., 3.4009e-01,
-4.5142e-01, -3.9795e-01],
...,
[ 3.4229e-01, -9.1187e-02, -2.0190e-01, ..., 1.7065e-01,
8.5220e-03, 3.3105e-01],
[-4.1412e-02, 1.2920e+00, -1.4209e+00, ..., 2.1641e+00,
2.1240e-01, 6.8506e-01],
[-4.2188e-01, -6.0693e-01, 9.3604e-01, ..., 3.0249e-01,
-1.8291e+00, 1.2568e+00]],
[[-5.4609e+00, 1.7480e-01, -4.5234e+00, ..., -1.2031e+00,
-2.2705e-01, 7.3281e+00],
[-6.9580e-01, -1.4834e+00, -7.8906e-01, ..., 8.5986e-01,
3.0609e-02, 1.8054e-01],
[ 1.5495e-02, 2.2522e-02, 4.1962e-02, ..., -7.6294e-02,
-6.7334e-01, -4.7534e-01],
...,
[ 1.9080e-01, -2.9224e-01, -4.7729e-01, ..., 7.5342e-01,
2.1338e-01, 2.5562e-01],
[-4.1504e-01, 1.6699e+00, -9.0723e-01, ..., 2.3906e+00,
-1.5088e-01, 1.1572e+00],
[-2.4805e-01, -9.8975e-01, 1.0293e+00, ..., 1.8506e-01,
-2.4609e+00, 8.1152e-01]]]], device='cuda:0', dtype=torch.float16), tensor([[[[-3.2921e-03, -4.9683e-02, 2.1936e-01, ..., -3.7231e-01,
-4.8511e-01, 1.2695e+00],
[-4.5850e-01, -2.6062e-02, -3.0005e-01, ..., -3.4302e-01,
1.1816e-01, -4.1455e-01],
[-8.7830e-02, 2.4561e-01, -3.7988e-01, ..., -1.7859e-01,
2.5366e-01, -1.2805e-01],
...,
[-3.9253e-03, 5.4626e-02, -2.1729e-01, ..., -3.1250e-01,
1.6931e-01, 6.1572e-01],
[ 1.5466e-01, 9.3701e-01, -5.0293e-01, ..., 2.3071e-01,
-1.8921e-01, 3.4570e-01],
[ 3.1763e-01, -3.5498e-01, 7.3425e-02, ..., -7.6782e-02,
2.1133e-02, -1.6455e-01]],
[[-4.2969e-02, -1.5503e-01, 4.6600e-02, ..., -8.5986e-01,
-7.2363e-01, 1.5879e+00],
[-5.7861e-01, 1.8298e-01, -1.7969e-01, ..., -4.5166e-01,
2.6587e-01, -2.1997e-01],
[ 1.7664e-01, 3.4961e-01, -2.9004e-01, ..., -9.4580e-01,
7.4158e-02, 3.0566e-01],
...,
[ 1.3245e-01, 3.6401e-01, -3.4521e-01, ..., -7.2070e-01,
-1.1426e-01, 2.8589e-01],
[-1.6711e-01, 3.8770e-01, 1.7993e-01, ..., 6.2683e-02,
-1.9507e-01, -1.9312e-01],
[-2.1106e-01, -1.3928e-01, -1.7059e-02, ..., -3.5840e-01,
2.2974e-01, -3.2739e-01]],
[[-2.1838e-01, 5.0146e-01, -1.3456e-03, ..., -4.4385e-01,
-3.4302e-01, 1.4951e+00],
[-6.4111e-01, 2.3364e-01, -1.2354e-01, ..., -4.9854e-01,
5.5542e-02, -3.7085e-01],
[-1.0901e-01, 4.2432e-01, -3.8257e-01, ..., -3.7915e-01,
2.6196e-01, 3.0542e-01],
...,
[-2.5928e-01, 3.2251e-01, -1.5552e-01, ..., -9.0234e-01,
1.6943e-01, 2.9297e-01],
[-1.6272e-01, 1.2964e-01, 7.5562e-02, ..., -8.4595e-02,
-1.0297e-01, -1.8884e-01],
[-3.1738e-02, -1.1804e-01, 7.2754e-02, ..., -2.1191e-01,
-1.2427e-01, -6.3110e-02]],
...,
[[ 1.6736e-01, -4.6069e-01, 2.6929e-01, ..., -1.7078e-01,
2.4207e-01, 1.9521e+00],
[ 3.6157e-01, 7.0557e-01, -6.4014e-01, ..., 9.0723e-01,
-2.7026e-01, -1.2422e+00],
[ 1.2927e-01, 4.9951e-01, 3.4326e-01, ..., -5.0732e-01,
3.1958e-01, -6.5625e-01],
...,
[-3.4058e-01, 2.4792e-01, -2.7002e-01, ..., 4.4653e-01,
1.3271e+00, 9.6631e-01],
[ 1.2854e-01, -1.2335e-01, -1.9104e-01, ..., -1.2598e-01,
-2.0581e-01, -8.0261e-02],
[ 5.5371e-01, 2.3718e-01, -1.5588e-01, ..., 3.8513e-02,
-3.4961e-01, 7.6477e-02]],
[[-1.0352e+00, 3.8696e-01, -9.8633e-01, ..., 3.2422e-01,
2.6309e+00, 4.3286e-01],
[ 2.1790e-01, 3.5938e-01, 1.2803e+00, ..., 1.1387e+00,
-5.4053e-01, 1.5557e+00],
[ 1.0518e+00, 8.8281e-01, -2.8955e-01, ..., -3.4375e-01,
1.2988e-01, -2.3279e-01],
...,
[ 8.4473e-02, 7.0557e-01, 8.3789e-01, ..., 7.0020e-01,
4.9341e-01, -2.3499e-01],
[ 9.6289e-01, -7.3828e-01, -7.4805e-01, ..., -1.9824e-01,
6.6357e-01, -1.2236e+00],
[-1.3779e+00, 2.7612e-01, -7.2571e-02, ..., 7.2168e-01,
-2.4573e-01, -3.9233e-01]],
[[ 1.7998e+00, -7.7002e-01, 3.4937e-01, ..., -7.0898e-01,
-1.5344e-01, 2.4863e+00],
[-4.6814e-02, -4.4580e-01, -7.2461e-01, ..., 8.3887e-01,
2.6147e-01, -6.1279e-01],
[ 1.6357e-02, -1.8347e-01, 2.7368e-01, ..., -5.9766e-01,
3.5498e-01, -4.0747e-01],
...,
[ 1.0605e+00, -5.2441e-01, 4.9316e-01, ..., -1.4392e-01,
1.6514e+00, 1.0400e+00],
[-9.7168e-01, -7.6660e-01, 2.9370e-01, ..., -4.9243e-01,
4.9194e-01, -3.8501e-01],
[ 5.0000e-01, 3.0835e-01, -5.3125e-01, ..., -5.4199e-01,
2.4246e-02, -3.3398e-01]]]], device='cuda:0', dtype=torch.float16), None, None, tensor([[[ 4.2039e-45, 4.2039e-45, 4.2039e-45, ..., -8.0974e-02,
-7.0403e-02, -2.4205e-02],
[ 7.3190e-02, 1.3110e-02, 5.5102e-02, ..., 1.1241e-01,
1.1302e-01, 2.2441e-01],
[ 1.7696e-01, 2.4364e-01, -1.3768e-01, ..., -1.6252e-01,
8.9689e-02, 2.1497e-01],
...,
[-4.3121e-02, 4.5455e-02, -3.8870e-01, ..., 4.9832e-01,
-2.1960e-01, 5.9526e-01],
[ 3.6127e-02, 1.6692e-01, -3.4386e-01, ..., -2.2090e-01,
1.4602e-01, -9.0396e-02],
[-2.1450e-01, 2.7722e-01, 1.4294e-03, ..., 3.1621e-01,
3.1007e-01, 3.5658e-01]]], device='cuda:0'), 0.0, 0.125, False, -1, -1, False, None
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment