Created
November 16, 2023 08:09
-
-
Save ensean/0c19b1a00228ef7af5c38b8b0417c18d to your computer and use it in GitHub Desktop.
logs of sd on eks
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| base) [ec2-user@ip-172-16-78-199 stable-diffusion-on-eks]$ kubectl logs sdruntime-sd-on-eks-sd-webui-inference-api-6cfbc85c7d-tbsng | |
| Defaulted container "sd-webui-inference-api" out of: sd-webui-inference-api, sd-webui-queue-agent, xray-daemon | |
| The cache for model files in Transformers v4.22.0 has been updated. Migrating your old cache. This is a one-time only operation. You can interrupt this and resume the migration later on by calling `transformers.utils.move_cache()`. | |
| Launching Web UI with arguments: -f --api --skip-prepare-environment --no-hashing --listen --port 8080 --xformers --ckpt v1-5-pruned-emaonly.safetensors --ckpt-dir /tmp/models/stable-diffusion --controlnet-dir /tmp/models/controlnet --lora-dir /tmp/models/lora --vae-dir /tmp/models/vae | |
| Moving 0 files to the new cache system | |
| 0it [00:00, ?it/s] | |
| Checkpoint in --ckpt argument not found (Possible it was moved to /opt/ml/code/models/Stable-diffusion: v1-5-pruned-emaonly.safetensors | |
| 2023-11-15 23:29:49,101 - ControlNet - INFO - ControlNet v1.1.410 | |
| ControlNet preprocessor location: /opt/ml/code/extensions/sd-webui-controlnet/annotator/downloads | |
| 2023-11-15 23:29:49,234 - ControlNet - INFO - ControlNet v1.1.410 | |
| loading stable diffusion model: FileNotFoundError | |
| Traceback (most recent call last): | |
| File "/opt/conda/lib/python3.10/threading.py", line 973, in _bootstrap | |
| self._bootstrap_inner() | |
| File "/opt/conda/lib/python3.10/threading.py", line 1016, in _bootstrap_inner | |
| self.run() | |
| File "/opt/conda/lib/python3.10/threading.py", line 953, in run | |
| self._target(*self._args, **self._kwargs) | |
| File "/opt/ml/code/webui.py", line 318, in load_model | |
| shared.sd_model # noqa: B018 | |
| File "/opt/ml/code/modules/shared.py", line 754, in sd_model | |
| return modules.sd_models.model_data.get_sd_model() | |
| File "/opt/ml/code/modules/sd_models.py", line 439, in get_sd_model | |
| load_model() | |
| File "/opt/ml/code/modules/sd_models.py", line 466, in load_model | |
| checkpoint_info = checkpoint_info or select_checkpoint() | |
| File "/opt/ml/code/modules/sd_models.py", line 182, in select_checkpoint | |
| raise FileNotFoundError(error_message) | |
| FileNotFoundError: No checkpoints found. When searching for checkpoints, looked at: | |
| - file /opt/ml/code/v1-5-pruned-emaonly.safetensors | |
| - directory /opt/ml/code/models/Stable-diffusion | |
| - directory /tmp/models/stable-diffusionCan't run without a checkpoint. Find and place a .ckpt or .safetensors file into any of those locations. | |
| Stable diffusion model failed to load | |
| Applying attention optimization: xformers... done. | |
| loading stable diffusion model: FileNotFoundError | |
| Traceback (most recent call last): | |
| File "/opt/conda/lib/python3.10/threading.py", line 973, in _bootstrap | |
| self._bootstrap_inner() | |
| File "/opt/conda/lib/python3.10/threading.py", line 1016, in _bootstrap_inner | |
| self.run() | |
| File "/opt/conda/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 807, in run | |
| result = context.run(func, *args) | |
| File "/opt/ml/code/modules/api/api.py", line 331, in text2imgapi | |
| with closing(StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)) as p: | |
| File "/opt/ml/code/modules/shared.py", line 754, in sd_model | |
| return modules.sd_models.model_data.get_sd_model() | |
| File "/opt/ml/code/modules/sd_models.py", line 439, in get_sd_model | |
| load_model() | |
| File "/opt/ml/code/modules/sd_models.py", line 466, in load_model | |
| checkpoint_info = checkpoint_info or select_checkpoint() | |
| File "/opt/ml/code/modules/sd_models.py", line 182, in select_checkpoint | |
| raise FileNotFoundError(error_message) | |
| FileNotFoundError: No checkpoints found. When searching for checkpoints, looked at: | |
| - file /opt/ml/code/v1-5-pruned-emaonly.safetensors | |
| - directory /opt/ml/code/models/Stable-diffusion | |
| - directory /tmp/models/stable-diffusionCan't run without a checkpoint. Find and place a .ckpt or .safetensors file into any of those locations. | |
| Stable diffusion model failed to load | |
| loading stable diffusion model: FileNotFoundError | |
| Traceback (most recent call last): | |
| File "/opt/ml/code/modules/processing.py", line 660, in process_images | |
| sd_models.reload_model_weights() | |
| File "/opt/ml/code/modules/sd_models.py", line 547, in reload_model_weights | |
| checkpoint_info = info or select_checkpoint() | |
| File "/opt/ml/code/modules/sd_models.py", line 182, in select_checkpoint | |
| raise FileNotFoundError(error_message) | |
| FileNotFoundError: No checkpoints found. When searching for checkpoints, looked at: | |
| - file /opt/ml/code/v1-5-pruned-emaonly.safetensors | |
| - directory /opt/ml/code/models/Stable-diffusion | |
| - directory /tmp/models/stable-diffusionCan't run without a checkpoint. Find and place a .ckpt or .safetensors file into any of those locations. | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/opt/conda/lib/python3.10/threading.py", line 973, in _bootstrap | |
| self._bootstrap_inner() | |
| File "/opt/conda/lib/python3.10/threading.py", line 1016, in _bootstrap_inner | |
| self.run() | |
| File "/opt/conda/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 807, in run | |
| result = context.run(func, *args) | |
| File "/opt/ml/code/modules/api/api.py", line 342, in text2imgapi | |
| processed = process_images(p) | |
| File "/opt/ml/code/modules/processing.py", line 676, in process_images | |
| sd_models.apply_token_merging(p.sd_model, 0) | |
| File "/opt/ml/code/modules/processing.py", line 191, in sd_model | |
| return shared.sd_model | |
| File "/opt/ml/code/modules/shared.py", line 754, in sd_model | |
| return modules.sd_models.model_data.get_sd_model() | |
| File "/opt/ml/code/modules/sd_models.py", line 439, in get_sd_model | |
| load_model() | |
| File "/opt/ml/code/modules/sd_models.py", line 466, in load_model | |
| checkpoint_info = checkpoint_info or select_checkpoint() | |
| File "/opt/ml/code/modules/sd_models.py", line 182, in select_checkpoint | |
| raise FileNotFoundError(error_message) | |
| FileNotFoundError: No checkpoints found. When searching for checkpoints, looked at: | |
| - file /opt/ml/code/v1-5-pruned-emaonly.safetensors | |
| - directory /opt/ml/code/models/Stable-diffusion | |
| - directory /tmp/models/stable-diffusionCan't run without a checkpoint. Find and place a .ckpt or .safetensors file into any of those locations. | |
| Stable diffusion model failed to load | |
| *** API error: POST: http://localhost:8080/sdapi/v1/txt2img {'error': 'FileNotFoundError', 'detail': '', 'body': '', 'errors': "No checkpoints found. When searching for checkpoints, looked at:\n - file /opt/ml/code/v1-5-pruned-emaonly.safetensors\n - directory /opt/ml/code/models/Stable-diffusion\n - directory /tmp/models/stable-diffusionCan't run without a checkpoint. Find and place a .ckpt or .safetensors file into any of those locations."} | |
| Traceback (most recent call last): | |
| File "/opt/conda/lib/python3.10/site-packages/anyio/streams/memory.py", line 98, in receive | |
| return self.receive_nowait() | |
| File "/opt/conda/lib/python3.10/site-packages/anyio/streams/memory.py", line 93, in receive_nowait | |
| raise WouldBlock | |
| anyio.WouldBlock | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/base.py", line 78, in call_next | |
| message = await recv_stream.receive() | |
| File "/opt/conda/lib/python3.10/site-packages/anyio/streams/memory.py", line 118, in receive | |
| raise EndOfStream | |
| anyio.EndOfStream | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/opt/ml/code/modules/api/api.py", line 152, in exception_handling | |
| return await call_next(request) | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/base.py", line 84, in call_next | |
| raise app_exc | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/base.py", line 70, in coro | |
| await self.app(scope, receive_or_disconnect, send_no_error) | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/base.py", line 108, in __call__ | |
| response = await self.dispatch_func(request, call_next) | |
| File "/opt/ml/code/modules/api/api.py", line 116, in log_and_time | |
| res: Response = await call_next(req) | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/base.py", line 84, in call_next | |
| raise app_exc | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/base.py", line 70, in coro | |
| await self.app(scope, receive_or_disconnect, send_no_error) | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/cors.py", line 84, in __call__ | |
| await self.app(scope, receive, send) | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/gzip.py", line 24, in __call__ | |
| await responder(scope, receive, send) | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/gzip.py", line 44, in __call__ | |
| await self.app(scope, receive, self.send_with_gzip) | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__ | |
| raise exc | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__ | |
| await self.app(scope, receive, sender) | |
| File "/opt/conda/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__ | |
| raise e | |
| File "/opt/conda/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__ | |
| await self.app(scope, receive, send) | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__ | |
| await route.handle(scope, receive, send) | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle | |
| await self.app(scope, receive, send) | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/routing.py", line 66, in app | |
| response = await func(request) | |
| File "/opt/conda/lib/python3.10/site-packages/fastapi/routing.py", line 237, in app | |
| raw_response = await run_endpoint_function( | |
| File "/opt/conda/lib/python3.10/site-packages/fastapi/routing.py", line 165, in run_endpoint_function | |
| return await run_in_threadpool(dependant.call, **values) | |
| File "/opt/conda/lib/python3.10/site-packages/starlette/concurrency.py", line 41, in run_in_threadpool | |
| return await anyio.to_thread.run_sync(func, *args) | |
| File "/opt/conda/lib/python3.10/site-packages/anyio/to_thread.py", line 33, in run_sync | |
| return await get_asynclib().run_sync_in_worker_thread( | |
| File "/opt/conda/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 877, in run_sync_in_worker_thread | |
| return await future | |
| File "/opt/conda/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 807, in run | |
| result = context.run(func, *args) | |
| File "/opt/ml/code/modules/api/api.py", line 342, in text2imgapi | |
| processed = process_images(p) | |
| File "/opt/ml/code/modules/processing.py", line 660, in process_images | |
| sd_models.reload_model_weights() | |
| File "/opt/ml/code/modules/sd_models.py", line 547, in reload_model_weights | |
| checkpoint_info = info or select_checkpoint() | |
| File "/opt/ml/code/modules/sd_models.py", line 182, in select_checkpoint | |
| raise FileNotFoundError(error_message) | |
| FileNotFoundError: No checkpoints found. When searching for checkpoints, looked at: | |
| - file /opt/ml/code/v1-5-pruned-emaonly.safetensors | |
| - directory /opt/ml/code/models/Stable-diffusion | |
| - directory /tmp/models/stable-diffusionCan't run without a checkpoint. Find and place a .ckpt or .safetensors file into any of those locations. | |
| --- |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment