Created
August 15, 2022 23:48
-
-
Save clarng/f69cb474447d3e50cf2dd0fedba45a99 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| kopf==1.35.4 | |
| kubernetes==23.6.0 | |
| libclang==14.0.6 | |
| lightgbm==3.3.2 | |
| lightgbm-ray @ git+https://github.com/ray-project/lightgbm_ray@e3a35f7382b6a4f3526162e206e3759f7130d4f3 | |
| llvmlite==0.34.0 | |
| log-symbols==0.0.14 | |
| lxml==4.9.0 | |
| lz4==4.0.1 | |
| Markdown==3.4.1 | |
| markdown-it-py==1.1.0 | |
| MarkupSafe==2.1.1 | |
| matplotlib==3.5.2 | |
| matplotlib-inline==0.1.3 | |
| mccabe==0.6.1 | |
| mdit-py-plugins==0.2.8 | |
| memray==1.1.0 | |
| mistune==0.8.4 | |
| mock==4.0.3 | |
| more-itertools==8.13.0 | |
| moto==1.3.0 | |
| msal==1.18.0 | |
| msal-extensions==0.3.1 | |
| msgpack==1.0.3 | |
| msrest==0.6.21 | |
| msrestazure==0.6.4 | |
| multidict==6.0.2 | |
| mypy==0.782 | |
| mypy-extensions==0.4.3 | |
| myst-nb==0.13.1 | |
| myst-parser==0.15.2 | |
| nbclient==0.5.13 | |
| nbconvert==6.5.0 | |
| nbdime==3.1.1 | |
| nbformat==5.4.0 | |
| nest-asyncio==1.5.5 | |
| networkx==2.6.3 | |
| notebook==6.4.12 | |
| numba==0.51.2 | |
| numpy==1.21.6 | |
| nvidia-ml-py3==7.352.0 | |
| oauthlib==3.2.0 | |
| objgraph==3.5.0 | |
| opencensus==0.9.0 | |
| opencensus-context==0.1.2 | |
| openpyxl==3.0.10 | |
| opentelemetry-api==1.1.0 | |
| opentelemetry-exporter-otlp==1.1.0 | |
| opentelemetry-exporter-otlp-proto-grpc==1.1.0 | |
| opentelemetry-proto==1.1.0 | |
| opentelemetry-sdk==1.1.0 | |
| opentelemetry-semantic-conventions==0.20b0 | |
| opt-einsum==3.3.0 | |
| packaging==21.3 | |
| pandas==1.3.5 | |
| pandocfilters==1.5.0 | |
| paramiko==2.11.0 | |
| parso==0.8.3 | |
| path==16.4.0 | |
| path.py==12.5.0 | |
| pathspec==0.9.0 | |
| pathtools==0.1.2 | |
| pexpect==4.8.0 | |
| pickle5==0.0.11 | |
| pickleshare==0.7.5 | |
| Pillow==9.1.1 | |
| pip==21.2.2 | |
| pkginfo==1.8.2 | |
| platformdirs==2.5.2 | |
| pluggy==0.13.1 | |
| polars==0.13.50 | |
| portalocker==1.7.1 | |
| prometheus-client==0.13.1 | |
| promise==2.3 | |
| prompt-toolkit==3.0.29 | |
| protobuf==3.19.4 | |
| proxy.py==2.4.3 | |
| psutil==5.9.1 | |
| ptyprocess==0.7.0 | |
| py==1.11.0 | |
| py-spy==0.3.12 | |
| py4j==0.10.9 | |
| pyaml==21.10.1 | |
| pyarrow==6.0.1 | |
| pyasn1==0.4.8 | |
| pyasn1-modules==0.2.8 | |
| pycodestyle==2.7.0 | |
| pycosat==0.6.3 | |
| pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work | |
| pydantic==1.9.1 | |
| pydata-sphinx-theme==0.7.2 | |
| pyDeprecate==0.3.1 | |
| pyflakes==2.3.1 | |
| pygame==2.1.2 | |
| Pygments==2.11.2 | |
| PyJWT==2.4.0 | |
| PyNaCl==1.5.0 | |
| pyOpenSSL @ file:///opt/conda/conda-bld/pyopenssl_1643788558760/work | |
| pyparsing==3.0.9 | |
| pyrsistent==0.18.1 | |
| PySocks @ file:///tmp/build/80754af9/pysocks_1594394576006/work | |
| pyspark==3.1.2 | |
| pytest==5.4.3 | |
| pytest-asyncio==0.16.0 | |
| pytest-fixture-config==1.7.0 | |
| pytest-forked==1.4.0 | |
| pytest-lazy-fixture==0.6.3 | |
| pytest-rerunfailures==10.2 | |
| pytest-shutil==1.7.0 | |
| pytest-sugar==0.9.4 | |
| pytest-timeout==2.1.0 | |
| pytest-virtualenv==1.7.0 | |
| pytest_docker_tools==0.2.3 | |
| python-dateutil==2.8.2 | |
| python-json-logger==2.0.2 | |
| pytorch-lightning==1.5.10 | |
| pytz==2022.1 | |
| PyWavelets==1.3.0 | |
| PyYAML==6.0 | |
| pyzmq==23.2.0 | |
| -e git+ssh://git@github.com/ray-project/ray.git@be4e7a7d898ebb60cdd993c221e5fdf7a8ce3bbd#egg=ray&subdirectory=python | |
| ray-cpp==1.13.0 | |
| ray-lightning @ git+https://github.com/ray-project/ray_lightning@6aed848f757a03c03166c1a9bddfeea5153e7b90 | |
| -e git+ssh://git@github.com/ray-project/ray.git@be4e7a7d898ebb60cdd993c221e5fdf7a8ce3bbd#egg=ray_release&subdirectory=release | |
| redis==3.5.3 | |
| regex==2022.7.25 | |
| requests==2.25.1 | |
| requests-oauthlib==1.3.1 | |
| rich==12.4.4 | |
| rsa==4.8 | |
| ruamel-yaml-conda @ file:///tmp/build/80754af9/ruamel_yaml_1616016701961/work | |
| s3transfer==0.3.7 | |
| scikit-image==0.19.2 | |
| scikit-learn==0.24.2 | |
| scikit-optimize==0.9.0 | |
| scipy==1.7.3 | |
| Send2Trash==1.8.0 | |
| sentry-sdk==1.9.4 | |
| setproctitle==1.2.2 | |
| setuptools==59.5.0 | |
| shortuuid==1.0.9 | |
| six==1.13.0 | |
| smart-open==6.0.0 | |
| smmap==5.0.0 | |
| sniffio==1.2.0 | |
| snowballstemmer==2.2.0 | |
| soupsieve==2.3.2.post1 | |
| Sphinx==4.3.2 | |
| sphinx-book-theme==0.1.7 | |
| sphinx-click==3.0.2 | |
| sphinx-copybutton==0.4.0 | |
| sphinx-external-toc==0.2.3 | |
| sphinx-jsonschema==1.17.2 | |
| sphinx-panels==0.6.0 | |
| sphinx-sitemap==2.2.0 | |
| sphinx-thebe==0.1.1 | |
| sphinx-togglebutton==0.2.3 | |
| sphinx-version-warning==1.1.2 | |
| sphinxcontrib-applehelp==1.0.2 | |
| sphinxcontrib-devhelp==1.0.2 | |
| sphinxcontrib-htmlhelp==2.0.0 | |
| sphinxcontrib-jsmath==1.0.1 | |
| sphinxcontrib-qthelp==1.0.3 | |
| sphinxcontrib-serializinghtml==1.1.5 | |
| sphinxcontrib.yt==0.2.2 | |
| sphinxemoji==0.2.0 | |
| spinners==0.0.24 | |
| SQLAlchemy==1.4.39 | |
| starlette==0.18.0 | |
| tabulate==0.8.9 | |
| tensorboard==2.9.1 | |
| tensorboard-data-server==0.6.1 | |
| tensorboard-plugin-wit==1.8.1 | |
| tensorboardX==2.5 | |
| tensorflow==2.9.1 | |
| tensorflow-estimator==2.9.0 | |
| tensorflow-io-gcs-filesystem==0.26.0 | |
| termcolor==1.1.0 | |
| terminado==0.15.0 | |
| testfixtures==6.18.5 | |
| threadpoolctl==3.1.0 | |
| tifffile==2021.11.2 | |
| tinycss2==1.1.1 | |
| tokenizers==0.12.1 | |
| toml==0.10.2 | |
| tomli==1.2.3 | |
| torch==1.12.1 | |
| torchmetrics==0.9.3 | |
| tornado==6.1 | |
| tqdm @ file:///opt/conda/conda-bld/tqdm_1647339053476/work | |
| traitlets==5.3.0 | |
| transformers==4.21.1 | |
| tune-sklearn @ git+https://github.com/ray-project/tune-sklearn@67af5a931b9ed9cc7d75fa892d7146a40e569c1c | |
| typed-ast==1.4.3 | |
| typing_extensions==4.2.0 | |
| uritemplate==3.0.1 | |
| urllib3==1.26.11 | |
| uvicorn==0.16.0 | |
| virtualenv==20.14.1 | |
| wandb==0.13.1 | |
| wcwidth==0.2.5 | |
| webencodings==0.5.1 | |
| websocket-client==1.3.2 | |
| Werkzeug==2.1.2 | |
| wheel==0.37.1 | |
| widgetsnbextension==3.6.1 | |
| wrapt==1.14.1 | |
| xgboost==1.6.1 | |
| xgboost-ray @ git+https://github.com/ray-project/xgboost_ray@08f3bc1fba96d98b5571e5e746b3b87b55e31d16 | |
| xlrd==2.0.1 | |
| xmltodict==0.13.0 | |
| yarl==1.7.2 | |
| zipp==3.8.0 | |
| [INFO 2022-08-15 15:46:09,065] minimal.py: 190 Tests uses compute template with name oss-release-tests_mWEC__compute__pipelined_data_ingest_benchmark__d052e40b4a27396f394add1ce0f63ea8d9965f19ce3b4e001f9cb2240f08b25c. Looking up existing cluster computes. | |
| [INFO 2022-08-15 15:46:09,430] minimal.py: 211 Cluster compute already exists with ID cpt_X3z4SS2vjMi9cG9zYuYhmwUg | |
| [INFO 2022-08-15 15:46:09,431] minimal.py: 29 Test uses a cluster env with name oss-release-tests_mWEC__env__pipelined_data_ingest_benchmark__c2217455c5ba64dcde7623c71c3ab32e137b11359257cd69f06879287f898bad. Looking up existing cluster envs with this name. | |
| [INFO 2022-08-15 15:46:09,534] minimal.py: 58 Cluster env not found. Creating new one. | |
| [INFO 2022-08-15 15:46:09,737] minimal.py: 80 Cluster env created with ID apt_NB6SbmtjcGYQGhnb2Dm2QwP4 | |
| [INFO 2022-08-15 15:46:09,811] minimal.py: 131 Waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish... | |
| [INFO 2022-08-15 15:46:09,811] minimal.py: 133 Track progress here: https://console.anyscale-staging.com/o/anyscale-internal/configurations/app-config-details/bld_y5iNN72jJM5UvmWCVstPLFza | |
| [INFO 2022-08-15 15:46:40,060] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (30 seconds) ... | |
| [INFO 2022-08-15 15:47:10,156] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (60 seconds) ... | |
| [INFO 2022-08-15 15:47:40,963] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (91 seconds) ... | |
| [INFO 2022-08-15 15:48:11,098] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (121 seconds) ... | |
| [INFO 2022-08-15 15:48:40,374] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (150 seconds) ... | |
| [INFO 2022-08-15 15:49:10,233] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (180 seconds) ... | |
| [INFO 2022-08-15 15:49:40,356] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (210 seconds) ... | |
| [INFO 2022-08-15 15:50:12,117] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (242 seconds) ... | |
| [INFO 2022-08-15 15:50:39,940] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (270 seconds) ... | |
| [INFO 2022-08-15 15:51:10,304] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (300 seconds) ... | |
| [INFO 2022-08-15 15:51:39,908] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (330 seconds) ... | |
| [INFO 2022-08-15 15:52:10,212] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (360 seconds) ... | |
| [INFO 2022-08-15 15:52:39,926] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (390 seconds) ... | |
| [INFO 2022-08-15 15:53:10,786] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (420 seconds) ... | |
| [INFO 2022-08-15 15:53:40,679] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (450 seconds) ... | |
| [INFO 2022-08-15 15:54:10,751] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (480 seconds) ... | |
| [INFO 2022-08-15 15:54:40,619] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (510 seconds) ... | |
| [INFO 2022-08-15 15:55:10,156] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (540 seconds) ... | |
| [INFO 2022-08-15 15:55:40,750] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (570 seconds) ... | |
| [INFO 2022-08-15 15:56:09,822] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (600 seconds) ... | |
| [INFO 2022-08-15 15:56:41,379] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (631 seconds) ... | |
| [INFO 2022-08-15 15:57:10,185] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (660 seconds) ... | |
| [INFO 2022-08-15 15:57:41,639] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (691 seconds) ... | |
| [INFO 2022-08-15 15:58:10,717] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (720 seconds) ... | |
| [INFO 2022-08-15 15:58:39,969] minimal.py: 140 ... still waiting for build bld_y5iNN72jJM5UvmWCVstPLFza to finish (750 seconds) ... | |
| [INFO 2022-08-15 15:58:42,330] minimal.py: 156 Build succeeded. | |
| [INFO 2022-08-15 15:58:42,331] full.py: 28 Creating cluster pipelined_data_ingest_benchmark_1660603564 | |
| [INFO 2022-08-15 15:58:42,331] full.py: 29 Autosuspend time: 15 minutes | |
| [INFO 2022-08-15 15:58:43,110] full.py: 45 Starting cluster pipelined_data_ingest_benchmark_1660603564 (ses_kJVJXBUVXRiLF8rVm76sQvcQ) | |
| [INFO 2022-08-15 15:58:43,110] full.py: 49 Link to cluster: https://console.anyscale-staging.com/o/anyscale-internal/projects/prj_mWECugke9RzMh79BZQqeykjN/clusters/ses_kJVJXBUVXRiLF8rVm76sQvcQ | |
| [INFO 2022-08-15 15:58:43,570] full.py: 63 Waiting for cluster pipelined_data_ingest_benchmark_1660603564... | |
| [INFO 2022-08-15 15:59:13,877] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660603564 (30 seconds) ... | |
| [INFO 2022-08-15 15:59:44,145] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660603564 (60 seconds) ... | |
| [INFO 2022-08-15 16:00:13,649] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660603564 (90 seconds) ... | |
| [INFO 2022-08-15 16:00:43,816] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660603564 (120 seconds) ... | |
| [INFO 2022-08-15 16:01:14,163] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660603564 (150 seconds) ... | |
| [INFO 2022-08-15 16:01:44,069] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660603564 (180 seconds) ... | |
| [INFO 2022-08-15 16:02:14,041] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660603564 (210 seconds) ... | |
| [INFO 2022-08-15 16:02:43,500] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660603564 (240 seconds) ... | |
| [INFO 2022-08-15 16:03:13,349] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660603564 (270 seconds) ... | |
| [INFO 2022-08-15 16:03:43,320] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660603564 (300 seconds) ... | |
| [INFO 2022-08-15 16:04:13,399] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660603564 (330 seconds) ... | |
| [INFO 2022-08-15 16:04:31,903] session_controller.py: 29 Uploading <cwd> to <cwd> using SessionController | |
| [WARNING 2022-08-15 16:04:33,994] util.py: 235 Dropping the empty legacy field head_node. head_nodeis not supported for ray>=2.0.0. It is recommended to removehead_node from the cluster config. | |
| [WARNING 2022-08-15 16:04:33,994] util.py: 235 Dropping the empty legacy field worker_nodes. worker_nodesis not supported for ray>=2.0.0. It is recommended to removeworker_nodes from the cluster config. | |
| 2022-08-15 16:04:34,000 INFO commands.py:349 -- Checking External environment settings | |
| 2022-08-15 16:04:34,000 VINFO utils.py:150 -- Creating AWS resource `ec2` in `us-west-2` | |
| 2022-08-15 16:04:37,461 VINFO utils.py:150 -- Creating AWS resource `ec2` in `us-west-2` | |
| 2022-08-15 16:04:38,433 INFO updater.py:239 -- [1/2] Processing file mounts | |
| 2022-08-15 16:04:38,679 INFO command_runner.py:393 -- Fetched IP: 34.221.217.222 | |
| 2022-08-15 16:04:38,679 INFO log_timer.py:30 -- NodeUpdater: ins_pmgpS1aMRuPAuXYmSRm7a7d9: Got IP [LogTimer=246ms] | |
| 2022-08-15 16:04:38,680 VINFO command_runner.py:554 -- Running `mkdir -p /tmp/ray_tmp_mount/anyscale-dev-stag-f28aaaf56a12fa6c/home/ray && chown -R ubuntu /tmp/ray_tmp_mount/anyscale-dev-stag-f28aaaf56a12fa6c/home/ray` | |
| 2022-08-15 16:04:38,680 VVINFO command_runner.py:557 -- Full command is `ssh -tt -i /home/ray/.ssh/anyscale/prj_mWECugke9RzMh79BZQqeykjN/ses_kJVJXBUVXRiLF8rVm76sQvcQ.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_070dd72385/41236c44ed/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@34.221.217.222 bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (mkdir -p /tmp/ray_tmp_mount/anyscale-dev-stag-f28aaaf56a12fa6c/home/ray && chown -R ubuntu /tmp/ray_tmp_mount/anyscale-dev-stag-f28aaaf56a12fa6c/home/ray)'` | |
| Warning: Permanently added '34.221.217.222' (ECDSA) to the list of known hosts. | |
| Shared connection to 34.221.217.222 closed. | |
| 2022-08-15 16:04:39,418 VINFO command_runner.py:597 -- Running `rsync --rsh ssh -i /home/ray/.ssh/anyscale/prj_mWECugke9RzMh79BZQqeykjN/ses_kJVJXBUVXRiLF8rVm76sQvcQ.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_070dd72385/41236c44ed/%C -o ControlPersist=10s -o ConnectTimeout=120s -avz --exclude **/.git --exclude **/.git/** --filter dir-merge,- .gitignore /home/ray/github/ray/release/nightly_tests/dataset/ ubuntu@34.221.217.222:/tmp/ray_tmp_mount/anyscale-dev-stag-f28aaaf56a12fa6c/home/ray/oss-release-tests/` | |
| sending incremental file list | |
| ./ | |
| .anyscale.yaml | |
| app_config.yaml | |
| data_ingest_benchmark.py | |
| data_ingest_benchmark_compute.yaml | |
| dataset_ingest_400G_compute.yaml | |
| dataset_random_access.py | |
| dataset_shuffle_data_loader.py | |
| inference.py | |
| inference.yaml | |
| parquet_metadata_resolution.py | |
| pipelined_ingestion_app.yaml | |
| pipelined_ingestion_compute.yaml | |
| pipelined_training.py | |
| pipelined_training_app.yaml | |
| pipelined_training_compute.yaml | |
| ray_sgd_runner.py | |
| ray_sgd_training.py | |
| ray_sgd_training_app.yaml | |
| ray_sgd_training_compute.yaml | |
| ray_sgd_training_compute_no_gpu.yaml | |
| ray_sgd_training_smoke_compute.yaml | |
| shuffle_app_config.yaml | |
| shuffle_compute.yaml | |
| sort.py | |
| wait_cluster.py | |
| sent 27,317 bytes received 494 bytes 55,622.00 bytes/sec | |
| total size is 75,052 speedup is 2.70 | |
| 2022-08-15 16:04:39,662 VINFO command_runner.py:554 -- Running `docker inspect -f '{{.State.Running}}' ray_container || true` | |
| 2022-08-15 16:04:39,662 VVINFO command_runner.py:557 -- Full command is `ssh -tt -i /home/ray/.ssh/anyscale/prj_mWECugke9RzMh79BZQqeykjN/ses_kJVJXBUVXRiLF8rVm76sQvcQ.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_070dd72385/41236c44ed/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@34.221.217.222 bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (docker inspect -f '"'"'{{.State.Running}}'"'"' ray_container || true)'` | |
| Shared connection to 34.221.217.222 closed. | |
| 2022-08-15 16:04:39,817 VINFO updater.py:537 -- `rsync`ed /home/ray/github/ray/release/nightly_tests/dataset/ (local) to /home/ray/oss-release-tests/ (remote) | |
| 2022-08-15 16:04:39,818 INFO updater.py:234 -- /home/ray/oss-release-tests/ from /home/ray/github/ray/release/nightly_tests/dataset/ | |
| 2022-08-15 16:04:39,818 INFO log_timer.py:30 -- NodeUpdater: ins_pmgpS1aMRuPAuXYmSRm7a7d9: Synced /home/ray/github/ray/release/nightly_tests/dataset/ to /home/ray/oss-release-tests/ [LogTimer=1385ms] | |
| 2022-08-15 16:04:39,818 INFO updater.py:257 -- [2/2] No worker file mounts to sync | |
| (anyscale +18m35.3s) Pushed to session pipelined_data_ingest_benchmark_1660603564. View at https://console.anyscale-staging.com/projects/prj_mWECugke9RzMh79BZQqeykjN/clusters/ses_kJVJXBUVXRiLF8rVm76sQvcQ | |
| [INFO 2022-08-15 16:04:39,820] sdk_runner.py: 90 Running command in cluster pipelined_data_ingest_benchmark_1660603564: TEST_OUTPUT_JSON=/tmp/release_test_out.json python wait_cluster.py 20 3000 | |
| [INFO 2022-08-15 16:04:39,820] sdk_runner.py: 95 Link to cluster: https://console.anyscale-staging.com/o/anyscale-internal/projects/prj_mWECugke9RzMh79BZQqeykjN/clusters/ses_kJVJXBUVXRiLF8rVm76sQvcQ | |
| [INFO 2022-08-15 16:05:10,665] sdk_runner.py: 121 ... command still running ...(30 seconds) ... | |
| [INFO 2022-08-15 16:05:40,797] sdk_runner.py: 121 ... command still running ...(60 seconds) ... | |
| [INFO 2022-08-15 16:06:10,717] sdk_runner.py: 121 ... command still running ...(90 seconds) ... | |
| [INFO 2022-08-15 16:06:40,717] sdk_runner.py: 121 ... command still running ...(120 seconds) ... | |
| [INFO 2022-08-15 16:07:10,023] sdk_runner.py: 121 ... command still running ...(150 seconds) ... | |
| [INFO 2022-08-15 16:07:40,603] sdk_runner.py: 121 ... command still running ...(180 seconds) ... | |
| [INFO 2022-08-15 16:08:10,881] sdk_runner.py: 121 ... command still running ...(211 seconds) ... | |
| [INFO 2022-08-15 16:08:40,329] sdk_runner.py: 121 ... command still running ...(240 seconds) ... | |
| [INFO 2022-08-15 16:09:10,556] sdk_runner.py: 121 ... command still running ...(270 seconds) ... | |
| [INFO 2022-08-15 16:09:40,408] sdk_runner.py: 121 ... command still running ...(300 seconds) ... | |
| [INFO 2022-08-15 16:10:10,244] sdk_runner.py: 121 ... command still running ...(330 seconds) ... | |
| [INFO 2022-08-15 16:10:22,921] sdk_runner.py: 90 Running command in cluster pipelined_data_ingest_benchmark_1660603564: TEST_OUTPUT_JSON=/tmp/release_test_out.json python data_ingest_benchmark.py --dataset-size-gb=200 --num-workers=20 --streaming | |
| [INFO 2022-08-15 16:10:22,921] sdk_runner.py: 95 Link to cluster: https://console.anyscale-staging.com/o/anyscale-internal/projects/prj_mWECugke9RzMh79BZQqeykjN/clusters/ses_kJVJXBUVXRiLF8rVm76sQvcQ | |
| [INFO 2022-08-15 16:10:53,648] sdk_runner.py: 121 ... command still running ...(30 seconds) ... | |
| [INFO 2022-08-15 16:11:24,298] sdk_runner.py: 121 ... command still running ...(61 seconds) ... | |
| [INFO 2022-08-15 16:11:53,537] sdk_runner.py: 121 ... command still running ...(90 seconds) ... | |
| [INFO 2022-08-15 16:12:23,377] sdk_runner.py: 121 ... command still running ...(120 seconds) ... | |
| [INFO 2022-08-15 16:12:53,486] sdk_runner.py: 121 ... command still running ...(150 seconds) ... | |
| [INFO 2022-08-15 16:13:24,049] sdk_runner.py: 121 ... command still running ...(180 seconds) ... | |
| [INFO 2022-08-15 16:13:53,408] sdk_runner.py: 121 ... command still running ...(210 seconds) ... | |
| [INFO 2022-08-15 16:14:23,656] sdk_runner.py: 121 ... command still running ...(240 seconds) ... | |
| [INFO 2022-08-15 16:14:53,972] sdk_runner.py: 121 ... command still running ...(271 seconds) ... | |
| [ERROR 2022-08-15 16:15:23,712] glue.py: 283 Cluster command timed out after 300 seconds. | |
| Traceback (most recent call last): | |
| File "/home/ray/github/ray/release/ray_release/glue.py", line 254, in run_release_test | |
| command, env=command_env, timeout=command_timeout | |
| File "/home/ray/github/ray/release/ray_release/command_runner/sdk_runner.py", line 116, in run_command | |
| f"Cluster command timed out after {timeout} seconds." | |
| ray_release.exception.CommandTimeout: Cluster command timed out after 300 seconds. | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/home/ray/github/ray/release/ray_release/glue.py", line 261, in run_release_test | |
| raise TestCommandTimeout(e) | |
| ray_release.exception.TestCommandTimeout: Cluster command timed out after 300 seconds. | |
| [INFO 2022-08-15 16:15:27,677] log.py: 11 Test pipelined_data_ingest_benchmark finished after 0.00 seconds. Last logs: | |
| (ConsumingActor pid=244, ip=172.31.95.211) | |
| Stage 0: : 3it [02:55, 70.03s/it] | |
| (ConsumingActor pid=236, ip=172.31.95.211) | |
| Stage 0: : 3it [02:55, 70.31s/it] | |
| (ConsumingActor pid=241, ip=172.31.95.211) | |
| Stage 0: : 3it [02:55, 70.31s/it] | |
| (ConsumingActor pid=246, ip=172.31.95.211) | |
| Stage 0: : 3it [02:59, 71.69s/it] | |
| (ConsumingActor pid=242, ip=172.31.95.211) | |
| Stage 0: : 3it [03:00, 72.16s/it] | |
| [INFO 2022-08-15 16:15:27,678] log.py: 17 Got the following metadata: | |
| name: pipelined_data_ingest_benchmark | |
| status: timeout | |
| runtime: 0.00 | |
| stable: True | |
| buildkite_url: | |
| wheels_url: https://s3-us-west-2.amazonaws.com/ray-wheels/master/34c494260f0de6539b96554319f5394c0d004a06/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl | |
| cluster_url: https://console.anyscale-staging.com/o/anyscale-internal/projects/prj_mWECugke9RzMh79BZQqeykjN/clusters/ses_kJVJXBUVXRiLF8rVm76sQvcQ | |
| [INFO 2022-08-15 16:15:27,678] log.py: 36 Did not find any results. | |
| [ERROR 2022-08-15 16:15:27,678] run_release_test.py: 159 Cluster command timed out after 300 seconds. | |
| Traceback (most recent call last): | |
| File "/home/ray/github/ray/release/ray_release/glue.py", line 254, in run_release_test | |
| command, env=command_env, timeout=command_timeout | |
| File "/home/ray/github/ray/release/ray_release/command_runner/sdk_runner.py", line 116, in run_command | |
| f"Cluster command timed out after {timeout} seconds." | |
| ray_release.exception.CommandTimeout: Cluster command timed out after 300 seconds. | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "release/ray_release/scripts/run_release_test.py", line 155, in main | |
| no_terminate=no_terminate, | |
| File "/home/ray/github/ray/release/ray_release/glue.py", line 333, in run_release_test | |
| raise pipeline_exception | |
| File "/home/ray/github/ray/release/ray_release/glue.py", line 261, in run_release_test | |
| raise TestCommandTimeout(e) | |
| ray_release.exception.TestCommandTimeout: Cluster command timed out after 300 seconds. | |
| [INFO 2022-08-15 16:15:27,678] run_release_test.py: 163 Release test pipeline for test pipelined_data_ingest_benchmark completed. Returning with exit code = 42 | |
| (base) ray@90b689302baa:~/github/ray$ | |
| (base) ray@90b689302baa:~/github/ray$ | |
| (base) ray@90b689302baa:~/github/ray$ | |
| (base) ray@90b689302baa:~/github/ray$ | |
| (base) ray@90b689302baa:~/github/ray$ | |
| (base) ray@90b689302baa:~/github/ray$ | |
| (base) ray@90b689302baa:~/github/ray$ | |
| (base) ray@90b689302baa:~/github/ray$ python release/ray_release/scripts/run_release_test.py pipelined_data_ingest_benchmark --env staging --ray-wheels master | |
| Cloning into '/tmp/tmpl78c5kx6'... | |
| remote: Enumerating objects: 15771, done. | |
| remote: Total 15771 (delta 0), reused 0 (delta 0), pack-reused 15771 | |
| Receiving objects: 100% (15771/15771), 7.96 MiB | 5.79 MiB/s, done. | |
| Resolving deltas: 100% (35/35), done. | |
| [INFO 2022-08-15 16:20:36,589] wheels.py: 187 Using Ray wheels URL: https://s3-us-west-2.amazonaws.com/ray-wheels/master/a3236b62254348ed1c09062d57be7a2fd8e6c4af/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl | |
| [INFO 2022-08-15 16:20:36,590] wheels.py: 160 Waiting up to 7200 seconds until URL is available (https://s3-us-west-2.amazonaws.com/ray-wheels/master/a3236b62254348ed1c09062d57be7a2fd8e6c4af/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl) | |
| [INFO 2022-08-15 16:20:36,749] wheels.py: 177 URL is now available: https://s3-us-west-2.amazonaws.com/ray-wheels/master/a3236b62254348ed1c09062d57be7a2fd8e6c4af/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl | |
| [INFO 2022-08-15 16:20:37,435] aws.py: 36 Loaded anyscale credentials from local storage. | |
| Authenticating | |
| Loaded Anyscale authentication token from ANYSCALE_CLI_TOKEN. | |
| [INFO 2022-08-15 16:20:39,738] util.py: 133 Re-installing `anyscale` package | |
| [INFO 2022-08-15 16:20:43,468] glue.py: 185 Installed python packages: | |
| absl-py==1.2.0 | |
| adal==1.2.7 | |
| aiobotocore==1.2.2 | |
| aiohttp==3.7.4.post0 | |
| aiohttp-cors==0.7.0 | |
| aiohttp-middlewares==1.2.1 | |
| aioitertools==0.10.0 | |
| aioredis==1.3.1 | |
| aiorwlock==1.3.0 | |
| aiosignal==1.2.0 | |
| alabaster==0.7.12 | |
| anyio==3.6.1 | |
| anyscale==0.5.42 | |
| applicationinsights==0.11.10 | |
| argcomplete==1.12.3 | |
| argon2-cffi==21.3.0 | |
| argon2-cffi-bindings==21.2.0 | |
| asgiref==3.5.2 | |
| astunparse==1.6.3 | |
| async-exit-stack==1.0.1 | |
| async-generator==1.10 | |
| async-timeout==3.0.1 | |
| asyncmock==0.4.2 | |
| asynctest==0.13.0 | |
| attrs==21.4.0 | |
| autodoc-pydantic==1.6.1 | |
| aws-xray-sdk==0.95 | |
| azure-cli-core==2.29.1 | |
| azure-cli-telemetry==1.0.6 | |
| azure-common==1.1.28 | |
| azure-core==1.24.1 | |
| azure-identity==1.7.0 | |
| azure-mgmt-compute==23.1.0 | |
| azure-mgmt-core==1.2.2 | |
| azure-mgmt-network==19.0.0 | |
| azure-mgmt-resource==20.0.0 | |
| Babel==2.10.3 | |
| backcall==0.2.0 | |
| backoff==1.10.0 | |
| bcrypt==3.2.2 | |
| beautifulsoup4==4.11.1 | |
| black==21.12b0 | |
| bleach==5.0.0 | |
| blessed==1.19.1 | |
| boto==2.49.0 | |
| boto3==1.16.52 | |
| botocore==1.19.52 | |
| brotlipy==0.7.0 | |
| cachetools==5.2.0 | |
| certifi==2022.6.15 | |
| cffi @ file:///opt/conda/conda-bld/cffi_1642701102775/work | |
| chardet==4.0.0 | |
| charset-normalizer @ file:///tmp/build/80754af9/charset-normalizer_1630003229654/work | |
| click==7.1.2 | |
| cloudpickle==2.1.0 | |
| colorama==0.4.5 | |
| colorful==0.5.4 | |
| commonmark==0.9.1 | |
| conda==4.13.0 | |
| conda-content-trust @ file:///tmp/build/80754af9/conda-content-trust_1617045594566/work | |
| conda-pack==0.6.0 | |
| conda-package-handling @ file:///tmp/build/80754af9/conda-package-handling_1649105789509/work | |
| contextlib2==21.6.0 | |
| cookies==2.2.1 | |
| cryptography @ file:///tmp/build/80754af9/cryptography_1652083456434/work | |
| cycler==0.11.0 | |
| Cython==0.29.26 | |
| debugpy==1.6.0 | |
| decorator==5.1.1 | |
| defusedxml==0.7.1 | |
| distlib==0.3.4 | |
| dm-tree==0.1.7 | |
| docker==5.0.3 | |
| docker-pycreds==0.4.0 | |
| docutils==0.16 | |
| entrypoints==0.4 | |
| et-xmlfile==1.1.0 | |
| execnet==1.9.0 | |
| expiringdict==1.2.2 | |
| fairscale==0.4.6 | |
| fastapi==0.76.0 | |
| fastjsonschema==2.15.3 | |
| feather-format==0.4.1 | |
| filelock==3.7.1 | |
| flake8==3.9.1 | |
| flake8-bugbear==21.9.2 | |
| flake8-comprehensions==3.10.0 | |
| flake8-quotes==2.0.0 | |
| Flask==2.0.3 | |
| flatbuffers==1.12 | |
| fonttools==4.33.3 | |
| freezegun==1.2.2 | |
| frozenlist==1.3.0 | |
| fsspec==2022.5.0 | |
| future==0.18.2 | |
| gast==0.4.0 | |
| gitdb==4.0.9 | |
| GitPython==3.1.27 | |
| google-api-core==2.8.1 | |
| google-api-python-client==1.7.8 | |
| google-auth==2.6.6 | |
| google-auth-httplib2==0.1.0 | |
| google-auth-oauthlib==0.4.6 | |
| google-cloud-core==2.3.1 | |
| google-cloud-storage==2.4.0 | |
| google-crc32c==1.3.0 | |
| google-oauth==1.0.1 | |
| google-pasta==0.2.0 | |
| google-resumable-media==2.3.3 | |
| googleapis-common-protos==1.56.2 | |
| gpustat==1.0.0b1 | |
| graphviz==0.20 | |
| greenlet==1.1.2 | |
| grpcio==1.43.0 | |
| gym==0.21.0 | |
| gym-minigrid==1.0.3 | |
| h11==0.13.0 | |
| h5py==3.7.0 | |
| halo==0.0.31 | |
| hiredis==2.0.0 | |
| httplib2==0.20.4 | |
| huggingface-hub==0.8.1 | |
| humanfriendly==9.2 | |
| idna==2.10 | |
| imageio==2.19.3 | |
| imagesize==1.3.0 | |
| importlib-metadata==4.11.4 | |
| importlib-resources==5.7.1 | |
| ipykernel==6.15.0 | |
| ipython==7.34.0 | |
| ipython-genutils==0.2.0 | |
| ipywidgets==7.7.1 | |
| iso8601==1.0.2 | |
| isodate==0.6.1 | |
| isort==5.10.1 | |
| itsdangerous==2.1.2 | |
| jedi==0.18.1 | |
| Jinja2==3.1.2 | |
| jmespath==0.10.0 | |
| joblib==1.1.0 | |
| jsondiff==1.1.1 | |
| jsonpatch==1.32 | |
| jsonpickle==2.2.0 | |
| jsonpointer==2.3 | |
| jsonschema==4.6.0 | |
| jupyter-cache==0.4.3 | |
| jupyter-client==7.3.4 | |
| jupyter-core==4.10.0 | |
| jupyter-server==1.18.0 | |
| jupyter-server-mathjax==0.2.5 | |
| jupyter-sphinx==0.3.2 | |
| jupyterlab-pygments==0.2.2 | |
| jupyterlab-widgets==1.1.1 | |
| jupytext==1.13.6 | |
| keras==2.9.0 | |
| Keras-Preprocessing==1.1.2 | |
| kiwisolver==1.4.2 | |
| knack==0.8.2 | |
| kopf==1.35.4 | |
| kubernetes==23.6.0 | |
| libclang==14.0.6 | |
| lightgbm==3.3.2 | |
| lightgbm-ray @ git+https://github.com/ray-project/lightgbm_ray@e3a35f7382b6a4f3526162e206e3759f7130d4f3 | |
| llvmlite==0.34.0 | |
| log-symbols==0.0.14 | |
| lxml==4.9.0 | |
| lz4==4.0.1 | |
| Markdown==3.4.1 | |
| markdown-it-py==1.1.0 | |
| MarkupSafe==2.1.1 | |
| matplotlib==3.5.2 | |
| matplotlib-inline==0.1.3 | |
| mccabe==0.6.1 | |
| mdit-py-plugins==0.2.8 | |
| memray==1.1.0 | |
| mistune==0.8.4 | |
| mock==4.0.3 | |
| more-itertools==8.13.0 | |
| moto==1.3.0 | |
| msal==1.18.0 | |
| msal-extensions==0.3.1 | |
| msgpack==1.0.3 | |
| msrest==0.6.21 | |
| msrestazure==0.6.4 | |
| multidict==6.0.2 | |
| mypy==0.782 | |
| mypy-extensions==0.4.3 | |
| myst-nb==0.13.1 | |
| myst-parser==0.15.2 | |
| nbclient==0.5.13 | |
| nbconvert==6.5.0 | |
| nbdime==3.1.1 | |
| nbformat==5.4.0 | |
| nest-asyncio==1.5.5 | |
| networkx==2.6.3 | |
| notebook==6.4.12 | |
| numba==0.51.2 | |
| numpy==1.21.6 | |
| nvidia-ml-py3==7.352.0 | |
| oauthlib==3.2.0 | |
| objgraph==3.5.0 | |
| opencensus==0.9.0 | |
| opencensus-context==0.1.2 | |
| openpyxl==3.0.10 | |
| opentelemetry-api==1.1.0 | |
| opentelemetry-exporter-otlp==1.1.0 | |
| opentelemetry-exporter-otlp-proto-grpc==1.1.0 | |
| opentelemetry-proto==1.1.0 | |
| opentelemetry-sdk==1.1.0 | |
| opentelemetry-semantic-conventions==0.20b0 | |
| opt-einsum==3.3.0 | |
| packaging==21.3 | |
| pandas==1.3.5 | |
| pandocfilters==1.5.0 | |
| paramiko==2.11.0 | |
| parso==0.8.3 | |
| path==16.4.0 | |
| path.py==12.5.0 | |
| pathspec==0.9.0 | |
| pathtools==0.1.2 | |
| pexpect==4.8.0 | |
| pickle5==0.0.11 | |
| pickleshare==0.7.5 | |
| Pillow==9.1.1 | |
| pip==21.2.2 | |
| pkginfo==1.8.2 | |
| platformdirs==2.5.2 | |
| pluggy==0.13.1 | |
| polars==0.13.50 | |
| portalocker==1.7.1 | |
| prometheus-client==0.13.1 | |
| promise==2.3 | |
| prompt-toolkit==3.0.29 | |
| protobuf==3.19.4 | |
| proxy.py==2.4.3 | |
| psutil==5.9.1 | |
| ptyprocess==0.7.0 | |
| py==1.11.0 | |
| py-spy==0.3.12 | |
| py4j==0.10.9 | |
| pyaml==21.10.1 | |
| pyarrow==6.0.1 | |
| pyasn1==0.4.8 | |
| pyasn1-modules==0.2.8 | |
| pycodestyle==2.7.0 | |
| pycosat==0.6.3 | |
| pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work | |
| pydantic==1.9.1 | |
| pydata-sphinx-theme==0.7.2 | |
| pyDeprecate==0.3.1 | |
| pyflakes==2.3.1 | |
| pygame==2.1.2 | |
| Pygments==2.11.2 | |
| PyJWT==2.4.0 | |
| PyNaCl==1.5.0 | |
| pyOpenSSL @ file:///opt/conda/conda-bld/pyopenssl_1643788558760/work | |
| pyparsing==3.0.9 | |
| pyrsistent==0.18.1 | |
| PySocks @ file:///tmp/build/80754af9/pysocks_1594394576006/work | |
| pyspark==3.1.2 | |
| pytest==5.4.3 | |
| pytest-asyncio==0.16.0 | |
| pytest-fixture-config==1.7.0 | |
| pytest-forked==1.4.0 | |
| pytest-lazy-fixture==0.6.3 | |
| pytest-rerunfailures==10.2 | |
| pytest-shutil==1.7.0 | |
| pytest-sugar==0.9.4 | |
| pytest-timeout==2.1.0 | |
| pytest-virtualenv==1.7.0 | |
| pytest_docker_tools==0.2.3 | |
| python-dateutil==2.8.2 | |
| python-json-logger==2.0.2 | |
| pytorch-lightning==1.5.10 | |
| pytz==2022.1 | |
| PyWavelets==1.3.0 | |
| PyYAML==6.0 | |
| pyzmq==23.2.0 | |
| -e git+ssh://git@github.com/ray-project/ray.git@be4e7a7d898ebb60cdd993c221e5fdf7a8ce3bbd#egg=ray&subdirectory=python | |
| ray-cpp==1.13.0 | |
| ray-lightning @ git+https://github.com/ray-project/ray_lightning@6aed848f757a03c03166c1a9bddfeea5153e7b90 | |
| -e git+ssh://git@github.com/ray-project/ray.git@be4e7a7d898ebb60cdd993c221e5fdf7a8ce3bbd#egg=ray_release&subdirectory=release | |
| redis==3.5.3 | |
| regex==2022.7.25 | |
| requests==2.25.1 | |
| requests-oauthlib==1.3.1 | |
| rich==12.4.4 | |
| rsa==4.8 | |
| ruamel-yaml-conda @ file:///tmp/build/80754af9/ruamel_yaml_1616016701961/work | |
| s3transfer==0.3.7 | |
| scikit-image==0.19.2 | |
| scikit-learn==0.24.2 | |
| scikit-optimize==0.9.0 | |
| scipy==1.7.3 | |
| Send2Trash==1.8.0 | |
| sentry-sdk==1.9.4 | |
| setproctitle==1.2.2 | |
| setuptools==59.5.0 | |
| shortuuid==1.0.9 | |
| six==1.13.0 | |
| smart-open==6.0.0 | |
| smmap==5.0.0 | |
| sniffio==1.2.0 | |
| snowballstemmer==2.2.0 | |
| soupsieve==2.3.2.post1 | |
| Sphinx==4.3.2 | |
| sphinx-book-theme==0.1.7 | |
| sphinx-click==3.0.2 | |
| sphinx-copybutton==0.4.0 | |
| sphinx-external-toc==0.2.3 | |
| sphinx-jsonschema==1.17.2 | |
| sphinx-panels==0.6.0 | |
| sphinx-sitemap==2.2.0 | |
| sphinx-thebe==0.1.1 | |
| sphinx-togglebutton==0.2.3 | |
| sphinx-version-warning==1.1.2 | |
| sphinxcontrib-applehelp==1.0.2 | |
| sphinxcontrib-devhelp==1.0.2 | |
| sphinxcontrib-htmlhelp==2.0.0 | |
| sphinxcontrib-jsmath==1.0.1 | |
| sphinxcontrib-qthelp==1.0.3 | |
| sphinxcontrib-serializinghtml==1.1.5 | |
| sphinxcontrib.yt==0.2.2 | |
| sphinxemoji==0.2.0 | |
| spinners==0.0.24 | |
| SQLAlchemy==1.4.39 | |
| starlette==0.18.0 | |
| tabulate==0.8.9 | |
| tensorboard==2.9.1 | |
| tensorboard-data-server==0.6.1 | |
| tensorboard-plugin-wit==1.8.1 | |
| tensorboardX==2.5 | |
| tensorflow==2.9.1 | |
| tensorflow-estimator==2.9.0 | |
| tensorflow-io-gcs-filesystem==0.26.0 | |
| termcolor==1.1.0 | |
| terminado==0.15.0 | |
| testfixtures==6.18.5 | |
| threadpoolctl==3.1.0 | |
| tifffile==2021.11.2 | |
| tinycss2==1.1.1 | |
| tokenizers==0.12.1 | |
| toml==0.10.2 | |
| tomli==1.2.3 | |
| torch==1.12.1 | |
| torchmetrics==0.9.3 | |
| tornado==6.1 | |
| tqdm @ file:///opt/conda/conda-bld/tqdm_1647339053476/work | |
| traitlets==5.3.0 | |
| transformers==4.21.1 | |
| tune-sklearn @ git+https://github.com/ray-project/tune-sklearn@67af5a931b9ed9cc7d75fa892d7146a40e569c1c | |
| typed-ast==1.4.3 | |
| typing_extensions==4.2.0 | |
| uritemplate==3.0.1 | |
| urllib3==1.26.11 | |
| uvicorn==0.16.0 | |
| virtualenv==20.14.1 | |
| wandb==0.13.1 | |
| wcwidth==0.2.5 | |
| webencodings==0.5.1 | |
| websocket-client==1.3.2 | |
| Werkzeug==2.1.2 | |
| wheel==0.37.1 | |
| widgetsnbextension==3.6.1 | |
| wrapt==1.14.1 | |
| xgboost==1.6.1 | |
| xgboost-ray @ git+https://github.com/ray-project/xgboost_ray@08f3bc1fba96d98b5571e5e746b3b87b55e31d16 | |
| xlrd==2.0.1 | |
| xmltodict==0.13.0 | |
| yarl==1.7.2 | |
| zipp==3.8.0 | |
| [INFO 2022-08-15 16:20:43,469] minimal.py: 191 Tests uses compute template with name oss-release-tests_mWEC__compute__pipelined_data_ingest_benchmark__d052e40b4a27396f394add1ce0f63ea8d9965f19ce3b4e001f9cb2240f08b25c. Looking up existing cluster computes. | |
| [INFO 2022-08-15 16:20:43,648] minimal.py: 212 Cluster compute already exists with ID cpt_X3z4SS2vjMi9cG9zYuYhmwUg | |
| [INFO 2022-08-15 16:20:43,648] minimal.py: 29 Test uses a cluster env with name oss-release-tests_mWEC__env__pipelined_data_ingest_benchmark__6b2fa44394b437e8c8263b16a4f7df6f3376f505e93db8e5767a964400927ddf. Looking up existing cluster envs with this name. | |
| [INFO 2022-08-15 16:20:43,748] minimal.py: 59 Cluster env not found. Creating new one. | |
| [INFO 2022-08-15 16:20:44,494] minimal.py: 81 Cluster env created with ID apt_YeiejycgrEtNQFi15YNURXPB | |
| [INFO 2022-08-15 16:20:44,613] minimal.py: 132 Waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish... | |
| [INFO 2022-08-15 16:20:44,613] minimal.py: 134 Track progress here: https://console.anyscale-staging.com/o/anyscale-internal/configurations/app-config-details/bld_9LsGK6whMKM8QXhPN1ZNP13E | |
| [INFO 2022-08-15 16:21:15,247] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (30 seconds) ... | |
| [INFO 2022-08-15 16:21:44,657] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (60 seconds) ... | |
| [INFO 2022-08-15 16:22:15,341] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (90 seconds) ... | |
| [INFO 2022-08-15 16:22:45,705] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (121 seconds) ... | |
| [INFO 2022-08-15 16:23:14,675] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (150 seconds) ... | |
| [INFO 2022-08-15 16:23:45,152] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (180 seconds) ... | |
| [INFO 2022-08-15 16:24:14,656] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (210 seconds) ... | |
| [INFO 2022-08-15 16:24:44,711] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (240 seconds) ... | |
| [INFO 2022-08-15 16:25:15,666] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (271 seconds) ... | |
| [INFO 2022-08-15 16:25:44,851] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (300 seconds) ... | |
| [INFO 2022-08-15 16:26:15,340] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (330 seconds) ... | |
| [INFO 2022-08-15 16:26:45,689] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (361 seconds) ... | |
| [INFO 2022-08-15 16:27:15,614] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (391 seconds) ... | |
| [INFO 2022-08-15 16:27:45,384] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (420 seconds) ... | |
| [INFO 2022-08-15 16:28:16,481] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (451 seconds) ... | |
| [INFO 2022-08-15 16:28:45,475] minimal.py: 141 ... still waiting for build bld_9LsGK6whMKM8QXhPN1ZNP13E to finish (480 seconds) ... | |
| [INFO 2022-08-15 16:29:14,081] minimal.py: 157 Build succeeded. | |
| [INFO 2022-08-15 16:29:14,081] full.py: 28 Creating cluster pipelined_data_ingest_benchmark_1660605637 | |
| [INFO 2022-08-15 16:29:14,081] full.py: 29 Autosuspend time: 25 minutes | |
| [INFO 2022-08-15 16:29:14,656] full.py: 45 Starting cluster pipelined_data_ingest_benchmark_1660605637 (ses_G2hdYH7zFcgWuvFrgzXj7rf9) | |
| [INFO 2022-08-15 16:29:14,656] full.py: 49 Link to cluster: https://console.anyscale-staging.com/o/anyscale-internal/projects/prj_mWECugke9RzMh79BZQqeykjN/clusters/ses_G2hdYH7zFcgWuvFrgzXj7rf9 | |
| [INFO 2022-08-15 16:29:15,005] full.py: 63 Waiting for cluster pipelined_data_ingest_benchmark_1660605637... | |
| [INFO 2022-08-15 16:29:45,174] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (30 seconds) ... | |
| [INFO 2022-08-15 16:30:15,431] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (60 seconds) ... | |
| [INFO 2022-08-15 16:30:45,784] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (90 seconds) ... | |
| [INFO 2022-08-15 16:31:15,799] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (120 seconds) ... | |
| [INFO 2022-08-15 16:31:45,174] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (150 seconds) ... | |
| [INFO 2022-08-15 16:32:15,067] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (180 seconds) ... | |
| [INFO 2022-08-15 16:32:45,455] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (210 seconds) ... | |
| [INFO 2022-08-15 16:33:15,149] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (240 seconds) ... | |
| [INFO 2022-08-15 16:33:44,847] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (270 seconds) ... | |
| [INFO 2022-08-15 16:34:15,628] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (300 seconds) ... | |
| [INFO 2022-08-15 16:34:45,716] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (331 seconds) ... | |
| [INFO 2022-08-15 16:35:15,484] full.py: 77 ... still waiting for cluster pipelined_data_ingest_benchmark_1660605637 (360 seconds) ... | |
| [INFO 2022-08-15 16:35:23,898] session_controller.py: 29 Uploading <cwd> to <cwd> using SessionController | |
| [WARNING 2022-08-15 16:35:26,085] util.py: 235 Dropping the empty legacy field head_node. head_nodeis not supported for ray>=2.0.0. It is recommended to removehead_node from the cluster config. | |
| [WARNING 2022-08-15 16:35:26,085] util.py: 235 Dropping the empty legacy field worker_nodes. worker_nodesis not supported for ray>=2.0.0. It is recommended to removeworker_nodes from the cluster config. | |
| 2022-08-15 16:35:26,092 INFO commands.py:349 -- Checking External environment settings | |
| 2022-08-15 16:35:26,092 VINFO utils.py:150 -- Creating AWS resource `ec2` in `us-west-2` | |
| 2022-08-15 16:35:29,525 VINFO utils.py:150 -- Creating AWS resource `ec2` in `us-west-2` | |
| 2022-08-15 16:35:30,354 INFO updater.py:239 -- [1/2] Processing file mounts | |
| 2022-08-15 16:35:30,464 INFO command_runner.py:393 -- Fetched IP: 35.91.112.85 | |
| 2022-08-15 16:35:30,465 INFO log_timer.py:30 -- NodeUpdater: ins_PREAPr34Wye3zPGf3qZdKPPu: Got IP [LogTimer=110ms] | |
| 2022-08-15 16:35:30,465 VINFO command_runner.py:554 -- Running `mkdir -p /tmp/ray_tmp_mount/anyscale-dev-stag-ed560d261d9a6181/home/ray && chown -R ubuntu /tmp/ray_tmp_mount/anyscale-dev-stag-ed560d261d9a6181/home/ray` | |
| 2022-08-15 16:35:30,465 VVINFO command_runner.py:557 -- Full command is `ssh -tt -i /home/ray/.ssh/anyscale/prj_mWECugke9RzMh79BZQqeykjN/ses_G2hdYH7zFcgWuvFrgzXj7rf9.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_070dd72385/26a4b1b06f/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@35.91.112.85 bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (mkdir -p /tmp/ray_tmp_mount/anyscale-dev-stag-ed560d261d9a6181/home/ray && chown -R ubuntu /tmp/ray_tmp_mount/anyscale-dev-stag-ed560d261d9a6181/home/ray)'` | |
| Warning: Permanently added '35.91.112.85' (ECDSA) to the list of known hosts. | |
| Shared connection to 35.91.112.85 closed. | |
| 2022-08-15 16:35:31,247 VINFO command_runner.py:597 -- Running `rsync --rsh ssh -i /home/ray/.ssh/anyscale/prj_mWECugke9RzMh79BZQqeykjN/ses_G2hdYH7zFcgWuvFrgzXj7rf9.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_070dd72385/26a4b1b06f/%C -o ControlPersist=10s -o ConnectTimeout=120s -avz --exclude **/.git --exclude **/.git/** --filter dir-merge,- .gitignore /home/ray/github/ray/release/nightly_tests/dataset/ ubuntu@35.91.112.85:/tmp/ray_tmp_mount/anyscale-dev-stag-ed560d261d9a6181/home/ray/oss-release-tests/` | |
| sending incremental file list | |
| ./ | |
| .anyscale.yaml | |
| app_config.yaml | |
| data_ingest_benchmark.py | |
| data_ingest_benchmark_compute.yaml | |
| dataset_ingest_400G_compute.yaml | |
| dataset_random_access.py | |
| dataset_shuffle_data_loader.py | |
| inference.py | |
| inference.yaml | |
| parquet_metadata_resolution.py | |
| pipelined_ingestion_app.yaml | |
| pipelined_ingestion_compute.yaml | |
| pipelined_training.py | |
| pipelined_training_app.yaml | |
| pipelined_training_compute.yaml | |
| ray_sgd_runner.py | |
| ray_sgd_training.py | |
| ray_sgd_training_app.yaml | |
| ray_sgd_training_compute.yaml | |
| ray_sgd_training_compute_no_gpu.yaml | |
| ray_sgd_training_smoke_compute.yaml | |
| shuffle_app_config.yaml | |
| shuffle_compute.yaml | |
| sort.py | |
| wait_cluster.py | |
| sent 27,318 bytes received 494 bytes 55,624.00 bytes/sec | |
| total size is 75,052 speedup is 2.70 | |
| 2022-08-15 16:35:31,517 VINFO command_runner.py:554 -- Running `docker inspect -f '{{.State.Running}}' ray_container || true` | |
| 2022-08-15 16:35:31,518 VVINFO command_runner.py:557 -- Full command is `ssh -tt -i /home/ray/.ssh/anyscale/prj_mWECugke9RzMh79BZQqeykjN/ses_G2hdYH7zFcgWuvFrgzXj7rf9.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_070dd72385/26a4b1b06f/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@35.91.112.85 bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (docker inspect -f '"'"'{{.State.Running}}'"'"' ray_container || true)'` | |
| Shared connection to 35.91.112.85 closed. | |
| 2022-08-15 16:35:31,682 VINFO updater.py:537 -- `rsync`ed /home/ray/github/ray/release/nightly_tests/dataset/ (local) to /home/ray/oss-release-tests/ (remote) | |
| 2022-08-15 16:35:31,682 INFO updater.py:234 -- /home/ray/oss-release-tests/ from /home/ray/github/ray/release/nightly_tests/dataset/ | |
| 2022-08-15 16:35:31,683 INFO log_timer.py:30 -- NodeUpdater: ins_PREAPr34Wye3zPGf3qZdKPPu: Synced /home/ray/github/ray/release/nightly_tests/dataset/ to /home/ray/oss-release-tests/ [LogTimer=1328ms] | |
| 2022-08-15 16:35:31,683 INFO updater.py:257 -- [2/2] No worker file mounts to sync | |
| (anyscale +14m54.3s) Pushed to session pipelined_data_ingest_benchmark_1660605637. View at https://console.anyscale-staging.com/projects/prj_mWECugke9RzMh79BZQqeykjN/clusters/ses_G2hdYH7zFcgWuvFrgzXj7rf9 | |
| [INFO 2022-08-15 16:35:31,685] sdk_runner.py: 90 Running command in cluster pipelined_data_ingest_benchmark_1660605637: TEST_OUTPUT_JSON=/tmp/release_test_out.json python wait_cluster.py 20 3000 | |
| [INFO 2022-08-15 16:35:31,685] sdk_runner.py: 95 Link to cluster: https://console.anyscale-staging.com/o/anyscale-internal/projects/prj_mWECugke9RzMh79BZQqeykjN/clusters/ses_G2hdYH7zFcgWuvFrgzXj7rf9 | |
| [INFO 2022-08-15 16:36:01,906] sdk_runner.py: 121 ... command still running ...(30 seconds) ... | |
| [INFO 2022-08-15 16:36:31,788] sdk_runner.py: 121 ... command still running ...(60 seconds) ... | |
| [INFO 2022-08-15 16:37:02,755] sdk_runner.py: 121 ... command still running ...(91 seconds) ... | |
| [INFO 2022-08-15 16:37:32,276] sdk_runner.py: 121 ... command still running ...(120 seconds) ... | |
| [INFO 2022-08-15 16:38:01,773] sdk_runner.py: 121 ... command still running ...(150 seconds) ... | |
| [INFO 2022-08-15 16:38:32,139] sdk_runner.py: 121 ... command still running ...(180 seconds) ... | |
| [INFO 2022-08-15 16:39:01,865] sdk_runner.py: 121 ... command still running ...(210 seconds) ... | |
| [INFO 2022-08-15 16:39:32,186] sdk_runner.py: 121 ... command still running ...(240 seconds) ... | |
| [INFO 2022-08-15 16:40:02,169] sdk_runner.py: 121 ... command still running ...(270 seconds) ... | |
| [INFO 2022-08-15 16:40:31,806] sdk_runner.py: 121 ... command still running ...(300 seconds) ... | |
| [INFO 2022-08-15 16:41:02,234] sdk_runner.py: 121 ... command still running ...(330 seconds) ... | |
| [INFO 2022-08-15 16:41:04,438] sdk_runner.py: 90 Running command in cluster pipelined_data_ingest_benchmark_1660605637: TEST_OUTPUT_JSON=/tmp/release_test_out.json python data_ingest_benchmark.py --dataset-size-gb=200 --num-workers=20 --streaming | |
| [INFO 2022-08-15 16:41:04,439] sdk_runner.py: 95 Link to cluster: https://console.anyscale-staging.com/o/anyscale-internal/projects/prj_mWECugke9RzMh79BZQqeykjN/clusters/ses_G2hdYH7zFcgWuvFrgzXj7rf9 | |
| [INFO 2022-08-15 16:41:35,218] sdk_runner.py: 121 ... command still running ...(30 seconds) ... | |
| [INFO 2022-08-15 16:42:05,614] sdk_runner.py: 121 ... command still running ...(61 seconds) ... | |
| [INFO 2022-08-15 16:42:34,923] sdk_runner.py: 121 ... command still running ...(90 seconds) ... | |
| [INFO 2022-08-15 16:43:04,524] sdk_runner.py: 121 ... command still running ...(120 seconds) ... | |
| [INFO 2022-08-15 16:43:35,346] sdk_runner.py: 121 ... command still running ...(150 seconds) ... | |
| [INFO 2022-08-15 16:44:04,892] sdk_runner.py: 121 ... command still running ...(180 seconds) ... | |
| [INFO 2022-08-15 16:44:34,957] sdk_runner.py: 121 ... command still running ...(210 seconds) ... | |
| [INFO 2022-08-15 16:45:04,901] sdk_runner.py: 121 ... command still running ...(240 seconds) ... | |
| [INFO 2022-08-15 16:45:34,458] sdk_runner.py: 121 ... command still running ...(270 seconds) ... | |
| [INFO 2022-08-15 16:46:04,361] sdk_runner.py: 121 ... command still running ...(300 seconds) ... | |
| [ERROR 2022-08-15 16:46:06,606] glue.py: 283 Command returned non-success status: 1 | |
| Traceback (most recent call last): | |
| File "/home/ray/github/ray/release/ray_release/glue.py", line 254, in run_release_test | |
| command, env=command_env, timeout=command_timeout | |
| File "/home/ray/github/ray/release/ray_release/command_runner/sdk_runner.py", line 141, in run_command | |
| raise CommandError(f"Command returned non-success status: {status_code}") | |
| ray_release.exception.CommandError: Command returned non-success status: 1 | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/home/ray/github/ray/release/ray_release/glue.py", line 257, in run_release_test | |
| raise TestCommandError(e) | |
| ray_release.exception.TestCommandError: Command returned non-success status: 1 | |
| [INFO 2022-08-15 16:46:07,717] log.py: 11 Test pipelined_data_ingest_benchmark finished after 0.00 seconds. Last logs: | |
| File "data_ingest_benchmark.py", line 22, in consume | |
| File "data_ingest_benchmark.py", line 47, in DoConsume | |
| File "/home/ray/anaconda3/lib/python3.7/site-packages/ray/data/dataset_pipeline.py", line 199, in iter_batches | |
| shuffle_seed=local_shuffle_seed, | |
| File "/home/ray/anaconda3/lib/python3.7/site-packages/ray/data/_internal/block_batching.py", line 129, in batch_blocks | |
| yield from get_batches(block_window[0]) | |
| File "/home/ray/anaconda3/lib/python3.7/site-packages/ray/data/_internal/block_batching.py", line 87, in get_batches | |
| block = ray.get(block) | |
| ray.exceptions.OutOfDiskError: ('3f9ebfdbb6064d2effffffffffffffffffffffff0200000001000000', b'\n\x1c\\Q\xb7+\x13G\xc3\xd0l\x01vW\xe5b&]\xc9)G7\x81\xfb`\x1d\xb1XN\xcf\x12\x0b172.31.72.1\x18\x97N"\x1c\x1b\x06\x8d\xfb\x90\t\xa9\xa5\x0c\xde\xc1\x8d\xd5\xa9\xca\x8fro[\xace&\x1d\xef\xaau\xe1>', '') | |
| The object cannot be created because the local object store is full and the local disk's utilization is over capacity (95% by default).Tip: Use `df` on this node to check disk usage and `ray memory` to check object store memory usage. | |
| [INFO 2022-08-15 16:46:07,718] log.py: 17 Got the following metadata: | |
| name: pipelined_data_ingest_benchmark | |
| status: error | |
| runtime: 0.00 | |
| stable: True | |
| buildkite_url: | |
| wheels_url: https://s3-us-west-2.amazonaws.com/ray-wheels/master/a3236b62254348ed1c09062d57be7a2fd8e6c4af/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl | |
| cluster_url: https://console.anyscale-staging.com/o/anyscale-internal/projects/prj_mWECugke9RzMh79BZQqeykjN/clusters/ses_G2hdYH7zFcgWuvFrgzXj7rf9 | |
| [INFO 2022-08-15 16:46:07,718] log.py: 36 Did not find any results. | |
| [ERROR 2022-08-15 16:46:07,718] run_release_test.py: 159 Command returned non-success status: 1 | |
| Traceback (most recent call last): | |
| File "/home/ray/github/ray/release/ray_release/glue.py", line 254, in run_release_test | |
| command, env=command_env, timeout=command_timeout | |
| File "/home/ray/github/ray/release/ray_release/command_runner/sdk_runner.py", line 141, in run_command | |
| raise CommandError(f"Command returned non-success status: {status_code}") | |
| ray_release.exception.CommandError: Command returned non-success status: 1 | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "release/ray_release/scripts/run_release_test.py", line 155, in main | |
| no_terminate=no_terminate, | |
| File "/home/ray/github/ray/release/ray_release/glue.py", line 333, in run_release_test | |
| raise pipeline_exception | |
| File "/home/ray/github/ray/release/ray_release/glue.py", line 257, in run_release_test | |
| raise TestCommandError(e) | |
| ray_release.exception.TestCommandError: Command returned non-success status: 1 | |
| [INFO 2022-08-15 16:46:07,718] run_release_test.py: 163 Release test pipeline for test pipelined_data_ingest_benchmark completed. Returning with exit code = 40 | |
| (base) ray@90b689302baa:~/github/ray$ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment