clarng

memory at iter 5 410.6875 218.6171875
memory at iter 6 410.6875 218.66796875
memory at iter 7 410.6875 218.68359375
memory at iter 8 410.6875 218.70703125
memory at iter 9 410.88671875 218.73828125
memory at iter 10 410.88671875 218.7734375
memory at iter 11 410.88671875 218.77734375
memory at iter 12 410.9375 218.796875
memory at iter 13 410.9375 218.81640625
memory at iter 14 410.9375 218.8359375

clarng

(Actor pid=68081) memory @132300 rss 99.22265625 uss 41.51171875
memory @132400 rss 71.08203125 uss 12.953125
(Actor pid=68081) memory @132400 rss 100.12109375 uss 42.3515625
memory @132500 rss 71.4296875 uss 13.1875
(Actor pid=68081) memory @132500 rss 99.375 uss 41.78125
memory @132600 rss 70.76171875 uss 13.09375
(Actor pid=68081) memory @132600 rss 99.21484375 uss 41.6328125
memory @132700 rss 71.2578125 uss 13.3046875
(Actor pid=68081) memory @132700 rss 100.875 uss 43.15234375
memory @132800 rss 71.40234375 uss 13.484375

clarng

from collections import defaultdict
import os
import logging
import tracemalloc


import psutil
import ray
import numpy as np
import matplotlib.pyplot as plt

clarng

import ray

ray.init('auto')

ray.remote(num_cpus=0.5)
def eat_memory():
    some_str = ' ' * 512000000
    print(some_str)
    some_str = some_str+' ' * 512000000
    print(some_str)

clarng

opencensus-context==0.1.2
openpyxl==3.0.10
opentelemetry-api==1.1.0
opentelemetry-exporter-otlp==1.1.0
opentelemetry-exporter-otlp-proto-grpc==1.1.0
opentelemetry-proto==1.1.0
opentelemetry-sdk==1.1.0
opentelemetry-semantic-conventions==0.20b0
opt-einsum==3.3.0
packaging==21.3

clarng

kopf==1.35.4
kubernetes==23.6.0
libclang==14.0.6
lightgbm==3.3.2
lightgbm-ray @ git+https://github.com/ray-project/lightgbm_ray@e3a35f7382b6a4f3526162e206e3759f7130d4f3
llvmlite==0.34.0
log-symbols==0.0.14
lxml==4.9.0
lz4==4.0.1
Markdown==3.4.1

clarng

(base) ray@ip-172-31-89-194:~/workspace-project-rtoom_pipeline_data_ingest2/release/nightly_tests/dataset$ python data_ingest_benchmark.py --dataset-size-gb=200 --num-workers=20 --streaming

2022-08-16 07:55:17,950 INFO worker.py:1203 -- Using address localhost:9031 set in the environment variable RAY_ADDRESS


2022-08-16 07:55:20,407 INFO worker.py:1312 -- Connecting to existing Ray cluster at address: 172.31.89.194:9031...
2022-08-16 07:55:20,414 INFO worker.py:1487 -- Connected to Ray cluster. View the dashboard at https://session-aqwatdmgevphyalje148wqvj.i.anyscaleuserdata-staging.com/auth/?token=agh0_CkcwRQIgIk7mkj-gUcqPe1OsvZ_H8CPE_wsJBiwWJBszQZGOg8gCIQDWWn_c_wIHFEow6P51__K6H3uqXn3u3z5ucA_mHK8lJBJmEiD6fIFsE2xA7OC9UAjVkUdqDfyAqnjZHYJO1YMBnRNSfRgCIgNuL2E6DAii24bjBhCIutWeA0IMCKLY7pcGEIi61Z4D-gEeChxzZXNfQXF3YVREbWdFdnBoeUFMSkUxNDhXUXZK&redirect_to=dashboard.
2022-08-16 07:55:20,682 INFO packaging.py:342 -- Pushing file package 'gcs://_ray_pkg_b140455d05812ebf9ae368f7805fc29c.zip' (91.13MiB) to Ray cluster..

clarng

Stage 0: : 5it [00:07,  1.39s/it]1.87.52) 
Stage 0: : 5it [00:07,  1.40s/it]1.85.195) 
Stage 0: : 8it [00:15,  2.11s/it] pid=23663) 
Stage 0: : 6it [00:10,  2.01s/it]1.93.21) 3) 
Stage 0: : 6it [00:10,  2.02s/it]1.76.4) 
Stage 0: : 6it [00:10,  2.03s/it]1.90.220) 
Stage 0: : 6it [00:10,  2.02s/it]1.75.180) 
Stage 0: : 6it [00:10,  2.03s/it]1.79.152) 
Stage 0: : 6it [00:10,  2.03s/it]1.76.82) 
Stage 0: : 6it [00:10,  2.04s/it]1.92.6) 

clarng

python release/air_tests/air_benchmarks/workloads/data_benchmark.py --dataset-size-gb=100 --num-workers=10





diff --git a/release/air_tests/air_benchmarks/workloads/data_benchmark.py b/release/air_tests/air_benchmarks/workloads/data_benchmark.py
index 0f65a8ff83aee..01b20c031c2a0 100644
--- a/release/air_tests/air_benchmarks/workloads/data_benchmark.py
+++ b/release/air_tests/air_benchmarks/workloads/data_benchmark.py

clarng

from ray.experimental.packaging.load_package import load_package
import ray

mypkg = load_package('https://raw.githubusercontent.com/ray-project/ray/master/python/ray/experimental/packaging/example_pkg/ray_pkg.yaml')

print(mypkg._runtime_env)

print(ray.get(mypkg.my_func.remote()))