prefetch.yaml

cluster_name: prefetch

provider:
    type: aws
    region: us-west-1
    cache_stopped_nodes: false

auth:
    ssh_user: ubuntu

max_workers: 5

available_node_types:
    1_gpu_node:
        node_config:
            InstanceType: m5.4xlarge
            IamInstanceProfile:
                Arn: arn:aws:iam::959243851260:instance-profile/ray-autoscaler-v1
            BlockDeviceMappings:
                - DeviceName: /dev/sda1
                  Ebs:
                      VolumeSize: 500
        resources:
          CPU: 0
          GPU: 4
        min_workers: 1
        max_workers: 1
    memory_node:
        node_config:
            InstanceType: m5.4xlarge
            IamInstanceProfile:
                Arn: arn:aws:iam::959243851260:instance-profile/ray-autoscaler-v1
            BlockDeviceMappings:
                - DeviceName: /dev/sda1
                  Ebs:
                      VolumeSize: 500
        resources: {}
        min_workers: 4
        max_workers: 4

head_node_type: memory_node

setup_commands:
    - pip install llvmlite --ignore-installed
    - pip install "ray[tune,data,default] @ https://s3-us-west-2.amazonaws.com/ray-wheels/ubranch-1.11.0/56aa48eff7cd780637ac96978b49e527885fd92e/ray-2.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl" jupyter torch awscli pyarrow tensorboard boto3 pickle5 mlflow torchvision
    - pip uninstall aiohttp -y
    - pip install aiohttp==3.7
    - pip install pyarrow==6.0.1
    - mkdir -p data inference models runs