gistfile1.txt

# maybe change "fairseq-benchmark" below to your own tag
BENCHMARK=4node aws ec2 run-instances \
    --image-id ami-0707a23a1930bb11c \
    --count 4 \
    --instance-type p3dn.24xlarge \
    --key-name fair-$USER \
    --tag-specifications "ResourceType=instance,Tags=[{Key=fair-user,Value=$USER},{Key=fairseq-benchmark,Value=''}]" \
    --placement "GroupName = ${PLACEMENT_GROUP_NAME}" \
    --network-interfaces "DeviceIndex=0,InterfaceType=efa,Groups=${SECURITY_GROUP_ID},SubnetId=${SUBNET_ID}" \
    --block-device-mappings file://block_device_mappings.gp2.json
```

# running a job
```
## run job on 4 nodes
#
# usage: ./run_fairseq_benchmark_masked_lm.sh: MASTER_ADDR WORLD_SIZE HOST RANK
#
# - we use the first host as the "master"
# - RANK should be the starting worker rank (i.e., the node's rank multiplied by 8)
#
for I in $(seq 0 3); do
    HOST_I=$(cat hosts.txt | head -n $((I+1)) | tail -n 1);
    ./run_fairseq_benchmark_masked_lm.sh \
        $(cat hosts.txt | head -n 1) \
        32 \
        $HOST_I \
        $((I*8)) &
done