GNU Parallel autoscaling workload example

run-batch-job.sh

#!/usr/bin/env bash

(
while ((1)); do
  gcloud --format=json compute instances list --filter="name~'nlp-workers-.*'" | \                         
    jq -rc '.[]|select(.status=="RUNNING").name' > workers.hosts.txt                                       
  sleep 30s
done
) & # (or, just run this watcher in a separate script)

# Set an autoscaling instance group using cheaper preemptible vms 
gcloud compute instance-groups managed resize nlp-workers --size=4
find /data/sentence-jobs/ -type f | parallel --eta --slf workers.hosts.txt --joblog job.log 'openie {} > {}.out'
gcloud compute instance-groups managed resize nlp-workers --size=0