bqwiki notes

gistfile1.txt

startup.sh
==========
set -e
set -x

DUMPFILE=$(curl http://metadata/computeMetadata/v1/instance/attributes/dumpfile -H "Metadata-Flavor: Google")
echo dumpfile $DUMPFILE

VMID=$(curl -s http://metadata/computeMetadata/v1/instance/hostname -H "Metadata-Flavor: Google" 2>&1 | cut -d. -f1)
echo vmid $VMID

vol=$(ls -l /dev/disk/by-id/google-biggerdisk1 | cut -d/ -f 7)
mkdir -p /mnt/bigdisk
sudo mount -a /dev/$vol /mnt/bigdisk
go run bqwiki/bqwiki.go -f /mnt/bigdisk/$DUMPFILE
gsutil cp *.csv gs://bq-wikipedia/
gcutil deleteinstance -f --delete_boot_pd $VMID










robot army general
==================

for ((i=1;i<=$num;i++)); do
  DUMPFILE=$(sed -n "${VMID}p" files)
  echo instance-$i processing $DUMPFILE

  # Create a boot disk for the new instance
  gcutil adddisk disk-$i --project=bq-wikipedia \
    --zone=us-central1-a --source_snapshot=workersnap

  # Create the instance with the startup script to process a file
  gcutil addinstance instance-$i --project=bq-wikipedia \
    --zone=us-central1-a --machine_type=n1-standard-1 \
    --network=default --external_ip_address=ephemeral \
    --service_account_scopes=compute-rw,storage-rw \
    --disk=disk-$i,deviceName=disk-$i,mode=READ_WRITE,boot \
    --disk=biggerdisk1,deviceName=bigdisk,mode=READ_ONLY \
    --metadata=startup-script-url:gs://bq-wikipedia/startup.sh
    --metadata=dumpfile:$DUMPFILE
done