Download gaia data release 2

dl-gaia.sh

#! /bin/bash

index_url="http://cdn.gea.esac.esa.int/Gaia/gdr2/gaia_source/csv/"
index_html=`curl $index_url`

while read -r line; do
  # echo $line;
  match=`echo $line | grep -oE 'href\="(.*.csv.gz)'`;
  # echo $match;
  match=${match#href=\"} ;# remove prefix starting with 'href="'
  # echo $match;
  url="$index_url/$match";
  echo $url;
  ec=18; 
  while [ $ec -eq 18 ]; do
    curl -O -C - "$url";
    ec=$?;
  done

done <<< "$index_html"