Download all of GreenGenes database from SecondGenome

scrape-greengenes.sh

#!/bin/bash

set -euo pipefail

function get_source() {
        curl --silent "http://greengenes.secondgenome.com/downloads/database/13_5"
}

function get_urls() {
        get_source | grep 'https:' | grep s3 | cut -d '"' -f 2
}

n_urls=$(get_urls | wc -l)

echo "--> fetching ${n_urls} files"

for n in $(seq 1 ${n_urls}); do
        url=$(get_urls | head -n ${n} | tail -n 1)
        echo="--> downloading ${n} ${url}"
        wget ${url}
done