Skip to content

Instantly share code, notes, and snippets.

@cwpearson
Last active April 26, 2021 21:35
Show Gist options
  • Save cwpearson/f0dae0615b846085cf6d3afe8eed23d7 to your computer and use it in GitHub Desktop.
Save cwpearson/f0dae0615b846085cf6d3afe8eed23d7 to your computer and use it in GitHub Desktop.
Bash script to download suitesparse matrices if they're missing
#! /bin/bash
set -eou pipefail
match_size_or_get() {
url=$1
echo "working on $url"
# get expected size of url content.
# first, follow redirects to get effective url
# then, ask how big the content at the effective URL is
# remove the carriage return from the header
urleff=$(curl -sLI -o /dev/null -w %{url_effective} $url)
esz=$(curl -sI $urleff | grep -i Content-Length: | cut -d' ' -f2)
esz=`echo -n $esz | tr -d '\r'` # get rid of carriage return
# check size of file (0 if absent)
name=`basename $url`
if [ -f $name ]; then
sz=$(wc -c < $name | tr -d '[:space:]\n')
else
sz=0
fi
# convert to number
sz=$(( $sz ))
esz=$(( $esz ))
# redownload if local file does not match expected size
if [ $sz != $esz ]; then
echo $name "was" $sz "(expected $esz)"
curl -sSLO $url
else
echo $name "was expected size" $esz
fi
set -x
# get expected extracted size. assuming a single file in the tar
# head will read the first line and quit, causing exit 141 when the tar
# continues to try to write into it
esz=$(tar tzvf $name | head -n 1 | tr -s ' ' | cut -d' ' -f5 || if [[ $? -eq 141 ]]; then true; else exit $?; fi )
extname=$(basename -s .tar.gz $name)
extname=$extname.mtx
# get size of extracted file, if present
if [ -f $extname ]; then
sz=$(wc -c < $extname | tr -d '[:space:]\n')
else
sz=0
fi
# convert to number
sz=$(( $sz ))
esz=$(( $esz ))
# re-extract file if size is not expected
if [ $sz != $esz ]; then
echo $extname "was" $sz "(expected $esz)"
tar --strip-components 1 -xvf $name
else
echo $extname "was expected size" $esz
fi
}
match_size_or_get https://suitesparse-collection-website.herokuapp.com/MM/AG-Monien/wave.tar.gz
# match_size_or_get https://suitesparse-collection-website.herokuapp.com/MM/SNAP/roadNet-CA.tar.gz
# match_size_or_get https://suitesparse-collection-website.herokuapp.com/MM/SNAP/roadNet-PA.tar.gz
# match_size_or_get https://suitesparse-collection-website.herokuapp.com/MM/SNAP/roadNet-TX.tar.gz
# match_size_or_get https://suitesparse-collection-website.herokuapp.com/MM/DIMACS10/delaunay_n18.tar.gz
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment