Last active
April 26, 2021 21:35
-
-
Save cwpearson/f0dae0615b846085cf6d3afe8eed23d7 to your computer and use it in GitHub Desktop.
Bash script to download suitesparse matrices if they're missing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /bin/bash | |
set -eou pipefail | |
match_size_or_get() { | |
url=$1 | |
echo "working on $url" | |
# get expected size of url content. | |
# first, follow redirects to get effective url | |
# then, ask how big the content at the effective URL is | |
# remove the carriage return from the header | |
urleff=$(curl -sLI -o /dev/null -w %{url_effective} $url) | |
esz=$(curl -sI $urleff | grep -i Content-Length: | cut -d' ' -f2) | |
esz=`echo -n $esz | tr -d '\r'` # get rid of carriage return | |
# check size of file (0 if absent) | |
name=`basename $url` | |
if [ -f $name ]; then | |
sz=$(wc -c < $name | tr -d '[:space:]\n') | |
else | |
sz=0 | |
fi | |
# convert to number | |
sz=$(( $sz )) | |
esz=$(( $esz )) | |
# redownload if local file does not match expected size | |
if [ $sz != $esz ]; then | |
echo $name "was" $sz "(expected $esz)" | |
curl -sSLO $url | |
else | |
echo $name "was expected size" $esz | |
fi | |
set -x | |
# get expected extracted size. assuming a single file in the tar | |
# head will read the first line and quit, causing exit 141 when the tar | |
# continues to try to write into it | |
esz=$(tar tzvf $name | head -n 1 | tr -s ' ' | cut -d' ' -f5 || if [[ $? -eq 141 ]]; then true; else exit $?; fi ) | |
extname=$(basename -s .tar.gz $name) | |
extname=$extname.mtx | |
# get size of extracted file, if present | |
if [ -f $extname ]; then | |
sz=$(wc -c < $extname | tr -d '[:space:]\n') | |
else | |
sz=0 | |
fi | |
# convert to number | |
sz=$(( $sz )) | |
esz=$(( $esz )) | |
# re-extract file if size is not expected | |
if [ $sz != $esz ]; then | |
echo $extname "was" $sz "(expected $esz)" | |
tar --strip-components 1 -xvf $name | |
else | |
echo $extname "was expected size" $esz | |
fi | |
} | |
match_size_or_get https://suitesparse-collection-website.herokuapp.com/MM/AG-Monien/wave.tar.gz | |
# match_size_or_get https://suitesparse-collection-website.herokuapp.com/MM/SNAP/roadNet-CA.tar.gz | |
# match_size_or_get https://suitesparse-collection-website.herokuapp.com/MM/SNAP/roadNet-PA.tar.gz | |
# match_size_or_get https://suitesparse-collection-website.herokuapp.com/MM/SNAP/roadNet-TX.tar.gz | |
# match_size_or_get https://suitesparse-collection-website.herokuapp.com/MM/DIMACS10/delaunay_n18.tar.gz |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment