download-images-from-block-site

clean.sh

#!/usr/bin/env bash
# identify and remove broke images
# with imageMagick

for i in $(ls -d train/cat/* train/fish/* val/cat/* val/fish/* test/cat/* test/fish/*); do
    identify $i > /dev/null 2>&1
    if [ $? -ne 0 ]; then
      echo "file $i is broken, removing it"
      rm $i
    fi
done

download.sh

#!/usr/bin/env bash
# https://github.com/Dreamacro/clash/issues/592
export ALL_PROXY=socks5://127.0.0.1:7891

#loon batch -f images.csv 'curl -k -o {2}/{1}/"$(basename {0})" -C - {0}'
loon batch -f images.csv 'out={2}/{1}/"$(basename {0})"; echo processing $out; if ! [ -f $out ]; then curl -k -o $out -C - {0}; fi'

images.csv