远程下载zip文件。远程机器网速快但存储不够。适用于中国用户下载国外资源。

remote server

cd /path/to/dir && wget -w 30 -i url-list.txt


# run by crond, */5 * * * * bash /path/to/dir/test.sh > /dev/null

cd /path/to/dir
for i in `ls *.zip`; do
  unzip -t $i > /dev/null
  if [[ $? -eq 0 ]]; then
    mv $i download_done/
  fi
done

storage server

# run by crond, */5 * * * * bash /path/to/dir/download_and_test.sh > /dev/null
cd /path/to/dir
while(true)
do
  scp usr@remote_server:/path/to/dir/download_done/*.zip .
  for i in `ls *.zip`; do
    unzip -t $i >/dev/null
    if [[ $? -eq 0 ]]; then
      mv $i transfer_succ/
      ssh usr@remote_server "rm /path/to/dir/download_done/$i"
    else
      rm $i
    fi
  done
  sleep 30
done