Plowshare single thread implementation for downloading from one source file (file with links) Removes the link downloaded from the source file and keeps a backup of the source file.

download.sh

#!/bin/bash
#
#

PD_MAX_RETRIES="--max-retries=2"
PD_TIMEOUT="--timeout=60"
PD_OUTPUT="-o /media/disk1/plowshare/download"
PD_TEMP="--temp-rename --temp-directory /media/disk1/plowshare/temp"
PD_VERBOSE="-v 2"

# Main
test $# -ge 1 || {debug "Usage: $(basename $0) FILE_WITH_LINKS" exit 1}

INPUT_FILE=$1
DATE=`date +%s`
BACKUP_FILE="${INPUT_FILE}_${DATE}"
URL=`head -n 1 ${INPUT_FILE}`
shift
trap "kill 0" SIGINT SIGTERM EXIT

debug() { echo "$@" >&2; }

download_start() {
  URL=$1
  MODULE=`plowdown --get-module ${URL}`
  OUTPUT="/root/${MODULE}_OUT.txt"
  DATE=`date +"%F %H:%M:%S"`
  echo "" >> ${OUTPUT}
  echo "# START ${DATE}" >> ${OUTPUT}
  /usr/bin/plowdown ${PD_VERBOSE} ${PD_MAX_RETRIES} ${PD_TIMEOUT} ${PD_OUTPUT} ${PD_TEMP} -m ${URL} &>> ${OUTPUT}
}

# backup file
cp $INPUT_FILE $BACKUP_FILE

# main routine
file=$(<$INPUT_FILE)
echo "$file" | {
  while read line; do
    if [[ $line == *$URL* ]]; then
      download_start $URL
      rc=$?
      if [ "$rc" -ne "0" ]; then
        if [ "$rc" -ne "13" ]; then
    echo $line
        fi
      fi
    fi
    if [[ $line != *$URL* ]]; then
      echo $line
    fi
  done
} > $INPUT_FILE