crawler.sh

#!/bin/bash
DOWNLOAD_DIR="download"
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
printf "${BLUE}[INFO]${NC} Creating download directory: \"$DOWNLOAD_DIR\"\n"
mkdir -p $DOWNLOAD_DIR

SOURCE_FILE="source.html"
printf "${BLUE}[INFO]${NC} Downloading source of \"$1\"\n"
wget  -O source.html $1
regex_source="*\/([\w\d\-]+)"
if [[ $1 =~ regex_source ]]
then
    SOURCE_FILE="${BASH_REMATCH[1]}"
    printf "${BLUE}[INFO]${NC} Using sourcefile: \"$SOURCE_FILE\"\n"
else
     printf "${RED}[ERROR]${NC} REGEX CAN'T FIND FILENAME:  \"$SOURCE_FILE\"\n"
 fi
printf "${BLUE}[INFO]${NC} Executing regex on \"$SOURCE_FILE\"\n"
regex="^\/download_file\/([\d]+)$"



myvar=$(cat $SOURCE_FILE | grep -oP 'https:\/\/anime.thehylia.com\/download_file\/([0-9]+)')
    #    if [[ $line =~ $regex ]]
    echo $myvar | tr " " "\n" >> filelist

#    then
#
#        name="${BASH_REMATCH[1]}"
#        echo "${name}.jpg"    # concatenate strings
#        echo ${name} >> filelist
#    else
#        echo "$line doesn't match" >&2
#    fi


printf "${BLUE}[INFO]${NC} Starting downloading...  \n"

#cat filelist | parallel --gnu -j1 "wget --referer=$1 -P download {}"
wget --referer=$1 -P download -i filelist