Bash Divide&Conquer scripts

divide-and-conquer.sh

#!/usr/bin/env bash
TASK_LIMIT=4

# ########################## HOW-TO ########################################## #
# 1. Include in your script with source <(curl -s <URL>)                       #
# 2. Override max simultaneous workers by TASK_LIMIT                           #
# 3. Define func1 that will generate workers data/ atleast their ids           #
# 4. Define func2 that will contain logic to handle segmet of worker           #
# 5. Define func3 that will agregate computed results                          #
# 6. Run divide_and_conquer func1 func2 func2 and wait for script to end       #
# ############################################################################ #

wait_for_tasks() {
  # PIDS ...
  if [ "$#" == "0" ]; then
    return 1
  fi
  if [ "$#" -ge "${TASK_LIMIT}" ]; then
    echo -e "\e[91m$(date) | Wait for [$#] tasks to complete.\e[39m"
    wait $@
    return 0
  fi
  return 1
}

divide_and_conquer() {
  # CHUNK_GENERATOR / TASK_ROUTINE / RESULT_JOINER
  local PIDS=""
  local CHUNK_I=1
  local CHUNKS=$(eval $1)
  local CHUNKS_LEN=$(( 1 + `echo ${CHUNKS} | tr -cd ' ' | wc -c | sed 's,^ *,,; s, *$,,'` ))
  for CHUNK in  ${CHUNKS} ; do
    echo -e "\e[33m$(date) | Executing task ${CHUNK_I}/${CHUNKS_LEN} [${CHUNK}] \e[39m"
    eval $2 ${CHUNK} &
    PIDS="$PIDS $!"
    if wait_for_tasks ${PIDS}; then
      PIDS=""
    fi
    CHUNK_I=$(( 1 + ${CHUNK_I} ))
  done
  local TASK_LIMIT=1
  wait_for_tasks ${PIDS}
  eval $3 ${CHUNKS}
}

jump-host-scp-downloader.sh

#!/usr/bin/env bash
source <(curl -s https://gist.githubusercontent.com/aenniw/01b386986117c3d915f49401902628c4/raw/48f0f5cc91ab94cf5e8d4c0f7067ac7999e62805/divide-and-conquer.sh)
TASK_LIMIT=6

URL=$1
HOST_NAME=$2
FILE_NAME_ORIGINAL=${URL##*/}
FILE_NAME=$(tempfile)

task_creator() {
  cd /tmp
  wget -O ${FILE_NAME} ${URL}
  split -b 20M ${FILE_NAME} ${FILE_NAME}-part- && rm ${FILE_NAME}
  cd $oldpwd
  echo $(ls ${FILE_NAME}-part-*)
}

task_routine() {
  scp $1 ${HOST_NAME}:$1 && rm $1
}

task_joiner() {
  ssh ${HOST_NAME} "cat $@ > /tmp/${FILE_NAME_ORIGINAL}"
  ssh ${HOST_NAME} "rm $@"
}

divide_and_conquer "task_creator" 'task_routine' "task_joiner"