[Download CDH and CM install packages] Using axel download CDH and CM install packages. Need manaully take an https list. #download #cdh #shellfor

download_cdh_repo.sh

#!/usr/bin/env bash
#
# Copyright futeng@tydic.com
# May 9, 2020 
#
# 脚本：download-cdh-cm.sh
# 功能：从 CDH 网站使用 axel 多线程下载介质
# 位置：任意目录执行
# 执行：脚本硬编码了，需要包含一个 list 文件（都是 http 链接）。可根据实际情况稍加修改。


# CM：http://archive.cloudera.com/cm7/7.0.3/redhat7/yum/RPMS/x86_64/
# CDH：http://archive.cloudera.com/cdh7/7.0.3.0/parcels/


#axel -n 2 -o /home/cdh/cdh5.13.3/tar-src https://archive.cloudera.com/cdh5/cdh/5/avro-1.7.6-cdh5.13.3-src.tar.gz


list_suffix=list-cdh-cm7

# 1. 获取待下载列表


ALL_LIST=$(ls -l | grep $list_suffix | awk '{print $9}')
DIR_HOME="/home/cdh"


# 2. 逐个下载
for element in ${ALL_LIST[@]}
do
    echo $element
    DIR=$(echo $element | cut -c1-9)
    SUB_DIR=$(echo ${element##*_} | cut -d "." -f1)
    DOWN_DIR="$DIR_HOME/$DIR/$SUB_DIR"
    mkdir -p $DOWN_DIR
    #echo $DIR
    #echo $SUB_DIR
    cat $element | while read URL
    do
        echo "------------->"
          echo "${URL}"
        axel -n 5 -o  $DOWN_DIR     ${URL}
    done
done