Mirror openwrt repo

mirror-openwrt-repo.sh

#!/bin/bash
#
#     version: 1.12
# description: reads Packages.gz and download ipk and checks md5sum
#     purpose: archive.openwrt.org have had several uptime issues, last some disk issues
#              since it acts as repo for older releases, it's better to be safe than sorry
#              freifunk offers a cloned repo for [chaos calmer - 15.05.1]
#
# history: v1.00 - initial script
#          v1.01 - fixed awk typo in f_parse_pkgz
#          v1.10 - added mirror and verify checking before dowloadning
#                  added comments and local vars for easier reading
#          v1.11 - added usage section and final statistics
#          v1.12 - added check for script update
#

gist=https://gist.github.com/agail/901fb7b74034b684025aae600ae561f1
official=http://archive.openwrt.org
mirror=http://download.pinneberg.freifunk.net/openwrt
sites="official mirror"
tree=chaos_calmer/15.05.1/ar71xx/generic/packages
repos="base packages luci"
local_dir=./chaos_calmer/15.05.1/ar71xx/generic/packages
ipkfail=0
ipkmiss=0
chkupdate=1             # toggle automatic update check 0|1
myopt=$1                # enter 'd' q/o qoutes for downloading, default is check ipk's

f_chk_site () {         # check remote repo connectivity
for s in ${sites}; do
        _stat="$(curl -siL --retry 2 --connect-timeout 2 ${!s} | grep '200 OK')"
        if [ ! "${_stat}x" == "x" ]; then
                echo -e "\n \e[32mUsing $s: ${!s}\e[0m"
                main_url=${!s}
                return
        else
                echo -e "\e[91mConnection failure to $s (${!s})\e[0m"
        fi
done
}

f_dirs () {             # build local repo structure
 local _repo=$1
 if [ ! -d ${local_dir}/${_repo} ]; then
        mkdir -p ${local_dir}/${_repo}
 fi
}

f_fetch_pkgz () {       # pull initial Package.gz, keep initial if same as next pull.
 local _repo=$1; local _url=$2; local _file=$3
 f_dirs ${_repo}
 if [ ! -r ${local_dir}/${_repo}/${_file} ]; then
     curl -sL -o ${local_dir}/${_repo}/${_file} ${_url}/${_file}
 else
     curl -sL -o ${local_dir}/${_repo}/${_file}.tmp ${_url}/${_file}
     if [ ! "$(f_chk_md5 ${_repo} ${_file})" == "$(f_chk_md5 ${_repo} ${_file}.tmp)" ]; then
        cp ${local_dir}/${_repo}/${_file}.tmp ${local_dir}/${_repo}/${_file}
     fi
     rm ${local_dir}/${_repo}/${_file}.tmp
 fi
}

f_parse_pkgz () {       # Package.gz fetch filename and md5sum for each ipk
 awk -F "[ ]" '/^F.*e:/ || /^MD5.*:/ { print $NF}' | sed -rz 's/\n([^\n]*\n)/ \1/g'
}

f_read_pkgz () {        # parse packages
 local repo=$1; local url=$2; local pkgz=$3
 while read -r pack; do
    f_chk_ipk $repo ${pack/ *} ${url} ${pkgz} ${pack/* }
 done < <( zcat ${local_dir}/${repo}/${pkgz} | f_parse_pkgz )
}

f_chk_ipk () {          # check ipk integrity, download if missing or failed
 local repo=$1; local ipk=$2; local url=$3; local pkgz=$4; local imd5=$5
 if [ -r ${local_dir}/${repo}/${ipk} ]; then
    if [ "$(f_chk_md5 ${repo} ${ipk})" == "${imd5}" ]; then
        echo -e "\e[32mChksum OK\e[0m\t${ipk}"
    else
        if [ "${myopt}" == "d" ]; then        # If $opt is d, download
            f_dl_pkg $repo $ipk $url ${imd5}
        else
            ((ipkfail++))
            echo -e "\e[91mChksum Fail\e[0m\t${ipk}"
        fi
    fi
 else
    if [ "${myopt}" == "d" ]; then            # If $opt is d, download
         f_dl_pkg $repo $ipk $url ${imd5}
    else
         ((ipkmiss++))
         echo -e "\e[93mMissing ipk\e[0m\t${ipk}"
    fi
 fi
}

f_dl_pkg () {           # download ipk
 local repo=$1; local ipk=$2; local url=$3; local imd5=$4
 curl -sLo ${local_dir}/${repo}/${ipk} ${url}/${ipk}
 if [ "$(f_chk_md5 ${repo} ${ipk})" == "${imd5}" ]; then
     echo -e "\e[32mChksum OK\e[0m\t$2"
 else ((ipkfail++)); echo -e "\e[91mChksum Fail\e[0m\t$2"
 fi
}

f_chk_md5 () {          # md5sum each ipk
 local repo=$1; local ipk=$2
 md5sum --tag ${local_dir}/${repo}/${ipk} | awk '{print $NF}'
}

f_update () {
if [ ${chkupdate} -eq 0 ]; then return; fi
gver=$(curl -s ${gist} | grep -m1 -oP '(?<=version:.).*(?=</span>)')
sver=$(grep -m1 -oP '(?<=version:.).*(?=)' $0)
if [ ${gver/.} -gt ${sver/.} ]; then
    echo -e "\n \e[94ma new version [${gver}] of ${0##*/} is available on ${gist}i\e[0m\n"
fi
}

f_usage () {
cat <<-EOF
$(echo -e "\e[94m")$(sed -ne '/^#$/,/^$/p' $0)$(echo -e "\e[0m")

 ${0##*/} [opt]

 opt:     c - check ipk's integrity
          d - download missing or failed ipk's

EOF
}

f_update
if [ $# -eq 0 ]; then f_usage; exit; fi
f_chk_site

for repo in ${repos}; do
        echo -e "\n \e[90mProcessing [\e[0m${repo}\e[90m]\e[0m\n"
        f_fetch_pkgz ${repo} ${main_url}/${tree}/${repo} Packages.gz
        f_read_pkgz ${repo} ${main_url}/${tree}/${repo} Packages.gz
done
echo
if [ ${ipkfail} -gt 0 ]; then echo -e "\e[91m ${ipkfail} ipk failed"; fi
if [ ${ipkmiss} -gt 0 ]; then echo -e "\e[93m ${ipkmiss} ipk missing"; fi
if [[ ${ipkfail} -gt 0 || ${ipkmiss} -gt 0 ]]; then echo -e "\n \e[94m** re-run with \e[0md\e[94m to fix **\e[0m\n"; fi