repo sync

create_repos.sh

#!/usr/bin/env bash

SELF=${0##*/} SDIR=${0%/*}
########################################################################################################################
######################################################## config ########################################################
# Set some defaults
VERSION=0.3

CENTOS_RELEASE="7"

GEN_UPDATEINFO="$(readlink -f ${SDIR}/generate_updateinfo.py)"

EPEL_REPODATA="http://mirror.us.leaseweb.net/epel/${CENTOS_RELEASE}/x86_64/repodata"
CENTOS_LATEST_ERRATA="https://raw.githubusercontent.com/stevemeier/cefs/master/errata.latest.xml"

REPO_DOWNLOAD_DIR="/data/repo/html/centos/${CENTOS_RELEASE}"

REPO_LIST=(
    base updates extras centosplus sclo sclo-rh epel
    pgdg94 pgdg95 pgdg96 mariadb102 jams treasuredata
    elasticsearch-2 kibana-45 confluent-dist-41 confluent-41
)
####################################################### functions ######################################################
# Print usage information
function help() {
cat << EOF
Usage: ${SELF} [OPTION]...
Create and update yum repository with latest errata

  -h    Display this help message and exit
  -v    Verbose output

EOF
    return
}
########################################################################################################################
# Logging function
function log() {
    local level levels=(notice warning crit)
    level="+($(IFS='|';echo "${levels[*]}"))"

    shopt -s extglob; case ${1} in
        ${level}) level=${1}; shift ;;
        *) level=notice ;;
    esac; shopt -u extglob

    [[ -z ${RETVAL} ]] && { for RETVAL in "${!levels[@]}"; do
        [[ ${levels[${RETVAL}]} = "${level}" ]] && break
    done }

    logger -s -p ${level} -t "[${SELF}:${FUNCNAME[1]}()]" -- $@;
}
########################################################################################################################
# Log and then exit
function die() { local retval=${RETVAL:-$?}; log "$@"; exit ${retval}; }
########################################################################################################################
# Download CentOS/EPEL errata
function get_errata() {
    local retval updateinfo
    pwd

    # CentOS ----------------------------------------------------------------- #
    curl -sLk ${CENTOS_LATEST_ERRATA} > errata.latest.xml; retval=$?
    if [[ ${retval} -eq 0 ]]; then
        log notice "Successfully retrieved CentOS errata"

        ${GEN_UPDATEINFO} -r ${CENTOS_RELEASE} -l error -d ${PWD} -s all -t all errata.latest.xml; retval=$?
        if [[ ${retval} -eq 0 ]]; then
            log notice "${GEN_UPDATEINFO} was successful"
        else
            RETVAL=${retval} die "There was a problem running ${GEN_UPDATEINFO}"
        fi
    fi
    # ------------------------------------------------------------------------ #

    # EPEL ------------------------------------------------------------------- #
    updateinfo=$(curl -sLk ${EPEL_REPODATA} | sed -n 's/.*href="\([^"]*-updateinfo.xml.bz2\).*/\1/p')

    curl -sLk ${EPEL_REPODATA}/${updateinfo} | bunzip2 - > epel.updateinfo.xml; retval=$?
    if [[ ${retval} -eq 0 ]]; then
        log notice "Successfully retrieved EPEL errata"
    else
        RETVAL=${retval} die 'Unable to retrieve EPEL errata'
    fi
    # ------------------------------------------------------------------------ #
}
########################################################################################################################
# Prepare CentOS/EPEL errata
function build_errata() {
    local retval repo updateinfo_release_xml
    pwd

    find ${REPO_DOWNLOAD_DIR}/{base,epel}/repodata/ -name "*updateinfo.xml.*" -delete
    find ${REPO_DOWNLOAD_DIR}/* -maxdepth 1 -name .olddata -exec rm -rf {} +

    declare -A retval
    for repo in updates base epel; do
        case ${repo} in
            epel)   updateinfo_xml="epel.updateinfo.xml"                            ;;
            *)      updateinfo_xml="updateinfo-${CENTOS_RELEASE}/updateinfo.xml"    ;;
        esac
        modifyrepo ${updateinfo_xml} ${REPO_DOWNLOAD_DIR}/${repo}/repodata; retval[${repo}]=$?
    done

    for i in "${!retval[@]}"; do
        [[ ${retval[${i}]} -ne 0 ]] && RETVAL=${retval[${i}]} die "Unable to build ${i} errata"
    done

    log notice "Successfully built [$(echo ${!retval[@]}|sed 's/[ ]/, /g')] errata"

    find ${REPO_DOWNLOAD_DIR}/{base,epel,updates}/repodata -type f -exec chmod 644 {} \;
}
########################################################################################################################
# Synchronize repositories
function sync_repos() {
    local retval repo
    pwd

    declare -A retval
    for repo in ${REPO_LIST[@]}; do
        mkdir -p ${REPO_DOWNLOAD_DIR}/${repo}
        log notice "Starting ${repo} sync"
        reposync -m --download-metadata -r ${repo} -p ${REPO_DOWNLOAD_DIR}; retval[${repo}]=$?
    done

    unset repo; declare -a repo
    for i in "${!retval[@]}"; do [[ ${retval[${i}]} -ne 0 ]] && repo+=(${i}); done

    case ${#repo[@]} in
        0)  log notice "Successfully synchronized all repositories"                         ;;
        1)  log warning "Unable to sync ${repo[@]} repository"                              ;;
        *)  log crit "Unable to sync [$(echo ${repo[@]}|sed 's/[ ]/, /g')] repositories"    ;;
    esac
}
########################################################################################################################
# Create repositories
function make_repos() {
    local retval repo repo_dir
    pwd

    declare -A retval
    for repo in ${REPO_LIST[@]}; do
        log notice "Creating/updating ${repo} repository"
        repo_dir=${REPO_DOWNLOAD_DIR}/${repo}
        if [[ -e ${repo_dir}/comps.xml ]]; then
            createrepo --workers 2 -g ${repo_dir}/comps.xml --update ${repo_dir}; retval[${repo}]=$?
        else
            createrepo --workers 2 --update ${repo_dir}; retval[${repo}]=$?
        fi
    done

    unset repo; declare -a repo
    for i in "${!retval[@]}"; do [[ ${retval[${i}]} -ne 0 ]] && repo+=(${i}); done

    case ${#repo[@]} in
        0)  log notice "Successfully created/updated CentOS/EPEL repositories"                      ;;
        1)  log warning "Unable to create/update ${repo[@]} repository"                             ;;
        *)  log crit "Unable to create/update [$(echo ${repo[@]}|sed 's/[ ]/, /g')] repositories"  ;;
    esac
}
########################################################################################################################
########################################################################################################################
# Sanity checks
while getopts ":hv" opt; do
    case ${opt} in
        h)  help >&2; exit 1                                            ;;
        v)  VERBOSE=1                                                   ;;
        \?) echo "Invalid option: -${OPTARG}" >&2                       ;;
        :)  echo "Option -${OPTARG} requires an argument." >&2; exit 1  ;;
    esac
done; shift $((${OPTIND} - 1))

[[ ! -e ${GEN_UPDATEINFO} ]] && die "${GEN_UPDATEINFO} does not exist!"

req_progs=(bunzip2 createrepo curl logger modifyrepo reposync)
for p in ${req_progs[@]}; do
    hash "${p}" 2>&- || \
    { die "Required program \"${p}\" not found in \${PATH}."; }
done
######################################################### main #########################################################
function main() {
    local retval

    [[ ${VERBOSE} -eq 1 ]] || exec >${LOGFILE:-${SDIR}/${SELF%.*}.log} 2>&1

    get_errata
    build_errata
    sync_repos
    make_repos

    exit 0
}

WORKDIR="$(mktemp -d ${TMPDIR:-/tmp}/${SELF%.*}_XXXXXX)"

trap "/bin/rm -rf ${WORKDIR}" EXIT

cd ${WORKDIR} || die "Unable to change to temporary working directory: ${WORKDIR}"

main $@
########################################################################################################################