Download NetBSD-daily ISO images

download-netbsd-iso

#!/bin/sh
# @(#)$KimmoSuominen: .cfg/home/dot.cron/download-netbsd-iso,v 1.10 2022/12/31 14:12:37 kim Exp $
#
# Copyright (c) 2020-2022 Kimmo Suominen
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer
#    in the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

#
# Download NetBSD-daily ISO images
#
# 20201016  Kimmo Suominen
#

set -eu

PATH=/bin:/usr/bin:/usr/pkg/bin
export PATH

PROG="${0##*/}"

DAILY_URL=http://nycdn.netbsd.org/pub/NetBSD-daily
DLDIR="${HOME}/Downloads/NetBSD"
VZDIR=/var/lib/vz/template/iso

DEFAULT_ARCH=amd64

umask 022

msg()
{
    echo "${PROG}: ${@}"
}

warn()
{
    echo "${PROG}: ${@}" 1>&2
}

find_iso()
{
    local branch latest arch
    branch="${1}"
    latest="${2}"
    arch="${3}"
    curl -Ls "${DAILY_URL}/${branch}/${latest}/images/" \
    | sed -n '
	/^.*<a href="/ {
	    s///
	    s,".*$,,
	    /^NetBSD-.*-'"${arch}"'\.iso$/ p
	}
    ' \
    | sed 1q
}

find_latest()
{
    local branch
    branch="${1}"
    curl -Ls "${DAILY_URL}/${branch}/" \
    | sed -n '
	/^.*<a href="/ {
	    s///
	    s,/".*$,,
	    /^latest$/ {g; p; d;}
	    h
	}
    ' \
    | sed 1q
}

list_all()
{
    local arch branch latest iso
    arch="${1}"

    for branch in $(list_branches)
    do
	latest="$(find_latest "${branch}")"
	case "${latest}" in
	2*Z)
	    iso="$(find_iso "${branch}" "${latest}" "${arch}")"
	    case "${iso}" in
	    NetBSD-*-"${arch}".iso)
		echo "${DAILY_URL}/${branch}/${latest}/images/${iso}"
		;;
	    esac
	    ;;
	esac
    done
}

list_branches()
{
    curl -Ls "${DAILY_URL}/" \
    | sed -n '
	/^.*<a href="/ {
	    s///
	    s,/".*$,,
	    /^HEAD$/ p
	    /^netbsd-/ p
	}
    '
}

usage()
{
    cat <<EOF
Usage:	${PROG} [-dnqv] url [...]
	${PROG} [-dnqv] -a [-k N] [arch]
	${PROG} -h

Options:
-a	Fetch all the latest images (default arch: ${DEFAULT_ARCH})
-d	Show retrieval URLs and output filenames (debug)
-h	Show this usage message (help)
-k	Keep N most recent images (default: 3; specify 0 to keep all)
-n	Show what would be fetched (no-op)
-q	Do not show retrieval and removal commands (quiet)
-v	Output actions that are or would be taken (verbose)
EOF
}

all=false
debug=false
help=false
keep=3
noop=false
quiet=false
verbose=false

while getopts adhk:nqv opt
do
    case "${opt}" in
    a)	all=true;;
    d)	debug=true;;
    h)	help=true;;
    k)	keep="$(echo "${OPTARG}" | tr -cd 0-9)";;
    n)	noop=true;;
    q)	quiet=true; verbose=false;;
    v)	verbose=true; quiet=false;;
    *)  exit 1;;
    esac
done
shift $((${OPTIND} - 1))

${help} && { usage; exit 0; }
${noop} && verbose=true

if ${all}
then
    case ${#} in
    0) arch="${DEFAULT_ARCH}";;
    1) arch="${1}";;
    *) usage 1>&2; exit 1;;
    esac
    set -- $(list_all "${arch}")
else
    if [ ${#} -lt 1 ]
    then
	echo 'No URL provided (and -a not used)' 1>&2
	exit 1
    fi
    keep=0
fi


[ -d "${DLDIR}" ] || ${noop} || mkdir -p "${DLDIR}"

[ -t 1 ] || notty=true
${quiet} && notty=true

for url
do
    ${debug} && echo "<-- ${url}"

    iso="${url##*/}"
    ext="${iso##*.}"
    iso="${iso%.*}"

    buildid="${url%/*}"
    buildid="${buildid%/*}"
    buildid="${buildid##*/}"
    
    case "${buildid}" in
    202[0-9]*[0-9]Z)
	# looks like a date
	# 202010160010Z
	;;
    *)
	warn "Build ID '${buildid}' looks dubious"
	continue
	;;
    esac

    case "${ext}" in
    iso)
	# ok
	;;
    *)
	warn "File extension '${ext}' looks dubious"
	continue
	;;
    esac

    case "${iso}" in
    NetBSD-[98].*-*|NetBSD-1[0-9].*-*)
	# ok
	;;
    *)
	warn "Image basename '${iso}' looks dubious"
	continue
	;;
    esac

    outfile="${iso}-${buildid}.${ext}"
    ${debug} && echo "--> ${outfile}"

    if [ -e "${DLDIR}/${outfile}" ] || [ -e "${VZDIR}/${outfile}" ]
    then
	${verbose} && msg "${outfile} already exists"
    else
	${verbose} && msg "${outfile} needs to be downloaded"
	${noop} || (
	    ${quiet} || set -x
	    curl ${notty:+-s} -R -o "${DLDIR}/.${outfile}" "${url}" \
	    && mv "${DLDIR}/.${outfile}" "${DLDIR}/${outfile}"
	)
    fi
done

#
# CLEANUP
#

[ ${keep:-0} -lt 2 ] && exit 0
[ -d "${DLDIR}" ] || exit 0

get_archs()
{
    echo "${1}" \
    | cut -d- -f3 \
    | uniq
}

get_relarch_isos()
{
    echo "${1}" \
    | grep "^NetBSD-${2}-${3}-"
}

get_rels()
{
    echo "${1}" \
    | cut -d- -f2 \
    | sed 's/\.99\.[^-]*/.99/' \
    | uniq
}

list_isos()
{
    ls -1 "${DLDIR}" \
    | sed -n '/^NetBSD-.*-.*-.*\.iso$/ p'
}

isos="$(list_isos)"
for rel in $(get_rels "${isos}")
do
    qrel="$(echo "${rel}" | sed 's/\./\\./')"
    case "${rel}" in
    *.99)
	qrel="${qrel}.*"
	;;
    esac
    for arch in $(get_archs "${isos}")
    do
	get_relarch_isos "${isos}" "${qrel}" "${arch}"
	get_relarch_isos "${isos}" "${qrel}" "${arch}" \
	| sort -t. -k 1.8n,1 -k 2,2n -k 3,3n \
	| tail -${keep}
    done \
    | sort -t. -k 1.8n,1 -k 2,2n -k 3,3n \
    | uniq -u
done \
| while read rmfile
do
    # Could use xargs, but then the output would not match the download part.
    # Not that many files get removed, so inefficiency is ok.
    ${verbose} && msg "${rmfile} can be removed"
    ${noop} || (
	${quiet} || set -x
	rm "${DLDIR}/${rmfile}"
    )
done