Skip to content

Instantly share code, notes, and snippets.

@knutov
Forked from ThinGuy/fcb-image-server.sh
Created February 3, 2024 21:21
Show Gist options
  • Save knutov/a9ba1fac909e8f3d16967ceaf2cf4c7d to your computer and use it in GitHub Desktop.
Save knutov/a9ba1fac909e8f3d16967ceaf2cf4c7d to your computer and use it in GitHub Desktop.
For offline use: This script mirrors images for maas, lxd, cloud, along with juju agents and gui's. Meant to be used in a lxd container.
#!/bin/bash
#### NOTE: This is a combined image file for maas images, lxd images, and Juju Steams.
#### The server name defaults to "images"
#### If creating a ssl cert, the following Subject Alternate Names should be used:
# DNS:$(hostname -f), DNS: $(hostname -s), DNS: maas-images, DNS: lxd-images, DNS: cloud-images, DNS: juju-images, DNS:images.linuxcontainers.org, DNS: us.images.containers.org, DNS: uk.images.containers.org, DNS: canonical.imagecontainers.org, DNS:cloud-images.ubuntu.com, DNS: streams.canonical.com
#### If using DNS poisoning/hijacking, ensure all the above domains resolve to the images contaier's IP address
### Testing: export DRY_RUN=true before running this script
[[ -n ${DRY_RUN} && ${DRY_RUN} = true ]] && { export DRY_RUN=true DRY_RUN_CMD='--dry-run'; } || { export DRY_RUN=false; export DRY_RUN_CMD= ; }
# Where data will be stored
export MIRROR_DIR='/srv/fcb'
# Revisions of each os image, juju agent, and juju gui to keep. 1 = Only keep latest versions/builds
export MAX=1
# Whether to include new minimal image builds
export INCLUDE_MINIMAL=true
export LOG='/var/log/mirror-images.log'
###########################
###### Begin Filters ######
###########################
# The following are simplestreams filters.
# Enclose variable in single quotes and pipe separate multiple values
####### Architectures #######
export ARCHES='amd64'
# Full architecture list
# amd64
# arm64
# armel
# armhf
# i386
# powerpc
# ppc64el
# s390x
####### MAAS #######
# MAAS Images, with the exception of centos, use the codename
export MAAS_RELEASES='xenial|bionic|centos7'
export MAAS_BOOTLOADERS='grub*|pxelinux'
export MAAS_ARCHES='amd64'
####### Ubuntu Releases #######
# For Ubuntu, you may use code names or release numbers for releases
# Usually adequate to versions currently covered under LTS
export RELEASES='xenial|bionic'
####### Non-Ubuntu Releases #######
# Non-Ubuntu releases are grabbed from images.linuxcontainers.org
# See https://us.images.linuxcontainers.org/ or https://uk.images.linuxcontainers.org/ for a full list
# These images are dailys only
# They will not have cloudinit on them, install via add via a tool such as packer
export OTHER_OS_LIST='centos|debian|oracle'
export OTHER_OS_RELEASES='7|stretch'
# Note: images.linuxcontainer will roundrobin between UK and US-based image servers
# If you must use a US-based server, make sure you are mirroring us.images.linuxcontainers.org
# If you must use a UK-based server, make sure you are mirroring uk.images.linuxcontainers.org
####### Juju Agent (aka Tools) and GUI #######
# Support may ask you to run proposed or devel to address a bug, so best to have tbese mirrored.
export AGENT_BUILDS='released|proposed|devel'
export GUI_BUILDS='released|devel'
# Note: Juju GUI only comes in one flavor (a compressed archive)
# so there are no "releases" to specify
export AGENT_RELEASES='xenial|bionic|centos7'
# Juju Agent Releases follows same format as ubuntu cloud images (you can use codename or release number)
# At a minmimum, these should match what you plan to deploy using maas, lxd, and kvm
## Full juju agent release list:
# 12.04
# 12.10
# 13.04
# 13.10
# 14.04
# 14.10
# 15.04
# 15.10
# 16.04
# 16.10
# 17.04
# 17.10
# 18.04
# 18.10
# 19.04
# centos7
# win7
# win8
# win10
# win81
# win2012
# win2012hv
# win2012hvr2
# win2012r2
# win2016
# win2016hv
# win2016nano
####### FORMAT TYPES #######
# Include files with this in their name/extension
export FTYPE_LIST='gz|xz|squashfs|img|ova'
#########################
###### End Filters ######
#########################
###### Begin Main Script ######
# List Packages required for image server in a container
declare -ag PKG_LIST=(squashfuse wget curl ubuntu-cloudimage-keyring ubuntu-cloud-keyring simplestreams nginx)
# Root user check
[[ $EUID -eq 0 ]] && export SCMD= || export SCMD=sudo
# Ensure directory exists
[[ -d ${MIRROR_DIR} ]] || $SCMD mkdir -p ${MIRROR_DIR}
# Check if packages are installed
#if [[ ! -f /var/tmp/first.run.complete ]];then
printf "\n\e[1mPerforming package checks...\e[0m\n"
[[ ! -f ${LOG:-/var/log/mirror-images.log} ]] && { printf "\e[2G- Creating log (${LOG:-/var/log/mirror-images.log})...\n";$SCMD touch ${LOG:-/var/log/mirror-images.log}; }
printf "\e[2G- Updating apt. Please wait...\n"
$SCMD apt 2>/dev/null update -yq|sudo tee -a ${LOG:-/var/log/mirror-images.log} &>/dev/null
printf "\e[2G- Performing dist-upgrade. Please wait...\n"
$SCMD apt 2>/dev/null dist-upgrade -yq|sudo tee -a ${LOG:-/var/log/mirror-images.log} &>/dev/null
printf "\e[2G- Ensuring required packages are installed...\n"
printf "%s\n" ${PKG_LIST[@]}|xargs -n1 -P1 bash -c '[[ -n $(dpkg -l|awk "/\<^ii $0\> /") ]] || { printf "\e[4G- Installing \e[1m$0\e[0m\n";$SCMD apt install $0 -yq|sudo tee -a ${LOG:-/var/log/mirror-images.log} &>/dev/null; }'
printf "\e[2G- Running apt autoremove. Please wait...\n"
$SCMD apt 2>/dev/null autoremove -yq|sudo tee -a ${LOG:-/var/log/mirror-images.log} &>/dev/null
# printf "\e[2G- Marking first-run as completed...\n"
# $SCMD touch /var/tmp/first.run.complete
#fi
########################################
# Mirror maas images #
########################################
export KEYRING_FILE=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg
printf "\n\n\e[1mCalculating download for MAAS Images ($(printf '%s\n' ${MAAS_RELEASES//|/,}))...\e[0m\n"
$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --keyring=${KEYRING_FILE} https://images.maas.io/ephemeral-v3/daily/ ${MIRROR_DIR}/images.maas.io/ephemeral-v3/daily 'release~('${MAAS_RELEASES}')' 'arch~('${MAAS_ARCHES}')'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'
printf "\e[2G - Current size (MiB) of MAAS Images: $(du -sm ${MIRROR_DIR}/images.maas.io/ephemeral-v3/daily/ --exclude=bootloaders|awk '{print $1,"("$2")"}')\n"
printf "\n\n\e[1mCalculating download for MAAS Bootloaders ($(printf '%s\n' ${MAAS_BOOTLOADERS//|/,}))...\e[0m\n"
$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --keyring=${KEYRING_FILE} https://images.maas.io/ephemeral-v3/daily/ ${MIRROR_DIR}/images.maas.io/ephemeral-v3/daily 'os~('${MAAS_BOOTLOADERS}')'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'
printf "\e[2G - Current size (MiB) of MAAS Bootloaders: $(du -sm ${MIRROR_DIR}/images.maas.io/ephemeral-v3/daily/bootloaders/|awk '{print $1,"("$2")"}')\n"
########################################
# Mirror Ubuntu GA images #
########################################
printf "\n\n\e[1mCalculating download for Ubuntu Cloud-Images (GA) ($(printf '%s\n' ${RELEASES//|/,}))...\e[0m\n"
$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --path streams/v1/index.json https://cloud-images.ubuntu.com/releases/ ${MIRROR_DIR}/cloud-images.ubuntu.com/releases/ 'datatype~(image-downloads)' 'release~('${RELEASES}')' 'arch~('${ARCHES}')' 'ftype~('${FTYPE_LIST}')'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'
printf "\e[2G - Current size (MiB) of Ubuntu Cloud-Images (GA): $(du -sm ${MIRROR_DIR}/cloud-images.ubuntu.com/releases/|awk '{print $1,"("$2")"}')\n"
########################################
# Mirror Ubuntu Daily images #
########################################
printf "\n\n\e[1mCalculating download for Ubuntu Cloud-Images (Daily) ($(printf '%s\n' ${RELEASES//|/,}))...\e[0m\n"
$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --path streams/v1/index.json https://cloud-images.ubuntu.com/daily/ ${MIRROR_DIR}/cloud-images.ubuntu.com/daily/ 'datatype~(image-downloads)' 'release~('${RELEASES}')' 'arch~('${ARCHES}')' 'ftype~('${FTYPE_LIST}')'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'
printf "\e[2G - Current size (MiB) of Ubuntu Cloud-Images (Daily): $(du -sm ${MIRROR_DIR}/cloud-images.ubuntu.com/daily/|awk '{print $1,"("$2")"}')\n"
########################################
# Mirror Ubuntu Minimal images #
########################################
[[ -n ${INCLUDE_MINIMAL} && ${INCLUDE_MINIMAL} = true ]] && { printf "\n\n\e[1mCalculating download for Ubuntu Minimal (GA)...\e[0m\n";$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --path streams/v1/index.json https://cloud-images.ubuntu.com/minimal/releases/ ${MIRROR_DIR}/cloud-images.ubuntu.com/minimal/releases/ 'path~(released)' 'datatype~(image-downloads)' 'release~('${RELEASES}')' 'arch~('${ARCHES}')' 'ftype~('${FTYPE_LIST}')'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'; }
[[ -n ${INCLUDE_MINIMAL} && ${INCLUDE_MINIMAL} = true ]] && { printf "\e[2G - Current size (MiB) of Mininal Ubuntu Cloud-Images (GA): $(du -sm ${MIRROR_DIR}/cloud-images.ubuntu.com/minimal/releases/|awk '{print $1,"("$2")"}')\n" ; }
[[ -n ${INCLUDE_MINIMAL} && ${INCLUDE_MINIMAL} = true ]] && { printf "\n\n\e[1mCalculating download for Ubuntu Minimal (Daily)...\e[0m\n";$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --path streams/v1/index.json https://cloud-images.ubuntu.com/minimal/daily/ ${MIRROR_DIR}/cloud-images.ubuntu.com/minimal/daily/ 'path~(daily)' 'datatype~(image-downloads)' 'release~('${RELEASES}')' 'arch~('${ARCHES}')' 'ftype~('${FTYPE_LIST}')'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'; }
[[ -n ${INCLUDE_MINIMAL} && ${INCLUDE_MINIMAL} = true ]] && { printf "\e[2G - Current size (MiB) of Mininal Ubuntu Cloud-Images (Daily): $(du -sm ${MIRROR_DIR}/cloud-images.ubuntu.com/minimal/daily/|awk '{print $1,"("$2")"}')\n" ; }
########################################
# Mirror Non-Ubuntu images #
########################################
[[ -n ${OTHER_OS_LIST} ]] && { printf "\n\n\e[1mCalculating download for Non-Ubuntu OSes ($(printf '%s\n' ${OTHER_OS_LIST//|/,}))...\e[0m\n";$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --path streams/v1/index.json https://images.linuxcontainers.org/ ${MIRROR_DIR}/images.linuxcontainers.org/ 'path~('${OTHER_OS_LIST}')' 'release~('${OTHER_OS_RELEASES}')' 'arch~('${ARCHES}')'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'; }
[[ -n ${OTHER_OS_LIST} ]] && { printf "\e[2G - Current size (MiB) of Non-Ubuntu OSes: $(du -sm ${MIRROR_DIR}/images.linuxcontainers.org/|awk '{print $1,"("$2")"}')\n"; }
########################################
# Mirror Juju Agents #
########################################
if [[ -n $(grep -iE '\<released\>' <<< ${AGENT_BUILDS}) ]];then
printf "\n\n\e[1mCalculating download for \"released\" build of Juju Agent for $(printf '%s\n' ${RELEASES//|/,})...\e[0m\n"
$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --no-verify --path streams/v1/index.json https://streams.canonical.com/juju/tools/ ${MIRROR_DIR}/streams.canonical.com/juju/tools 'content_id~(released)' 'release~('${AGENT_RELEASES}')' 'arch~('${ARCHES}')' 'version~(^2.*)'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'
[[ ${DRY_RUN} = true ]] || printf "\e[2G - Current size (MiB) of Juju Agent:Released: $(du -sm ${MIRROR_DIR}/streams.canonical.com/juju/tools/agent --exclude=proposed --exclude=devel|awk '{print $1,"("$2")"}')\n"
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/tools/streams/v1/com.canonical.streams-released-tools.json https:///streams.canonical.com/juju/tools/streams/v1/com.canonical.streams-released-tools.json
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/tools/streams/v1/index.json https://streams.canonical.com/juju/tools/streams/v1/index.json
fi
if [[ -n $(grep -iE '\<proposed\>' <<< ${AGENT_BUILDS}) ]];then
printf "\n\n\e[1mCalculating download for \"proposed\" build of Juju Agent for $(printf '%s\n' ${RELEASES//|/,})...\e[0m\n"
$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --no-verify --path streams/v1/index2.json https://streams.canonical.com/juju/tools/ ${MIRROR_DIR}/streams.canonical.com/juju/tools 'content_id~(proposed)' 'release~('${AGENT_RELEASES}')' 'arch~('${ARCHES}')' 'version~(^2.*)'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'
[[ ${DRY_RUN} = true ]] || printf "\e[2G - Current size (MiB) of Juju Agent:Proposed: $(du -sm ${MIRROR_DIR}/streams.canonical.com/juju/tools --exclude=released --exclude=devel|awk '{print $1,"("$2")"}')\n"
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/tools/streams/v1/com.ubuntu.juju:proposed:tools.json https://streams.canonical.com/juju/tools/streams/v1/com.ubuntu.juju:proposed:tools.json
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/tools/streams/v1/index.json https://streams.canonical.com/juju/tools/streams/v1/index.json
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/tools/streams/v1/index2.json https://streams.canonical.com/juju/tools/streams/v1/index2.json
fi
if [[ -n $(grep -iE '\<devel\>' <<< ${AGENT_BUILDS}) ]];then
printf "\n\n\e[1mCalculating download for \"devel\" build of Juju Agent for $(printf '%s\n' ${RELEASES//|/,})...\e[0m\n"
$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --no-verify --path streams/v1/index2.json https://streams.canonical.com/juju/tools/ ${MIRROR_DIR}/streams.canonical.com/juju/tools 'content_id~(devel)' 'release~('${AGENT_RELEASES}')' 'arch~('${ARCHES}')' 'version~(^2.*)'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'
[[ ${DRY_RUN} = true ]] || printf "\e[2G - Current size (MiB) of Juju Agent:Devel: $(du -sm ${MIRROR_DIR}/streams.canonical.com/juju/tools/ --exclude=released --exclude=proposed|awk '{print $1,"("$2")"}')\n"
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/tools/streams/v1/com.ubuntu.juju:devel:tools.json https://streams.canonical.com/juju/tools/streams/v1/com.ubuntu.juju:devel:tools.json
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/tools/streams/v1/index.json https://streams.canonical.com/juju/tools/streams/v1/index.json
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/tools/streams/v1/index2.json https://streams.canonical.com/juju/tools/streams/v1/index2.json
fi
########################################
# Mirror Juju GUI #
########################################
if [[ -n $(grep -iE '\<released\>' <<< ${GUI_BUILDS}) ]];then
printf "\n\n\e[1mCalculating download for Juju GUI for build: \"Released\" ...\e[0m\n"
$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --no-verify --path streams/v1/index.json https://streams.canonical.com/juju/gui/ ${MIRROR_DIR}/streams.canonical.com/juju/gui/ 'content_id~(released)' 'juju-version~(2|3)'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'
[[ ${DRY_RUN} = true ]] || printf "\e[2G - Current size (MiB) of Juju GUI:Released: $(du -sm ${MIRROR_DIR}/streams.canonical.com/juju/gui/ --exclude=devel|awk '{print $1,"("$2")"}')\n"
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/gui/streams/v1/com.canonical.streams-released-gui.json https://streams.canonical.com/juju/gui/streams/v1/com.canonical.streams-released-gui.json
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/gui/streams/v1/index.json https://streams.canonical.com/juju/gui/streams/v1/index.json
fi
if [[ -n $(grep -iE '\<devel\>' <<< ${GUI_BUILDS}) ]];then
printf "\n\n\e[1mCalculating download for Juju GUI for build: \"Devel\" ...\e[0m\n"
$SCMD sstream-mirror 2>&1 ${DRY_RUN_CMD} --progress --max=${MAX} --no-verify --path streams/v1/com.canonical.streams-devel-gui.json https://streams.canonical.com/juju/gui/ ${MIRROR_DIR}/streams.canonical.com/juju/gui/ 'content_id~(devel)' 'juju-version~(2|3)'|sudo tee -a ${LOG:-/var/log/mirror-images.log}|awk '{if (/change/) print " - "$0;else if (/^+ /||/^- /) print "Mirroring "$5,"("$6,$7")"}'
[[ ${DRY_RUN} = true ]] || printf "\e[2G - Current size (MiB) of Juju GUI:Devel: $(du -sm ${MIRROR_DIR}/streams.canonical.com/juju/gui/ --exclude=released|awk '{print $1,"("$2")"}')\n"
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/gui/streams/v1/com.canonical.streams-devel-gui.json https://streams.canonical.com/juju/gui/streams/v1/com.canonical.streams-devel-gui.json
[[ ${DRY_RUN} = true ]] || $SCMD wget -qO ${MIRROR_DIR}/streams.canonical.com/juju/gui/streams/v1/index.json https://streams.canonical.com/juju/gui/streams/v1/index.json
fi
##
# Build nginx site file
[[ -h /etc/nginx/sites-enabled/default ]] && { printf "\n\e[1mRemoving default nginx site...\e[0m\n";$SCMD rm -f /etc/nginx/sites-enabled/default ; }
[[ -h /etc/nginx/sites-enabled/image-server ]] && { printf "\n\e[1mRemoving existing image-server site...\e[0m\n";$SCMD rm -f /etc/nginx/sites-enabled/image-server ; }
printf "\n\e[1mCreating nignx site: /etc/nginx/sites-available/image-server\e[0m\n"
cat << EOF|$SCMD tee /etc/nginx/sites-available/image-server
server {
listen 80;
listen [::]:80;
server_name images.maas.io maas-images;
location / {
alias ${MIRROR_DIR}/maas/images/;
autoindex on;
}
}
server {
listen 80;
listen [::]:80;
server_name cloud-images.ubuntu.com cloud-images;
location / {
alias ${MIRROR_DIR}/cloud-images.ubuntu.com/;
autoindex on;
}
}
server {
listen 80;
listen [::]:80;
server_name streams.canonical.com streams;
location / {
alias ${MIRROR_DIR}/streams.canonical.com/;
autoindex on;
}
}
server {
listen 80;
listen [::]:80;
server_name linuxcontainers.org images.linuxcontainers.org us.images.linuxcontainers.org uk.images.linuxcontainers.org canonical.images.linuxcontainers.org lxd-images;
location / {
alias ${MIRROR_DIR}/images.linuxcontainers.org/;
autoindex on;
}
}
EOF
printf "\nSymlinking \e[1m/etc/nginx/sites-available/image-server\e[0m to \e[1m/etc/nginx/sites-enabled/image-server\e[0m \n"
$SCMD ln -s /etc/nginx/sites-available/image-server /etc/nginx/sites-enabled/image-server
printf "\n\e[1mRestarting nignx...\e[0m\n"
$SCMD systemctl restart nginx
printf "\n\e[1m${0##*/} has completed! \e[0m\n"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment