Skip to content

Instantly share code, notes, and snippets.

@kescherCode
Last active March 6, 2022 12:46
Show Gist options
  • Save kescherCode/7957f487e8cd04206619fbc94ac620ea to your computer and use it in GitHub Desktop.
Save kescherCode/7957f487e8cd04206619fbc94ac620ea to your computer and use it in GitHub Desktop.
A mirror script for the arch4edu repositories. Assume MIT License.
#!/usr/bin/env bash
set -e
# This is a simple mirroring script. To save bandwidth it first checks a
# timestamp via HTTP and only runs rsync when the timestamp differs from the
# local copy. As of 2016, a single rsync run without changes transfers roughly
# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync
# is run every minute. Performing a simple check via HTTP first can thus save a
# lot of traffic.
# Directory where the repo is stored locally. Example: /srv/repo
target="/srv/mirror/arch4edu"
# Directory where files are downloaded to before being moved in place.
# This should be on the same filesystem as $target, but not a subdirectory of $target.
# Example: /var/tmp/repo-tmp
tmp="/var/tmp/arch4edu-tmp"
# Lockfile path
# It's recommended to put the lockfile on a tmpfs (such as /tmp on most systems),
# since after a new boot, the sync is aborted.
# If a lockfile was to be persistent in this scenario, it would require manual intervention.
lock="/tmp/.arch4edu-sync-lock"
# If you want to limit the bandwidth used by rsync set this.
# Use 0 to disable the limit.
# The default unit is KiB (see man rsync /--bwlimit for more)
bwlimit=0
# The source URL of the mirror you want to sync from.
# If you are a tier 1 mirror use:
# rsync://mirrors.tuna.tsinghua.edu.cn/arch4edu/
# Otherwise, check if any of the mirrors at
# https://github.com/arch4edu/mirrorlist/blob/master/mirrorlist.arch4edu
# support an rsync connection and use that instead.
source_url='rsync://mirrors.tuna.tsinghua.edu.cn/arch4edu/'
# An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror.
# If you are a tier 1 mirror use: https://mirrors.tuna.tsinghua.edu.cn/arch4edu/lastupdate
# Otherwise use the HTTP(S) URL from your chosen mirror.
lastupdate_url='https://mirrors.tuna.tsinghua.edu.cn/arch4edu/lastupdate'
#### END CONFIG
[ ! -d "${target}" ] && mkdir -p "${target}"
[ ! -d "${tmp}" ] && mkdir -p "${tmp}"
exec 9>"${lock}"
flock -n 9 || exit
rsync_cmd() {
local -a cmd=(rsync -rtliH --delete-after --delay-updates --safe-links --max-delete=1000 \
"--timeout=600" "--contimeout=60" -p --no-motd "--temp-dir=${tmp}")
if stty &>/dev/null; then
cmd+=(-h -v --progress)
else
cmd+=(--quiet)
fi
if ((bwlimit>0)); then
cmd+=("--bwlimit=$bwlimit")
fi
"${cmd[@]}" "$@"
}
# Only run when there are changes
if [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
# If your chosen mirror DOES have a lastsync file, uncomment the next three lines.
# rsync_cmd "$source_url/lastsync" "$target/lastsync"
# echo "Last sync was $(date -d @"$(cat ${target}/lastsync)" --rfc-3339=seconds -u)" > "${target}/info"
# echo "Last update was $(date -d @"$(cat ${target}/lastupdate)" --rfc-3339=seconds -u)" >> "${target}/info"
exit 0
fi
rsync_exclude=("--exclude=/info")
# Uncomment below if your chosen mirror DOES NOT have a lastsync file
rsync_exclude+=("--exclude=/lastsync")
rsync_cmd \
"${rsync_exclude[@]}" \
"${source_url}" \
"${target}"
# Uncomment below if your chosen mirror DOES NOT have a lastsync file
date -u "+%s" > "$target/lastsync"
# Uncomment below if you've set up everything above correctly.
echo "Last sync was $(date -d @"$(cat ${target}/lastsync)" --rfc-3339=seconds -u)" > "${target}/info"
echo "Last update was $(date -d @"$(cat ${target}/lastupdate)" --rfc-3339=seconds -u)" >> "${target}/info"
#!/usr/bin/env bash
set -e
# This is a simple mirroring script. To save bandwidth it first checks a
# timestamp via HTTP and only runs rsync when the timestamp differs from the
# local copy. As of 2016, a single rsync run without changes transfers roughly
# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync
# is run every minute. Performing a simple check via HTTP first can thus save a
# lot of traffic.
# Directory where the repo is stored locally. Example: /srv/repo
target="/srv/mirror/arch4edu"
# Directory where files are downloaded to before being moved in place.
# This should be on the same filesystem as $target, but not a subdirectory of $target.
# Example: /var/tmp/repo-tmp
tmp="/var/tmp/arch4edu-tmp"
# Lockfile path
# It's recommended to put the lockfile on a tmpfs (such as /tmp on most systems),
# since after a new boot, the sync is aborted.
# If a lockfile was to be persistent in this scenario, it would require manual intervention.
lock="/tmp/.arch4edu-sync-lock"
# If you want to limit the bandwidth used by rsync set this.
# Use 0 to disable the limit.
# The default unit is KiB (see man rsync /--bwlimit for more)
bwlimit=0
# The source URL of the mirror you want to sync from.
# If you are a tier 1 mirror use:
# rsync://mirrors.tuna.tsinghua.edu.cn/arch4edu/
# Otherwise, check if any of the mirrors at
# https://github.com/arch4edu/mirrorlist/blob/master/mirrorlist.arch4edu
# support an rsync connection and use that instead.
source_url='rsync://at.arch4edu.mirror.kescher.at/mirror/arch4edu/'
# An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror.
# If you are a tier 1 mirror use: https://mirrors.tuna.tsinghua.edu.cn/arch4edu/lastupdate
# Otherwise use the HTTP(S) URL from your chosen mirror.
lastupdate_url='https://at.arch4edu.mirror.kescher.at/lastupdate'
#### END CONFIG
[ ! -d "${target}" ] && mkdir -p "${target}"
[ ! -d "${tmp}" ] && mkdir -p "${tmp}"
exec 9>"${lock}"
flock -n 9 || exit
rsync_cmd() {
local -a cmd=(rsync -rtliH --delete-after --delay-updates --safe-links --max-delete=1000 \
"--timeout=600" "--contimeout=60" -p --no-motd "--temp-dir=${tmp}")
if stty &>/dev/null; then
cmd+=(-h -v --progress)
else
cmd+=(--quiet)
fi
if ((bwlimit>0)); then
cmd+=("--bwlimit=$bwlimit")
fi
"${cmd[@]}" "$@"
}
# Only run when there are changes
if [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
# If your chosen mirror DOES have a lastsync file, uncomment the next three lines.
rsync_cmd "$source_url/lastsync" "$target/lastsync"
echo "Last sync was $(date -d @"$(cat ${target}/lastsync)" --rfc-3339=seconds -u)" > "${target}/info"
echo "Last update was $(date -d @"$(cat ${target}/lastupdate)" --rfc-3339=seconds -u)" >> "${target}/info"
exit 0
fi
rsync_exclude=("--exclude=/info")
# Uncomment below if your chosen mirror DOES NOT have a lastsync file
# rsync_exclude+=("--exclude=/lastsync")
rsync_cmd \
"${rsync_exclude[@]}" \
"${source_url}" \
"${target}"
# Uncomment below if your chosen mirror DOES NOT have a lastsync file
# date -u "+%s" > "$target/lastsync"
# Uncomment below if you've set up everything above correctly.
echo "Last sync was $(date -d @"$(cat ${target}/lastsync)" --rfc-3339=seconds -u)" > "${target}/info"
echo "Last update was $(date -d @"$(cat ${target}/lastupdate)" --rfc-3339=seconds -u)" >> "${target}/info"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment