first-buster-core-build.sh

#!/bin/bash -ex

# convenience functions & vars

# To force all steps to run, FORCE=y

logfile=~/$(date +%s)-buster-setup.log
unset old_pool_path old_release
export FAB_PATH=${FAB_PATH:-/turnkey/fab}
export SRC_PATH=${SRC_PATH:-/turnkey/public}
[[ -n "$RELEASE" ]] && old_release=$RELEASE
export RELEASE=debian/buster
export CODENAME=$(basename $RELEASE)
export CHANKO_PATH=${CHANKO_PATH:-$FAB_PATH/chankos/$CODENAME.chanko}
[[ -n "$POOL_PATH" ]] && old_pool_path=$POOL_PATH
export POOL_PATH=$FAB_PATH/pools/$CODENAME
export CORE_PATH=${CORE_PATH:-$FAB_PATH/products/core}

info_log() {
    echo "$@" | tee -a $logfile
}

checkout_or_clone() {
    repo_dir=$1
    repo=$(basename $repo_dir)
    branch=$2
    remote=$3
    [[ -z "$branch" ]] && branch=master
    [[ -z "$remote" ]] && remote=turnkeylinux
    url=https://github.com/${remote}/${repo}.git
    if [[ -d "$repo_dir" ]]; then
        cd $repo_dir
        git remote add $remote $url 2>/dev/null || true
        git checkout $branch
        git pull $remote $branch

    else
        repo_base=$(dirname $repo_dir)
        mkdir -p $repo_base
        cd $repo_base
        git clone $url -o $remote
        cd $repo_dir
        git checkout $branch

    fi
    info_log "#$repo_dir: $(git log --oneline -1)"
    cd -
}

update_file() {
    existing=$1
    updated=$2
    if [[ -f "$existing.old" ]]; then
        if [[ "$(shasum < $existing)" == "$(shasum < $updated)" ]]; then
            info_log "# $existing SHA matches $updated, skipping"
        else
            info_log "# $existing.old already exists, but $existing SHA"\
                     " does NOT match $updated SHA"
            exit 1
        fi
    else
        mv $existing $existing.old
        cp $updated $existing
        info_log "# $existing replaced by $updated"
    fi
}

add_to_pool() {
    repo_dir=$1
    branch=$2
    remote=$3
    checkout_or_clone $repo_dir $branch $remote
    cd $POOL_PATH
    pool-register $repo_dir 2>&1 | sed "s|error:||" \
        || true
}

# XXX
info_log "cloning repos initially needed..."
# common: buster-dev now merged into master
checkout_or_clone $FAB_PATH/common
# fab: buster-dev now merged into master
checkout_or_clone $SRC_PATH/fab
# turnkey-pylib: buster-dev has been merged into master
checkout_or_clone $SRC_PATH/turnkey-pylib
# turnkey-lazyclass: this is reqd in buildroot for new packages
checkout_or_clone $SRC_PATH/turnkey-lazyclass

# clone bootstrap & buildroot build repos
checkout_or_clone /turnkey/bootstrap
checkout_or_clone /turnkey/buildroot
# cdroots: turnkey master
checkout_or_clone $FAB_PATH/cdroots

# XXX
info_log "updating files..."
# update fab files from source (dirty hack)
src=$SRC_PATH/fab
update_file /usr/share/fab/product.mk $src/share/product.mk
update_file /usr/lib/fab/pylib/installer.py $src/pylib/installer.py
update_file /usr/share/fab/bootstrap/debootstrap.py $src/share/bootstrap/debootstrap.py

# update turnkey-pylib files from source (dirty hack)
# These hacks allow debinfo & tar to handle .xz control files in python2
src=$SRC_PATH/turnkey-pylib
update_file /usr/lib/python2.7/dist-packages/ar.py $src/pylib/ar.py
update_file /usr/lib/python2.7/dist-packages/debinfo.py $src/pylib/debinfo.py

# pull in 'turnkey' debootrap script from TKLDev v16.0
_file=usr/share/debootstrap/scripts/turnkey
_url=https://raw.githubusercontent.com/turnkeylinux-apps/tkldev/master/overlay/$_file
[[ -f "/$_file" ]] || wget -O /$_file $_url

# XXX
info_log "installing new dependencies..."
# new python dependency for pool (to handle xz files)
apt update -q
apt install python-lzma

# Debootstrap needs new version (from buster) to handle .xz files
_file=tmp/debootrap.deb
_url='http://deb.debian.org/debian/pool/main/d/debootstrap/debootstrap_1.0.114_all.deb'
[[ -f "/$_file" ]] || wget -O /$_file $_url
dpkg -i /$_file

# XXX
info_log "setting up buster chanko"
mkdir -p $CHANKO_PATH/{archives,config,plan} # fake archives used by pool
cat > $CHANKO_PATH/config/sources.list <<EOF
deb http://deb.debian.org/debian buster main
deb http://deb.debian.org/debian buster contrib
EOF
cp /turnkey/bootstrap/plan/* $CHANKO_PATH/plan/
cat > $CHANKO_PATH/config/chanko.conf <<EOF 
RELEASE=$RELEASE
PLAN_CPP=-DTURNKEY=y
EOF

apt install debian-archive-keyring
cp /usr/share/keyrings/debian-archive-keyring.gpg $CHANKO_PATH/config/trustedkeys.gpg
cd $CHANKO_PATH
chanko get -f plan/*

# XXX
info_log "setting up pool"
# setup temporary fake buster buildroot so pool doesn't whinge
mkdir -p $FAB_PATH/buildroots/$CODENAME
mkdir -p $POOL_PATH

cd $POOL_PATH
pool-init $FAB_PATH/buildroots/$CODENAME 2>/dev/null \
    || true
pool-register $CHANKO_PATH/archives 2>/dev/null \
    || true

# XXX
info_log "building bootstrap"
export FAB_POOL=y DEBOOTSTRAP_SUITE=turnkey
new_bootstrap=$FAB_PATH/bootstraps/$CODENAME
deb_ver=etc/debian_version
if [[ -z "$FORCE" ]] && [[ -f "$new_bootstrap/$deb_ver" ]]; then
    info_log "found bootstrap for Debian $(cat $new_bootstrap/$deb_ver), skipping"
else
    cd /turnkey/bootstrap
    make clean
    make
    # copy bootstrap to bootstraps folder
    rsync --delete -Hac build/bootstrap/ $new_bootstrap/
    info_log "SUCCESS - buster bootstrap is built"
fi

# XXX
info_log "building buildroot stage 1"
cd /turnkey/buildroot
pushd  $CHANKO_PATH
sed -i '/^turnkey-lazyclass/d' /turnkey/buildroot/plan/main
chanko-get -f /turnkey/buildroot/plan/main
# fixes some new virtual packages
chanko-get -f python3-all 
chanko-get -f python3-all-dev
chanko-get -f dh-systemd
chanko-get -f docbook-utils 
chanko-get -f texlive-formats-extra
chanko-get -f git
popd

built_buildroot=/turnkey/buildroot/build/root.patched
if [[ -z "$FORCE" ]] && [[ -f "$built_buildroot/$deb_ver" ]]; then
    info_log "found buildroot for Debian $(cat $built_buildroot/$deb_ver), skipping step 1"
else
    make clean
    make
    info_log "SUCCESS - buster buildroot stage 1 complete"
fi

# XXX
info_log "building buildroot stage 2"
cd /turnkey/public
final_buildroot=$FAB_PATH/buildroots/$CODENAME
if [[ -z "$FORCE" ]] && [[ -f "$final_buildroot/$deb_ver" ]]; then
    info_log "found final buildroot for Debian $(cat $final_buildroot/$deb_ver), skipping step 2"
else
    PKGS="turnkey-pylib pyproject autoversion turnkey-lazyclass"
    mkdir -p $built_buildroot/root/builddeps
    for pkg in $PKGS; do
        checkout_or_clone $SRC_PATH/$pkg
        cp -a $SRC_PATH/$pkg $built_buildroot/root/builddeps
    done

    fab-chroot $built_buildroot <<EOF

for p in $PKGS; do
    cd /root/builddeps/\$p
    build-deb
    dpkg -i ../\${p}*.deb || apt --fix-broken install
done
rm -rf /root/builddeps
EOF

    rsync --delete -Hac -v $built_buildroot/ $final_buildroot/
    info_log "SUCCESS - buster buildroot stage 2 complete"
fi

# XXX
info_log "building core"
fab-plan-resolve \
    -I $FAB_PATH/common/plans/ \
    $CORE_PATH/plan/main > /tmp/core-plan
# remove turnkey pkgs from temp plan
info_log "# removing these pkgs from temp core plan:"
pkg_to_remove="di-live inithooks confconsole hubdns"
for word in turnkey tklbam webmin; do
    pkgs=$(grep $word /tmp/core-plan | cut -d'#' -f1 | grep $word)
    pkg_to_remove="$pkg_to_remove $pkgs"
done
for pkg in $pkg_to_remove; do
    sed -i "/^$pkg / s|^.*$||" /tmp/core-plan
done
cd $CHANKO_PATH
chanko-get -f /tmp/core-plan
chanko-get -f \
    python3-debconf libparted-fs-resize0 \
    libio-pty-perl libauthen-pam-perl \
    python3-simplejson libdebian-installer4 \
    python-pycurl libsasl2-modules grub-pc \
    os-prober libparted2 python-simplejson \
    jitterentropy-rngd linux-image-amd64 \
    python-crypto python-dateutil ntpdate \
    librsync1 systemd-sysv

PKGS="autoversion confconsole di-live hubdns inithooks pycurl-wrapper pyproject
      tklbam-duplicity tklbam tklbam-python-boto tklbam-squid tklpatch
      turnkey-command turnkey-conffile turnkey-fifobuffer turnkey-fileevent
      turnkey-lazyclass turnkey-netinfo turnkey-paged turnkey-paths
      turnkey-pidlock turnkey-popen4 turnkey-pylib turnkey-stdtrap
      turnkey-sysinfo turnkey-temp webmin webmin-tklbam"

for pkg in $PKGS; do
    add_to_pool $SRC_PATH/$pkg
done

# the deckdebuild base chroot dir needs to be created manually
mkdir -p $FAB_PATH/deckdebuilds/chroots/

cd $FAB_PATH/products/core
make clean
make

info_log "SUCCESS - buster core complete"
info_log "Please find the iso here: $FAB_PATH/products/core/build/product.iso"
[[ -n "$old_pool_path" ]] && export POOL_PATH=$old_pool_path
[[ -n "$old_release" ]] && export RELEASE=$old_release