Orchestration for dlang release building

release.sh

#!/usr/bin/env bash

set -ueo pipefail

set -x

ROOT="$PWD"
LATEST_D_VER=v$(curl -fsS http://downloads.dlang.org/releases/LATEST)
BUILD_LDC_VER=v1.32.0
D_VER=v2.109.0-beta.1
DUB_VER=v1.38.0-beta.1
[[ $DUB_VER =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-.*)?$ ]]
[[ $D_VER =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-.*)?$ ]]

if [ -z ${BASH_REMATCH[1]} ]; then
    BETA=0
    PRE=
else
    BETA=1
    BETA_SUFFIX=${BASH_REMATCH[1]:1} # e.g. beta.1 or rc.1
    PRE=pre-
fi
set +x

STEPS=(update merge_master dmd_version dub_version docarchives_version dlang.org tags tags_origin build sign test_install upload ddo_links ddo tags_upstream latest purge_changelog docarchives merge_stable upload_ftp docarchives_upload announce_beta announce)
# steps not run for betas
RELEASE_ONLY_STEPS=(purge_changelog announce)
FIRST_MAJOR_RELEASE_STEPS=(docarchives_version docarchives docarchives_upload)
FIRST_MAJOR_BETA_STEPS=(merge_master)
FIRST_BETA_STEPS=(announce_beta)

awsb2() {
    region=$(aws --profile ddo configure list | awk '$1 == "region" { print $2 }')
    aws --endpoint-url="https://s3.${region}.backblazeb2.com" $@
}

canFind() {
    local needle="$1"
    local haystack=("${@:2}")
    for e in "${haystack[@]}"; do
        if [ "$needle" = "$e" ]; then
            return 0
        fi
    done
    return 1
}

run_step() {
    local step=$1
    set -x
    cd "$ROOT"

    case $step in
        update)
            for proj in dmd phobos tools installer dlang.org dub downloads.dlang.org; do
                if ! test -d $proj; then
                    git clone --origin upstream git@github.com:dlang/$proj.git $proj
                    git -C $proj remote add origin git@github.com:ibuclaw/$proj.git
                fi
                git -C $proj fetch upstream
                if [ $proj != downloads.dlang.org ]; then
                    branch=stable
                else
                    branch=master
                fi
                git -C $proj checkout -B $branch upstream/$branch
            done
            ;;

        merge_master)
            for proj in dmd phobos tools installer dlang.org dub; do
                if ! git -C $proj merge upstream/master --no-ff; then
                    echo "==== Please resolve merge conflicts in '$proj', commit, and press Enter to continue ==="
                    read
                fi
                git -C $proj push upstream stable
            done
            ;;

        dmd_version)
            cd dmd
            echo $D_VER > VERSION
            git add VERSION
            git commit -m "bump VERSION to $D_VER"
            echo "==== Please review the version bump for 'dmd', and press Enter to continue ==="
            git diff upstream/stable..stable
            read
            git push upstream stable
            ;;

        dub_version)
            cd dub
            echo 'module dub.version_;' > source/dub/version_.d
            echo "enum dubVersion = \"$DUB_VER\";" >> source/dub/version_.d
            git add source/dub/version_.d
            git commit -m "update version to $DUB_VER"
            echo "==== Please review the version bump for 'dub', and press Enter to continue ==="
            git diff upstream/stable..stable
            read
            git push upstream stable
            ;;

        docarchives_version)
            [[ $D_VER =~ ^v[0-9]+\.([0-9]+)\.[0-9]+$ ]]
            D_MINOR_VER=${BASH_REMATCH[1]}
            cd dlang.org
            sed -i "s|var currentArchivedVersion = .*;|var currentArchivedVersion = ${D_MINOR_VER};|" js/listanchors.js
            git add js/listanchors.js
            git commit -m "bump current docarchives version to $D_VER"
            echo "==== Please review the version bump for 'docarchives.dlang.io', and press Enter to continue ==="
            git diff upstream/stable..stable
            read
            git push upstream stable
            ;;

        dlang.org)
            if [ -f dlang.org/changelog/${D_VER:1:7}_pre.dd ]; then
                DATE="$(LC_TIME=en_US.UTF-8 date -d "$(sed -n 's|.*VERSION \([A-Z][a-z][a-z][a-z]* [0-9][0-9]*, [0-9][0-9][0-9][0-9]\),.*|\1|p' dlang.org/changelog/${D_VER:1:7}_pre.dd)" +'%b %d, %Y')"
            else
                DATE="$(LC_TIME=en_US.UTF-8 date -d '+ 1 month' +'%b 01, %Y')"
            fi
            read -e -r -p "Planned release date: " -i "$DATE" response
            DATE=$(LC_TIME=en_US.UTF-8 date -d "$response" +'%b %d, %Y')

            cd tools
            if [ $BETA -eq 1 ]; then
                rdmd -version=Contributors_Lib changed $LATEST_D_VER..upstream/stable --version=${D_VER:1:7} --date="$DATE" --output=../dlang.org/changelog/${D_VER:1:7}_pre.dd
            else
                rdmd -version=Contributors_Lib changed $LATEST_D_VER..upstream/stable --version=${D_VER:1:7} --date="$DATE" --output=../dlang.org/changelog/${D_VER:1:7}.dd
            fi
            cd ../dlang.org
            if [ $BETA -eq 1 ]; then
                sed -i 's| BETA=$(COMMENT $0)| _=BETA=$(COMMENT $0)|; s|_=BETA=$0|BETA=$0|;' download.dd
                sed -i "s|B_DMDV2=.*|B_DMDV2=${D_VER:1:7}|; s|B_SUFFIX=.*|B_SUFFIX=$BETA_SUFFIX|" download.dd
                git add --force changelog/${D_VER:1:7}_pre.dd download.dd
            else
                sed -i 's|_=BETA=$(COMMENT $0)|BETA=$(COMMENT $0)|; s| BETA=$0| _=BETA=$0|;' download.dd
                echo ${D_VER:1:7} > VERSION
                git add VERSION changelog/${D_VER:1:7}.dd changelog/changelog.ddoc download.dd
                if [ -f changelog/${D_VER:1:7}_pre.dd ]; then
                    git rm changelog/${D_VER:1:7}_pre.dd
                fi
            fi
            git commit -m "update download and changelog for $D_VER"
            echo "==== Please review the changelog diff 'dlang.org', and press Enter to continue ==="
            git diff upstream/stable..stable
            read
            git push upstream stable
            ;;

        tags)
            for proj in dmd phobos tools installer dlang.org; do
                if ! git -C $proj tag --verify $D_VER &>/dev/null; then
                    git -C $proj tag -sm $D_VER $D_VER upstream/stable
                fi
            done
            if ! git -C dub tag --verify $DUB_VER &>/dev/null; then
                git -C dub tag -sm $DUB_VER $DUB_VER upstream/stable
            fi
            ;;

        tags_origin)
            for proj in dmd phobos tools installer dlang.org; do
                git -C $proj push -f origin $D_VER
            done
            git -C dub push -f origin $DUB_VER
            ;;

        build)
            cd installer/create_dmd_release
            rm -rf build

            sed -i '/\/dlang\/.*\/releases/! s|/dlang/|/ibuclaw/|' build_all.d
            rdmd build_all $BUILD_LDC_VER $D_VER
            ;;

        sign)
            cd installer/create_dmd_release
            for file in build/*; do
                gpg2 -b $file
            done
            ;;

        test_install)
            cd installer
            ./test/installation.sh ${D_VER:1}
            ;;

        upload)
            cd $ROOT/installer/create_dmd_release
            awsb2 --profile ddo s3 sync build/ s3://downloads-dlang-org/${PRE}releases/$(date '+%Y')/ --acl public-read --cache-control max-age=604800
            cd $ROOT/installer/script
            ncommits=$(git log --oneline $LATEST_D_VER..upstream/stable --no-merges -- install.sh | wc -l)
            if [ $ncommits -ne 0 ]; then
                ./deploy.sh
            fi
            cd $ROOT/dlang.org
            ncommits=$(git log --oneline $LATEST_D_VER..upstream/stable --no-merges -- d-keyring.gpg | wc -l)
            if [ $ncommits -ne 0 ]; then
                awsb2 --profile ddo s3 cp d-keyring.gpg s3://downloads-dlang-org/other/ --acl public-read --cache-control max-age=604800
                awsb2 --profile ddo s3 cp d-keyring.gpg.sig s3://downloads-dlang-org/other/ --acl public-read --cache-control max-age=604800
            fi
            ;;

        ddo_links)
            cd downloads.dlang.org
            git reset --hard
            sed -i 's|^makelink|#makelink|' make-links
            echo >> make-links
            ls ../installer/create_dmd_release/build | sed "s|^|makelink ${D_VER:1:7} ${PRE}releases/$(date '+%Y')/|" >> make-links
            ./make-links
            git add make-links
            git commit -m "$D_VER"
            git push upstream
            ;;

        ddo)
            cd downloads.dlang.org
            rm -rf ddo
            dub run build-gen-index -- -c s3_index -c generate
            awsb2 --profile ddo s3 sync ./ddo/ s3://downloads-dlang-org/ --acl public-read --cache-control max-age=604800
            echo '==== Check https://downloads.dlang.org ===='
            ;;

        tags_upstream)
            for proj in dmd phobos tools installer dlang.org; do
                git -C $proj push upstream $D_VER
            done
            git -C dub push upstream $DUB_VER
            ;;

        latest)
            echo -n ${D_VER:1} | awsb2 --profile ddo s3 cp - s3://downloads-dlang-org/${PRE}releases/LATEST --acl public-read
            if [ $BETA -eq 1 ]; then
                ssh digitalmars.com "echo -n ${D_VER:1} > /var/pub/digitalmars.com/LATEST_BETA"
            else
                ssh digitalmars.com "echo -n ${D_VER:1} > /var/pub/digitalmars.com/LATEST"
            fi
            ;;

        purge_changelog)
            for proj in dmd phobos tools installer dlang.org dub; do
                cd $proj
                git checkout stable
                git reset --hard upstream/stable
                [ $proj == dlang.org ] && local folder=language-changelog || local folder=changelog
                if git rm $folder/\*.dd; then
                    git commit -m 'purge changelog'
                    git push upstream stable
                fi
                cd ..
            done
            ;;

        docarchives)
            if ! test -d docarchives.dlang.io; then
                git clone --origin upstream git@github.com:dlang/docarchives.dlang.io.git docarchives.dlang.io
                git -C docarchives.dlang.io remote add origin git@github.com:ibuclaw/docarchives.dlang.io.git
            fi
            cd docarchives.dlang.io
            git fetch upstream
            git checkout -B master upstream/master
            git reset --hard
            docker run --rm -v $PWD:/build -it docarchives.dlang.io bash -e -c "source /root/dlang/*/activate; cd /build; ./builder.d ${D_VER}; chown -R $(id -u):$(id -g) archives"
            ;;

        merge_stable)
            for proj in dmd phobos tools installer dlang.org dub; do
                if git -C $proj rev-parse --verify origin/merge_stable; then # assume origin/merge_stable branch => pending PR
                    ncommits=$(git -C $proj log --oneline origin/merge_stable..upstream/stable --no-merges | wc -l)
                else
                    ncommits=$(git -C $proj log --oneline upstream/master..upstream/stable --no-merges | wc -l)
                fi
                if [ $ncommits -eq 0 ]; then
                    continue
                fi
                git -C $proj checkout -B merge_stable upstream/master
                if ! git -C $proj merge upstream/stable --no-ff; then
                    echo "==== Please resolve merge conflicts in '$proj', commit, and press Enter to continue ==="
                    read
                fi
                cd $proj
                gh pr create
                cd ..
            done
            ;;

        upload_ftp)
            cd installer/create_dmd_release
            rsync --progress --archive --no-perms --no-group --omit-dir-times --verbose --chmod=ug=rw build/ ibuclaw@digitalmars.com:/var/pub/digitalmars.com/
            ;;

        docarchives_upload)
            [[ $D_VER =~ ^v[0-9]+\.([0-9]+)\.[0-9]+$ ]]
            D_MINOR_VER=${BASH_REMATCH[1]}
            cd docarchives.dlang.io
            sed -i "/^<ul>/a<li><a href=\"./v2.${D_MINOR_VER}.0\">v2.${D_MINOR_VER}</a> (<a href=\"./v2.${D_MINOR_VER}.0/phobos\">Phobos</a>)</li>" archives/index.html
            sed -i "s|var currentArchivedVersion = .*;|var currentArchivedVersion = ${D_MINOR_VER};|" archives/*/js/listanchors.js
            echo "- [x] [${D_VER:1}](https://docarchives.dlang.io/${D_VER}) ([Phobos](https://docarchives.dlang.io/${D_VER}/phobos))" >> ARCHIVES.md
            git add archives/${D_VER} archives/index.html archives/*/js/listanchors.js ARCHIVES.md
            git commit -m "Add $D_VER"
            git push upstream master
            awsb2 --profile docarchives s3 sync ./archives/ s3://docarchives-dlang-io/ --acl public-read --cache-control max-age=604800
            ;;

        announce_beta)
            [ ${D_VER:7:1} = 0 ] && point_release='release' || point_release='point release'
            ncontributors=$(sed -n 's|.* \([0-9][0-9]*\) contributors.*|\1|p' dlang.org/changelog/${D_VER:1:7}_pre.dd)
            xclip -selection clipboard <<EOS
Glad to announce the first beta for the ${D_VER:1:7} $point_release, ♥ to the $ncontributors contributors.

http://dlang.org/download.html#dmd_beta
http://dlang.org/changelog/${D_VER:1:7}.html

As usual please report any bugs at
https://issues.dlang.org

-Iain
on behalf of the Dlang Core Team
EOS
            echo '==== Open the following link in your browser ===='
            echo "https://forum.dlang.org/newpost/announce?subject=Beta%20${D_VER:1:7}"
            ;;

        announce)
            ncontributors=$(sed -n 's|.* \([0-9][0-9]*\) contributors.*|\1|p' dlang.org/changelog/${D_VER:1:7}.dd)
            if [ ${D_VER:7:1} = 0 ]; then
                xclip -selection clipboard <<EOS
Glad to announce D ${D_VER:1}, ♥ to the $ncontributors contributors.

This release comes with...

http://dlang.org/download.html
http://dlang.org/changelog/${D_VER:1}.html

-Iain
on behalf of the Dlang Core Team
EOS
            else
                xclip -selection clipboard <<EOS
Glad to announce D ${D_VER:1}, ♥ to the $ncontributors contributors.

http://dlang.org/download.html

This point release fixes a few issues over ${LATEST_D_VER:1}, see the changelog for more details.

http://dlang.org/changelog/${D_VER:1}.html

-Iain
on behalf of the Dlang Core Team
EOS
            fi
            echo '==== Open the following link in your browser ===='
            echo "https://forum.dlang.org/newpost/announce?subject=Release%20D%20${D_VER:1}"
            ;;

        *)
            echo "Unknown step '$step'" 2>&1
            exit 1
            ;;
    esac

    set +x
}

# optional script argument to resume at named step
start_at=${1:-}
for step in "${STEPS[@]}"; do
    if [ ! -z $start_at ] && [ $step != $start_at ]; then
        continue
    fi
    start_at=
    if [ $BETA -eq 1 ] && canFind "$step" "${RELEASE_ONLY_STEPS[@]}"; then
        continue
    elif [[ $D_VER != *.0 ]] && canFind "$step" "${FIRST_MAJOR_RELEASE_STEPS[@]}"; then
        continue
    elif [[ $D_VER != *.0-beta.1 ]] && canFind "$step" "${FIRST_MAJOR_BETA_STEPS[@]}"; then
        continue
    elif [[ $D_VER != *-beta.1 ]] && canFind "$step" "${FIRST_BETA_STEPS[@]}"; then
        continue
    fi
    read -r -p "Run step $step? [y/N] " response
    if [[ ${response,,} =~ ^(yes|y)$ ]]; then
       run_step $step
    fi
done