A script for building a bunch of habitat binaries, so I can get a studio with my modified habitat binaries.

build-my-hab.sh

#!/bin/bash

# How to build a docker image of some package that contains custom
# built habitat binaries:
#
# 1. Follow this tutorial to make sure that you have habitat installed
#    and set up - https://www.habitat.sh/tutorials/download/. Follow
#    all the steps (means downloading the CLI, creating habitat
#    account, setting up origin and configuring your workstation).
#
# 2. Clone habitat repository.
#
# 3. Ensure that you are root.
#
# 4. Put this script in parent directory of the habitat git repo
#    directory. Or just put it anywhere and use --source-repo option
#    to point the script to your clone of the the habitat git
#    repository. The script will create a shallow clone of the git
#    repo, so the script won't change anything in the original one
#    (so, there will be no files with root as an owner).
#
# 5. Run the script. You can run it with --help flag to get some help
#    about how to use the script. You will most likely need to use the
#    --origin flag, because it defaults to author's one.
#
# 6. After habitat was built and uploaded to your depot (which may
#    take a while), install it with "hab pkg install <ORIGIN>/hab".
#
# 7. Use "hab pkg binlink --dest <DIR> <ORIGIN>/hab hab" to add a
#    symlink to just-built-and-installed hab binary in your the
#    directory. Make sure that <DIR> is in your path. Run "hab --help"
#    to make sure that you are running the custom built hab.
#
# 8. Call "hab pkg export docker <package>".

if [[ -n "${BMH_DEBUG}" ]]; then
    set -x
fi
set -e

if ! which hab >/dev/null 2>&1; then
    echo "hab binary not in PATH"
    exit 1
fi

d=$(dirname $0)
bmhd="${d}/BMH"

# TODO: maybe write a script that generates a default list of
# components to be built by this script, not sure if possible given
# that there are some deps between components, so they need to be
# built in a certain order

# drop/ignore:
## hab-butterfly
## builder-api-client
## builder-depot-client
## butterfly
## butterfly-test
## common
## eventsrv-client
## eventsrv-protocol
## hab-butterfly
## launcher-client
## launcher-protocol
## pkg-tarize
## plan-build-ps1
## rootless_studio
## sup-client
## sup-protocol

# got problems with building:
## eventsrv
## bintray-publish

reuse='no'
origin='krnowak'
components=('hab' 'sup' 'launcher' 'studio' 'plan-build' 'backline' 'pkg-aci' 'pkg-dockerize' 'pkg-mesosize' 'pkg-export-docker' 'pkg-export-kubernetes' 'pkg-export-tar' 'pkg-export-helm' 'pkg-cfize')
skip_build='no'
skip_upload='no'
studio_root="${bmhd}/studio"
source_repo="${d}/habitat"
work_repo="${bmhd}/work"
keep_work_repo='no'
built_branch='HEAD'
results_dir="${bmhd}/results"
name_prefix=''
resume_at=''

work_commit() {
    local msg="${1}"
    GIT_AUTHOR_NAME='BMH Script' \
    GIT_AUTHOR_EMAIL='bmh@example.com' \
    GIT_COMMITTER_NAME='BMH Script' \
    GIT_COMMITTER_EMAIL='bmh@example.com' \
    git -C "${work_repo}" commit \
        --message="${msg}"
}

escape_sed_replacement() {
    echo "${1}" | sed -e 's/[\/&]/\\&/g'
}

create_work_repo() {
    if [[ -e "${work_repo}" ]]; then
        echo "Work repo already exists, replace it or remove it before running this script" >&2
        exit 1
    fi
    if [[ -n $(git -C "${source_repo}" status --porcelain) ]]; then
        echo "Unclean tree, commit your changes and make sure that there are no untracked files" >&2
        exit 1
    fi
    local current_branch="${built_branch}"
    if [[ -z "${current_branch}" ]] || [[ "${current_branch}" = 'HEAD' ]]; then
        current_branch=$(git -C "${source_repo}" branch | grep '^\*' | sed 's/\* //')
    fi
    git clone \
        "--branch=${current_branch}" \
        --depth=1 \
        "file://$(realpath "${source_repo}")" \
        "${work_repo}"
}

fixup_work_repo() {
    # This replaces "git rev-list master --count" with "echo <number>"
    commits=$(git -C "${source_repo}" rev-list master --count)
    escaped_commits=$(escape_sed_replacement "${commits}")
    for file in $(git -C "${work_repo}" grep 'git rev-list master --count' | grep '^\(components\|support\)/' | cut -d: -f1 | sort -u | grep '\.sh$'); do
        echo "Hardcoding the commit count in ${file}"
        sed -i -e "s/git rev-list master --count/echo ${escaped_commits}/g" "${work_repo}/${file}"
        git -C "${work_repo}" add "${file}"
    done
    work_commit "bmh: Hardcode version based on commits count"

    # This replaces "core/hab" with "${origin}/${name_prefix}-hab"
    local escaped_origin=$(escape_sed_replacement "${origin}")
    local escaped_name_prefix=''
    local pkg_name_var='\$pkg_name'
    local potential_prog_name=''
    local escaped_potential_prog_name=''
    if [[ -n "${name_prefix}" ]]; then
        escaped_name_prefix=$(escape_sed_replacement "${name_prefix}-")
        for file in $(git -C "${work_repo}" ls-files | grep '^components/.*/plan.sh$' | grep -v '^components/[^/]\+/.*test' | sort -u); do
            echo "Prefixing pkg_name in ${file}"
            potential_prog_name=$(grep '^pkg_name=' "${work_repo}/${file}" | cut -d= -f2-)
            escaped_potential_prog_name=$(escape_sed_replacement "${potential_prog_name}")
            sed -i \
                -e "s/${pkg_name_var}/${escaped_potential_prog_name}/g" \
                -e "s/^\(pkg_name=\)/\1${escaped_name_prefix}/g" \
                -e "s/^\(pkg_origin=\)core/\1${escaped_origin}/g" \
                -e "s/\(pkg_path_for[[:space:]]\+\)\(hab\)/\1${escaped_name_prefix}\2/g" \
                "${work_repo}/${file}"
            git -C "${work_repo}" add "${file}"
        done
        work_commit "bmh: Prefix habitat package names"
    fi
    for file in $(git -C "${work_repo}" grep core/hab | grep '^components/' | cut -d: -f1 | sort -u | grep '\.\(rs\|sh\)$'); do
        echo "Making changes in ${file}"
        sed -i -e "s/core\/hab/${escaped_origin}\/${escaped_name_prefix}hab/g" "${work_repo}/${file}"
        git -C "${work_repo}" add "${file}"
    done
    work_commit "bmh: Rename habitat packages"

    # This replaces build_type and builder_build_type to --debug in plans
    for file in $(git -C "${work_repo}" grep 'build_type=".*"' | cut -d: -f1 | sort -u | grep '\.sh$'); do
        echo "Changing build type in ${file} to --debug"
        sed -i -e "s/\(build_type=\"\).*\(\"\)/\1--debug\2/g" "${work_repo}/${file}"
        git -C "${work_repo}" add "${file}"
    done
    work_commit "bmh: Set build type to --debug"
}

parse_options() {
    local default_built_branch="${built_branch}"
    local default_components=$(local IFS=','; echo "${components[*]}")
    local default_keep_work_repo="${keep_work_repo}"
    local default_name_prefix="${name_prefix}"
    local default_origin="${origin}"
    local default_results_dir="${results_dir}"
    local default_reuse="${reuse}"
    local default_skip_build="${skip_build}"
    local default_skip_upload="${skip_upload}"
    local default_source_repo="${source_repo}"
    local default_studio_root="${studio_root}"
    local default_work_repo="${work_repo}"
    local default_resume_at="${resume_at}"

    while [[ -n "${1}" ]]; do
        case "${1}" in
            --built-branch)
                built_branch="${2}"
                shift 2
                ;;
            --components)
                IFS=',' read -ra components <<< "${2}"
                shift 2
                ;;
            --help)
                cat <<EOF
Usage: $0 [FLAGS]
FLAGS:
--built-branch <BRANCH> - name of the branch to built, ignored if skipping building, default: ${default_built_branch}
--components <COMPONENTS> - a comma separated list of components to build in order, default: ${default_components}
--help - prints this message and quits
--keep-work-repo - do not delete the work repo after building, ignored if skipping building, default: ${default_keep_work_repo}
--name-prefix <PREFIX> - a string prepended to Habitat package name, if not empty the package will be named like <ORIGIN>/<PREFIX>-hab, default: ${default_name_prefix}
--origin <ORIGIN> - an origin override to use, default: ${default_origin}
--results-dir <DIR> - location where to put built artifacts, default: ${default_results_dir}
--resume-at <COMPONENT> - start building from the give component; uploads will happen for all of the components, though, default: ${default_rename_at}
--reuse - reuse an existing habitat studio instead of recreating it, ignored if skipping building, default: ${default_reuse}
--skip-build - do not build the components, jump to the upload part, default: ${default_skip_build}
--skip-upload - do not upload built artifacts, default: ${default_skip_upload}
--source-repo <DIR> - location of the habitat git repository, ignored if skipping building, default: ${default_source_repo}
--studio-root <DIR> - location of the habitat studio, ignored if skipping building, default: ${default_studio_root}
--work-repo <DIR> - location of the work clone of the habitat git repository, must not exist, ignored if skipping building, default: ${default_work_repo}
EOF
                exit 0
                ;;
            --keep-work-repo)
                keep_work_repo='yes'
                shift
                ;;
            --name-prefix)
                name_prefix="${2}"
                shift 2
                ;;
            --origin)
                origin="${2}"
                shift 2
                ;;
            --results-dir)
                results_dir="${2}"
                shift 2
                ;;
            --resume-at)
                resume_at="${2}"
                shift 2
                ;;
            --reuse)
                reuse='yes'
                shift
                ;;
            --skip-build)
                skip_build='yes'
                shift
                ;;
            --skip-upload)
                skip_upload='yes'
                shift
                ;;
            --source-repo)
                source_repo="${2}"
                shift 2
                ;;
            --studio-root)
                studio_root="${2}"
                shift 2
                ;;
            --work-repo)
                work_repo="${2}"
                shift 2
                ;;
            *=*)
                echo "--foo=bar flags are not supported, use --foo bar"
                exit 1
                ;;
            *)
                echo "unknown flag ${1}, use --help to get help" >&2
                exit 1
                ;;
        esac
    done
}

get_pkg_name() {
    local component="${1}"
    local pkg_name="hab-${component}"
    if [[ ${pkg_name} = 'hab-hab'* ]]; then
        pkg_name="${component}"
    fi
    if [[ -n "${name_prefix}" ]]; then
        echo "${name_prefix}-${pkg_name}"
    else
        echo "${pkg_name}"
    fi
}

parse_options "${@}"

if [ -n "${resume_at}" ]; then
    resume_at_found=0
    for c in "${components[@]}"; do
        if [ "${resume_at}" = "${c}" ]; then
            resume_at_found=1
            break
        fi
    done
    if [ ${resume_at_found} -eq 0 ]; then
        echo "Component '${resume_at}' to resume the build from is not found in components ${components[@]}"
        exit 1
    fi
fi

cat <<EOF
Using origin: ${origin}
Building components: ${components[@]}
Built branch: ${built_branch}
Keeping work repository: ${keep_work_repo}
Name prefix: ${name_prefix}
Results dir: ${results_dir}
Resume build from: ${resume_at}
Reusing studio: ${reuse}
Skipping building: ${skip_build}
Skipping upload to depot: ${skip_upload}
Studio root: ${studio_root}
Source habitat git repository: ${source_repo}
Work git repository: ${work_repo}
EOF

if [[ "${skip_build}" = 'no' ]]; then
    if [ -z "${resume_at}" ]; then
        if [ -e 'results' ]; then
            echo "You have results directory in working directory, move it elsewhere, as it is used by hab pkg build to put results into" >&2
            exit 1
        fi

        if [ -e "${results_dir}" ]; then
            echo "You have ${results_dir} directory, move it elsewhere" >&2
            exit 1
        fi

        echo "Creating work repo"
        create_work_repo

        echo "Preparing the sources"
        fixup_work_repo

        if [[ "${reuse}" = 'no' ]]; then
            echo "Recreating studio"
            hab studio -r "${studio_root}" rm
            hab studio -r "${studio_root}" new
        fi
    fi

    build_resumed=0
    for c in "${components[@]}"; do
        if [ -n "${resume_at}" ] && [ ${build_resumed} -eq 0 ]; then
            if [ "${resume_at}" = "${c}" ]; then
                build_resumed=1
            else
                echo "Skipping building ${c}"
                continue
            fi
        fi
        echo "Building ${c}"
        p="${work_repo}/components/${c}"
        echo "Running hab pkg build for ${c}"
        if ! hab pkg build --reuse --root "${studio_root}" --keys "${origin}" "${p}"; then
            # A terrible hack for the reproducible failure I was getting:
            #
            #    hab-studio: Creating Studio at /home/kv/projects/chef/habroot (default)
            #    hab-studio: Importing krnowak secret origin key
            # » Importing origin key from standard input
            # ★ Imported secret origin key krnowak-20170804100241.
            #    hab-studio: Building 'components/hab' in Studio at /home/kv/projects/chef/habroot (default)
            #    hab-studio: Exported: HAB_ORIGIN=krnowak
            # Script started, file is /src/results/logs/unknown.2017-08-04-132104.log
            # components/hab does not exist!
            #    unknown: Build time: 0m0s
            #    unknown: Exiting on error
            # Script done, file is /src/results/logs/unknown.2017-08-04-132104.log
            #
            # Second run always works for some reason.
            echo "Trying again."
            hab pkg build --reuse --root "${studio_root}" --keys "${origin}" "${p}"
        fi
        mkdir "results/${c}"
        mv "results/${origin}-$(get_pkg_name "${c}")"*'.hart' "results/${c}"
    done

    mv 'results' "${results_dir}"
fi

if [[ "${skip_upload}" = 'no' ]]; then
    echo 'Uploading to depot'
    for c in "${components[@]}"; do
        echo "Running hab pkg upload for ${c} and promoting it to stable"
        hart="${results_dir}/${c}/${origin}-*.hart"
        hart=$(echo ${hart})
        hab pkg upload "${hart}"
        hartname=$(basename "${hart}")
        pkg_name=$(get_pkg_name "${c}")
        release=$(echo "${hartname}" | sed 's/^'"${origin}-${pkg_name}"'-.*-\([[:digit:]]\{14\}\)-[^-]*-[^-]*.hart$/\1/')
        version=$(echo "${hartname}" | sed 's/^'"${origin}-${pkg_name}"'-\(.*\)-'"${release}"'-[^-]*-[^-]*.hart$/\1/')
        hab pkg promote "${origin}/${pkg_name}/${version}/${release}" stable
    done
fi

if [[ "${skip_build}" = 'no' ]]; then
    if [[ "${keep_work_repo}" = 'no' ]]; then
        echo "Removing work repo"
        rm -rf "${work_repo}"
    fi
fi

echo "Done"

@krnowak I just had the script fail on me because bash on this VM is in /bin/bash.

Would you be open to considering using #!/usr/bin/env bash?

Also, here the option is actually --source-repo.

Addressed both issues.