Last active
August 23, 2018 15:53
-
-
Save krnowak/3c854e94245e2f33a8366e629bfb09c8 to your computer and use it in GitHub Desktop.
A script for building a bunch of habitat binaries, so I can get a studio with my modified habitat binaries.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# How to build a docker image of some package that contains custom | |
# built habitat binaries: | |
# | |
# 1. Follow this tutorial to make sure that you have habitat installed | |
# and set up - https://www.habitat.sh/tutorials/download/. Follow | |
# all the steps (means downloading the CLI, creating habitat | |
# account, setting up origin and configuring your workstation). | |
# | |
# 2. Clone habitat repository. | |
# | |
# 3. Ensure that you are root. | |
# | |
# 4. Put this script in parent directory of the habitat git repo | |
# directory. Or just put it anywhere and use --source-repo option | |
# to point the script to your clone of the the habitat git | |
# repository. The script will create a shallow clone of the git | |
# repo, so the script won't change anything in the original one | |
# (so, there will be no files with root as an owner). | |
# | |
# 5. Run the script. You can run it with --help flag to get some help | |
# about how to use the script. You will most likely need to use the | |
# --origin flag, because it defaults to author's one. | |
# | |
# 6. After habitat was built and uploaded to your depot (which may | |
# take a while), install it with "hab pkg install <ORIGIN>/hab". | |
# | |
# 7. Use "hab pkg binlink --dest <DIR> <ORIGIN>/hab hab" to add a | |
# symlink to just-built-and-installed hab binary in your the | |
# directory. Make sure that <DIR> is in your path. Run "hab --help" | |
# to make sure that you are running the custom built hab. | |
# | |
# 8. Call "hab pkg export docker <package>". | |
if [[ -n "${BMH_DEBUG}" ]]; then | |
set -x | |
fi | |
set -e | |
if ! which hab >/dev/null 2>&1; then | |
echo "hab binary not in PATH" | |
exit 1 | |
fi | |
d=$(dirname $0) | |
bmhd="${d}/BMH" | |
# TODO: maybe write a script that generates a default list of | |
# components to be built by this script, not sure if possible given | |
# that there are some deps between components, so they need to be | |
# built in a certain order | |
# drop/ignore: | |
## hab-butterfly | |
## builder-api-client | |
## builder-depot-client | |
## butterfly | |
## butterfly-test | |
## common | |
## eventsrv-client | |
## eventsrv-protocol | |
## hab-butterfly | |
## launcher-client | |
## launcher-protocol | |
## pkg-tarize | |
## plan-build-ps1 | |
## rootless_studio | |
## sup-client | |
## sup-protocol | |
# got problems with building: | |
## eventsrv | |
## bintray-publish | |
reuse='no' | |
origin='krnowak' | |
components=('hab' 'sup' 'launcher' 'studio' 'plan-build' 'backline' 'pkg-aci' 'pkg-dockerize' 'pkg-mesosize' 'pkg-export-docker' 'pkg-export-kubernetes' 'pkg-export-tar' 'pkg-export-helm' 'pkg-cfize') | |
skip_build='no' | |
skip_upload='no' | |
studio_root="${bmhd}/studio" | |
source_repo="${d}/habitat" | |
work_repo="${bmhd}/work" | |
keep_work_repo='no' | |
built_branch='HEAD' | |
results_dir="${bmhd}/results" | |
name_prefix='' | |
resume_at='' | |
work_commit() { | |
local msg="${1}" | |
GIT_AUTHOR_NAME='BMH Script' \ | |
GIT_AUTHOR_EMAIL='bmh@example.com' \ | |
GIT_COMMITTER_NAME='BMH Script' \ | |
GIT_COMMITTER_EMAIL='bmh@example.com' \ | |
git -C "${work_repo}" commit \ | |
--message="${msg}" | |
} | |
escape_sed_replacement() { | |
echo "${1}" | sed -e 's/[\/&]/\\&/g' | |
} | |
create_work_repo() { | |
if [[ -e "${work_repo}" ]]; then | |
echo "Work repo already exists, replace it or remove it before running this script" >&2 | |
exit 1 | |
fi | |
if [[ -n $(git -C "${source_repo}" status --porcelain) ]]; then | |
echo "Unclean tree, commit your changes and make sure that there are no untracked files" >&2 | |
exit 1 | |
fi | |
local current_branch="${built_branch}" | |
if [[ -z "${current_branch}" ]] || [[ "${current_branch}" = 'HEAD' ]]; then | |
current_branch=$(git -C "${source_repo}" branch | grep '^\*' | sed 's/\* //') | |
fi | |
git clone \ | |
"--branch=${current_branch}" \ | |
--depth=1 \ | |
"file://$(realpath "${source_repo}")" \ | |
"${work_repo}" | |
} | |
fixup_work_repo() { | |
# This replaces "git rev-list master --count" with "echo <number>" | |
commits=$(git -C "${source_repo}" rev-list master --count) | |
escaped_commits=$(escape_sed_replacement "${commits}") | |
for file in $(git -C "${work_repo}" grep 'git rev-list master --count' | grep '^\(components\|support\)/' | cut -d: -f1 | sort -u | grep '\.sh$'); do | |
echo "Hardcoding the commit count in ${file}" | |
sed -i -e "s/git rev-list master --count/echo ${escaped_commits}/g" "${work_repo}/${file}" | |
git -C "${work_repo}" add "${file}" | |
done | |
work_commit "bmh: Hardcode version based on commits count" | |
# This replaces "core/hab" with "${origin}/${name_prefix}-hab" | |
local escaped_origin=$(escape_sed_replacement "${origin}") | |
local escaped_name_prefix='' | |
local pkg_name_var='\$pkg_name' | |
local potential_prog_name='' | |
local escaped_potential_prog_name='' | |
if [[ -n "${name_prefix}" ]]; then | |
escaped_name_prefix=$(escape_sed_replacement "${name_prefix}-") | |
for file in $(git -C "${work_repo}" ls-files | grep '^components/.*/plan.sh$' | grep -v '^components/[^/]\+/.*test' | sort -u); do | |
echo "Prefixing pkg_name in ${file}" | |
potential_prog_name=$(grep '^pkg_name=' "${work_repo}/${file}" | cut -d= -f2-) | |
escaped_potential_prog_name=$(escape_sed_replacement "${potential_prog_name}") | |
sed -i \ | |
-e "s/${pkg_name_var}/${escaped_potential_prog_name}/g" \ | |
-e "s/^\(pkg_name=\)/\1${escaped_name_prefix}/g" \ | |
-e "s/^\(pkg_origin=\)core/\1${escaped_origin}/g" \ | |
-e "s/\(pkg_path_for[[:space:]]\+\)\(hab\)/\1${escaped_name_prefix}\2/g" \ | |
"${work_repo}/${file}" | |
git -C "${work_repo}" add "${file}" | |
done | |
work_commit "bmh: Prefix habitat package names" | |
fi | |
for file in $(git -C "${work_repo}" grep core/hab | grep '^components/' | cut -d: -f1 | sort -u | grep '\.\(rs\|sh\)$'); do | |
echo "Making changes in ${file}" | |
sed -i -e "s/core\/hab/${escaped_origin}\/${escaped_name_prefix}hab/g" "${work_repo}/${file}" | |
git -C "${work_repo}" add "${file}" | |
done | |
work_commit "bmh: Rename habitat packages" | |
# This replaces build_type and builder_build_type to --debug in plans | |
for file in $(git -C "${work_repo}" grep 'build_type=".*"' | cut -d: -f1 | sort -u | grep '\.sh$'); do | |
echo "Changing build type in ${file} to --debug" | |
sed -i -e "s/\(build_type=\"\).*\(\"\)/\1--debug\2/g" "${work_repo}/${file}" | |
git -C "${work_repo}" add "${file}" | |
done | |
work_commit "bmh: Set build type to --debug" | |
} | |
parse_options() { | |
local default_built_branch="${built_branch}" | |
local default_components=$(local IFS=','; echo "${components[*]}") | |
local default_keep_work_repo="${keep_work_repo}" | |
local default_name_prefix="${name_prefix}" | |
local default_origin="${origin}" | |
local default_results_dir="${results_dir}" | |
local default_reuse="${reuse}" | |
local default_skip_build="${skip_build}" | |
local default_skip_upload="${skip_upload}" | |
local default_source_repo="${source_repo}" | |
local default_studio_root="${studio_root}" | |
local default_work_repo="${work_repo}" | |
local default_resume_at="${resume_at}" | |
while [[ -n "${1}" ]]; do | |
case "${1}" in | |
--built-branch) | |
built_branch="${2}" | |
shift 2 | |
;; | |
--components) | |
IFS=',' read -ra components <<< "${2}" | |
shift 2 | |
;; | |
--help) | |
cat <<EOF | |
Usage: $0 [FLAGS] | |
FLAGS: | |
--built-branch <BRANCH> - name of the branch to built, ignored if skipping building, default: ${default_built_branch} | |
--components <COMPONENTS> - a comma separated list of components to build in order, default: ${default_components} | |
--help - prints this message and quits | |
--keep-work-repo - do not delete the work repo after building, ignored if skipping building, default: ${default_keep_work_repo} | |
--name-prefix <PREFIX> - a string prepended to Habitat package name, if not empty the package will be named like <ORIGIN>/<PREFIX>-hab, default: ${default_name_prefix} | |
--origin <ORIGIN> - an origin override to use, default: ${default_origin} | |
--results-dir <DIR> - location where to put built artifacts, default: ${default_results_dir} | |
--resume-at <COMPONENT> - start building from the give component; uploads will happen for all of the components, though, default: ${default_rename_at} | |
--reuse - reuse an existing habitat studio instead of recreating it, ignored if skipping building, default: ${default_reuse} | |
--skip-build - do not build the components, jump to the upload part, default: ${default_skip_build} | |
--skip-upload - do not upload built artifacts, default: ${default_skip_upload} | |
--source-repo <DIR> - location of the habitat git repository, ignored if skipping building, default: ${default_source_repo} | |
--studio-root <DIR> - location of the habitat studio, ignored if skipping building, default: ${default_studio_root} | |
--work-repo <DIR> - location of the work clone of the habitat git repository, must not exist, ignored if skipping building, default: ${default_work_repo} | |
EOF | |
exit 0 | |
;; | |
--keep-work-repo) | |
keep_work_repo='yes' | |
shift | |
;; | |
--name-prefix) | |
name_prefix="${2}" | |
shift 2 | |
;; | |
--origin) | |
origin="${2}" | |
shift 2 | |
;; | |
--results-dir) | |
results_dir="${2}" | |
shift 2 | |
;; | |
--resume-at) | |
resume_at="${2}" | |
shift 2 | |
;; | |
--reuse) | |
reuse='yes' | |
shift | |
;; | |
--skip-build) | |
skip_build='yes' | |
shift | |
;; | |
--skip-upload) | |
skip_upload='yes' | |
shift | |
;; | |
--source-repo) | |
source_repo="${2}" | |
shift 2 | |
;; | |
--studio-root) | |
studio_root="${2}" | |
shift 2 | |
;; | |
--work-repo) | |
work_repo="${2}" | |
shift 2 | |
;; | |
*=*) | |
echo "--foo=bar flags are not supported, use --foo bar" | |
exit 1 | |
;; | |
*) | |
echo "unknown flag ${1}, use --help to get help" >&2 | |
exit 1 | |
;; | |
esac | |
done | |
} | |
get_pkg_name() { | |
local component="${1}" | |
local pkg_name="hab-${component}" | |
if [[ ${pkg_name} = 'hab-hab'* ]]; then | |
pkg_name="${component}" | |
fi | |
if [[ -n "${name_prefix}" ]]; then | |
echo "${name_prefix}-${pkg_name}" | |
else | |
echo "${pkg_name}" | |
fi | |
} | |
parse_options "${@}" | |
if [ -n "${resume_at}" ]; then | |
resume_at_found=0 | |
for c in "${components[@]}"; do | |
if [ "${resume_at}" = "${c}" ]; then | |
resume_at_found=1 | |
break | |
fi | |
done | |
if [ ${resume_at_found} -eq 0 ]; then | |
echo "Component '${resume_at}' to resume the build from is not found in components ${components[@]}" | |
exit 1 | |
fi | |
fi | |
cat <<EOF | |
Using origin: ${origin} | |
Building components: ${components[@]} | |
Built branch: ${built_branch} | |
Keeping work repository: ${keep_work_repo} | |
Name prefix: ${name_prefix} | |
Results dir: ${results_dir} | |
Resume build from: ${resume_at} | |
Reusing studio: ${reuse} | |
Skipping building: ${skip_build} | |
Skipping upload to depot: ${skip_upload} | |
Studio root: ${studio_root} | |
Source habitat git repository: ${source_repo} | |
Work git repository: ${work_repo} | |
EOF | |
if [[ "${skip_build}" = 'no' ]]; then | |
if [ -z "${resume_at}" ]; then | |
if [ -e 'results' ]; then | |
echo "You have results directory in working directory, move it elsewhere, as it is used by hab pkg build to put results into" >&2 | |
exit 1 | |
fi | |
if [ -e "${results_dir}" ]; then | |
echo "You have ${results_dir} directory, move it elsewhere" >&2 | |
exit 1 | |
fi | |
echo "Creating work repo" | |
create_work_repo | |
echo "Preparing the sources" | |
fixup_work_repo | |
if [[ "${reuse}" = 'no' ]]; then | |
echo "Recreating studio" | |
hab studio -r "${studio_root}" rm | |
hab studio -r "${studio_root}" new | |
fi | |
fi | |
build_resumed=0 | |
for c in "${components[@]}"; do | |
if [ -n "${resume_at}" ] && [ ${build_resumed} -eq 0 ]; then | |
if [ "${resume_at}" = "${c}" ]; then | |
build_resumed=1 | |
else | |
echo "Skipping building ${c}" | |
continue | |
fi | |
fi | |
echo "Building ${c}" | |
p="${work_repo}/components/${c}" | |
echo "Running hab pkg build for ${c}" | |
if ! hab pkg build --reuse --root "${studio_root}" --keys "${origin}" "${p}"; then | |
# A terrible hack for the reproducible failure I was getting: | |
# | |
# hab-studio: Creating Studio at /home/kv/projects/chef/habroot (default) | |
# hab-studio: Importing krnowak secret origin key | |
# » Importing origin key from standard input | |
# ★ Imported secret origin key krnowak-20170804100241. | |
# hab-studio: Building 'components/hab' in Studio at /home/kv/projects/chef/habroot (default) | |
# hab-studio: Exported: HAB_ORIGIN=krnowak | |
# Script started, file is /src/results/logs/unknown.2017-08-04-132104.log | |
# components/hab does not exist! | |
# unknown: Build time: 0m0s | |
# unknown: Exiting on error | |
# Script done, file is /src/results/logs/unknown.2017-08-04-132104.log | |
# | |
# Second run always works for some reason. | |
echo "Trying again." | |
hab pkg build --reuse --root "${studio_root}" --keys "${origin}" "${p}" | |
fi | |
mkdir "results/${c}" | |
mv "results/${origin}-$(get_pkg_name "${c}")"*'.hart' "results/${c}" | |
done | |
mv 'results' "${results_dir}" | |
fi | |
if [[ "${skip_upload}" = 'no' ]]; then | |
echo 'Uploading to depot' | |
for c in "${components[@]}"; do | |
echo "Running hab pkg upload for ${c} and promoting it to stable" | |
hart="${results_dir}/${c}/${origin}-*.hart" | |
hart=$(echo ${hart}) | |
hab pkg upload "${hart}" | |
hartname=$(basename "${hart}") | |
pkg_name=$(get_pkg_name "${c}") | |
release=$(echo "${hartname}" | sed 's/^'"${origin}-${pkg_name}"'-.*-\([[:digit:]]\{14\}\)-[^-]*-[^-]*.hart$/\1/') | |
version=$(echo "${hartname}" | sed 's/^'"${origin}-${pkg_name}"'-\(.*\)-'"${release}"'-[^-]*-[^-]*.hart$/\1/') | |
hab pkg promote "${origin}/${pkg_name}/${version}/${release}" stable | |
done | |
fi | |
if [[ "${skip_build}" = 'no' ]]; then | |
if [[ "${keep_work_repo}" = 'no' ]]; then | |
echo "Removing work repo" | |
rm -rf "${work_repo}" | |
fi | |
fi | |
echo "Done" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Also, here the option is actually
--source-repo
.