Skip to content

Instantly share code, notes, and snippets.

@kamaln7
Created October 25, 2021 19:16
Show Gist options
  • Save kamaln7/ffeee723762e40290fa350ebd068b8dd to your computer and use it in GitHub Desktop.
Save kamaln7/ffeee723762e40290fa350ebd068b8dd to your computer and use it in GitHub Desktop.
#!/bin/bash
# usage: bin/compile <build-dir> <cache-dir> <env-dir>
set -eo pipefail
mkdir -p "$1" "$2"
build=$(cd "$1/" && pwd)
cache=$(cd "$2/" && pwd)
env_dir="${3}"
buildpack=$(cd "$(dirname $0)/.." && pwd)
source "${buildpack}/lib/common.sh"
source "${buildpack}/lib/common_tools.sh"
source_stdlib
snapshotBinBefore
DefaultGoVersion="$(<${DataJSON} jq -r '.Go.DefaultVersion')"
DepVersion="$(<${DataJSON} jq -r '.Dep.DefaultVersion')"
GlideVersion="$(<${DataJSON} jq -r '.Glide.DefaultVersion')"
GovendorVersion="$(<${DataJSON} jq -r '.Govendor.DefaultVersion')"
GBVersion="$(<${DataJSON} jq -r '.GB.DefaultVersion')"
PkgErrorsVersion="$(<${DataJSON} jq -r '.PkgErrors.DefaultVersion')"
MercurialVersion="$(<${DataJSON} jq -r '.HG.DefaultVersion')"
BazaarVersion="$(<${DataJSON} jq -r '.BZR.DefaultVersion')"
MattesMigrateVersion="$(<${DataJSON} jq -r '.MattesMigrate.DefaultVersion')"
GolangMigrateVersion="$(<${DataJSON} jq -r '.GolangMigrate.DefaultVersion')"
TQVersion="$(<${DataJSON} jq -r '.tq.DefaultVersion')"
# For specified versions of Go we need to keep concurrency.sh
needConcurrency() {
<"${DataJSON}" jq -e '.Go.NeedsConcurrency | any(. == "'${1}'")' &> /dev/null
}
handleDefaultPkgSpec() {
if [ "${pkgs}" = "default" ]; then
warn "Installing package '.' (default)"
warn ""
case "${TOOL}" in
gomodules)
warn "To install a different package spec add a comment in the following form to your \`go.mod\` file:"
warn "// +heroku install ./cmd/..."
warn ""
;;
dep)
warn "To install a different package spec set 'metadata.heroku.install' in 'Gopkg.toml'"
warn ""
warn "For more details see: https://devcenter.heroku.com/articles/go-apps-with-dep#build-configuration"
warn ""
;;
govendor)
warn "To install a different package spec set 'heroku.install' in 'vendor/vendor.json'"
warn ""
warn "For more details see: https://devcenter.heroku.com/articles/go-apps-with-govendor#build-configuration"
warn ""
;;
glide)
warn "To install a different package spec for the next build run:"
warn ""
warn "'heroku config:set GO_INSTALL_PACKAGE_SPEC=\"<pkg spec>\"'"
warn ""
warn "For more details see: https://devcenter.heroku.com/articles/go-dependencies-via-glide"
warn ""
;;
esac
pkgs="."
fi
}
massagePkgSpecForVendor() {
local t=""
for pkg in $(echo $pkgs); do
if [ "${pkg:0:1}" = "." ] || [ ! -d "./vendor/$pkg" ]; then
t+="${pkg} "
else
t+="${name}/vendor/${pkg} "
fi
done
pkgs="${t}"
}
# TODO: go1 is missing from storage.googleapis.com and we haven't manually compiled it yet
urlFor() {
local ver=$1
case $ver in
devel*)
local sha=$(echo ${ver} | cut -d - -f 2) #assumes devel-<short sha> or devel-<full sha>
echo https://github.com/golang/go/archive/$sha.tar.gz
;;
*)
local file="${ver}.linux-amd64.tar.gz"
echo ${BucketURL}/${file}
;;
esac
}
# Expand a version identifier to supported versions of Go. If the version
# identifier starts with one or more digits, then "go" is prepended before
# expansion. So `1.9` => `go1.9`, `1.9.4` => `go1.9.4`, `devel` => `devel`. See
# data.json for supported expansions. All others are returned as is without go
# prepended to it.
expandVer() {
local v="${1}"
if [[ "${v}" =~ ^[[:digit:]]+ ]]; then
v="go${v}"
fi
echo $(<${DataJSON} jq -r 'if .Go.VersionExpansion."'${v}'" then .Go.VersionExpansion."'${v}'" else "'${v}'" end')
}
# Report development versions to user
# Use after expandVer
reportVer() {
local ver="${1}"
mcount "version.go.$ver"
case $ver in
devel*)
warn ""
warn "You are using a development build of Go."
warn "This is provided for users requiring an unreleased Go version"
warn "but is otherwise unsupported."
warn ""
warn "Build tests are NOT RUN!!"
warn ""
;;
esac
}
ensureGlide() {
local glideVersion="${1}"
local gPath="${cache}/glide/${glideVersion}/bin"
local gBin="${gPath}/glide"
local gFile="glide-${glideVersion}-linux-amd64.tar.gz"
local targetFile="${gPath}/${gFile}"
if [ -x "${gBin}" ]; then
step "Using glide ${glideVersion}"
addToPATH "${gPath}"
else
rm -rf "${cache}/glide"
step "Installing glide ${glideVersion}"
ensureInPath "${gFile}" "${gPath}" "tar -C ${gPath} --strip-components=1 -zxf"
chmod a+x "${gBin}"
rm -f "${targetFile}"
fi
}
ensureMigrateTool() {
local t="${1}"
local mtVersion="${2}"
local tmp="$(mktemp -d)"
local mtPath="${cache}/${t}/${mtVersion}"
local targetFile="${build}/bin/migrate"
if [ -x "targetFile" ]; then
warning '"migrate" binary already exists in ~/bin, not installing '${t}
else
rm -rf "$(dirname "${mtPath}")"
mkdir -p "${mtPath}"
step "Installing ${t} ${mtVersion}"
ensureFile "migrate-${mtVersion}-linux-amd64.tar.gz" "${tmp}" "tar -C ${mtPath} -zxf"
mv "${mtPath}/migrate.linux-amd64" "${targetFile}"
chmod a+x "${targetFile}"
fi
}
wantAdditionalTool() {
local addt="${1}"
case "${TOOL}" in
govendor)
<${vendorJSON} jq -e '.heroku.additionalTools | contains(["'${addt}'"])' &> /dev/null
;;
dep)
<${depTOML} tq '$.metadata.heroku["additional-tools"]' | grep "^${addt}$" &> /dev/null
;;
*)
false
;;
esac
}
additionalToolVersion() {
local addt="${1}"
local default="${2}"
local tv=""
case "${TOOL}" in
govendor)
tv="$(<${vendorJSON} jq -r '.heroku.additionalTools | map(select(contains("'${addt}'@"))) | if .[0] then .[0] else "@'${default}'" end | split("@") | .[1]')"
;;
dep)
f=$(<${depTOML} tq '$.metadata.heroku["additional-tools"]' | grep ^${addt} | sed s:${addt}::)
f=${f:-"@${default}"}
tv="$(echo $f | cut -d @ -f 2)"
;;
esac
echo $tv
}
installGolangMigrateIfWanted() {
local t="github.com/golang-migrate/migrate"
if wantAdditionalTool $t; then
ensureMigrateTool $t $(additionalToolVersion $t ${GolangMigrateVersion})
fi
}
installMattesMigrateIfWanted() {
local t="github.com/mattes/migrate"
if wantAdditionalTool $t; then
warn ""
warn "Mattes Migrate is now Golang Migrate"
warn "Support for Mattes Migrate will be removed in the future"
warn "Please change the addition tool specification from 'github.com/mattes/migrate' to"
warn "'github.com/golang-migrate/migrate'"
warn ""
warn "Note: Supported Golang Migrate versions start at v3.4.0"
warn ""
ensureMigrateTool $t $(additionalToolVersion $t ${MattesMigrateVersion})
fi
}
ensureGB() {
local gbVersion="${1}"
GOPATH="${cache}/gb/${gbVersion}"
PATH="${GOPATH}/bin:${PATH}"
local pkgErrsFile="errors-${PkgErrorsVersion}.tar.gz"
local pkgErrorsPath="${GOPATH}/src/github.com/pkg/errors"
local gbFile="gb-${gbVersion}.tar.gz"
local gbPath="${GOPATH}/src/github.com/constabulary/gb"
if [ -d "${GOPATH}" ]; then
step "Using GB ${gbVersion}"
else
rm -rf "${cache}/gb/*"
ensureFile "${pkgErrsFile}" "${pkgErrorsPath}" "tar -C ${pkgErrorsPath} --strip-components=1 -zxf"
rm -f "${pkgErrorsPath}/${pkgErrsFile}"
ensureFile "${gbFile}" "${gbPath}" "tar -C ${gbPath} --strip-components=1 -zxf"
rm -f "${gbPath}/${gbFile}"
start "Installing GB v${GBVersion}"
pushd "${gbPath}" &> /dev/null
go install ./...
popd &> /dev/null
finished
fi
}
ensureGo() {
local goVersion="${1}"
local goPath="${cache}/${goVersion}/go"
local goFile=""
local txt="Installing ${goVersion}"
if [ -d "${goPath}" ]; then
step "Using ${goVersion}"
else
#For a go version change, we delete everything
step "New Go Version, clearing old cache"
if [ -d "${cache}/go-path" ]; then
find "${cache}/go-path" ! -perm -u=w -print0 | xargs -r -0 chmod u+w 2>&1
fi
rm -rf ${cache}/*
case "${goVersion}" in
devel*)
local bGoVersion="$(expandVer ${DefaultGoVersion})"
goFile="${bGoVersion}.linux-amd64.tar.gz"
goPath="${cache}/${bGoVersion}/go"
txt="Installing bootstrap ${bGoVersion}"
;;
go1)
goFile="go.go1.linux-amd64.tar.gz"
;;
*)
goFile="${goVersion}.linux-amd64.tar.gz"
;;
esac
step "${txt}"
ensureFile "${goFile}" "${goPath}" "tar -C ${goPath} --strip-components=1 -zxf"
rm -f "${goPath}/${goFile}"
case "${goVersion}" in
devel*)
pushd "${cache}" &> /dev/null
mkdir -p "${goVersion}"
pushd "${goVersion}" &> /dev/null
local sha=$(echo ${goVersion} | cut -d - -f 2) #assumes devel-<short sha> or devel-<full sha>
local url="https://github.com/golang/go/archive/$sha.tar.gz"
start "Downloading development Go version ${goVersion}"
${CURL} ${url} | tar zxf -
mv go-${sha}* go
finished
step "Compiling development Go version ${goVersion}"
pushd go/src &> /dev/null
echo "devel +${sha} $(date "+%a %b %H:%M:%S %G %z")"> ../VERSION
GOROOT_BOOTSTRAP=$(pushd ${cache}/${bGoVersion}/go > /dev/null; pwd; popd > /dev/null) ./make.bash 2>&1
popd &> /dev/null
go/bin/go version
rm -rf "${goPath}"
popd &> /dev/null
popd &> /dev/null
goPath="${cache}/${goVersion}/go"
;;
*)
;;
esac
fi
export GOROOT="${goPath}"
PATH="${goPath}/bin:${PATH}"
# Export GOCACHE if Go >= 1.10
if go env | grep -q '^GOCACHE='; then
export GOCACHE="${cache}/go-build-cache"
fi
}
setGoVersionFromEnvironment() {
if [ -z "${GOVERSION}" ]; then
warn ""
warn "'GOVERSION' isn't set, defaulting to '${DefaultGoVersion}'"
warn ""
warn "Run 'heroku config:set GOVERSION=goX.Y' to set the Go version to use"
warn "for future builds"
warn ""
fi
ver=${GOVERSION:-$DefaultGoVersion}
}
warnGoVersionOverride() {
if [ ! -z "${GOVERSION}" ]; then
warn "Using \$GOVERSION override."
warn " \$GOVERSION = ${GOVERSION}"
warn ""
warn "If this isn't what you want please run:'"
warn " heroku config:unset GOVERSION -a <app>"
warn ""
fi
}
warnPackageSpecOverride() {
if [ ! -z "${GO_INSTALL_PACKAGE_SPEC}" ]; then
warn "Using \$GO_INSTALL_PACKAGE_SPEC override."
warn " \$GO_INSTALL_PACKAGE_SPEC = ${GO_INSTALL_PACKAGE_SPEC}"
warn ""
warn "If this isn't what you want please run:'"
warn " heroku config:unset GO_INSTALL_PACKAGE_SPEC -a <app>"
warn ""
fi
}
# Sets up GOPATH (and posibly other GO* env vars) and returns the location of
# the source code as $src. The output of this function is meant to be eval'd'
setupGOPATH() {
local name="${1}"
local t="$(mktemp -d)"
if [ "${GO_SETUP_GOPATH_IN_IMAGE}" = "true" ]; then
mv -t ${t} ${build}/*
GOPATH="${build}"
else
cp -R ${build}/* ${t}
GOPATH="${t}/.go"
echo export GOBIN="${build}/bin"
fi
local src="${GOPATH}/src/${name}"
mkdir -p "${src}"
mkdir -p "${build}/bin"
mv -t "${src}" "${t}"/*
echo "GOPATH=${GOPATH}"
echo "src=${src}"
}
installPkgs() {
step "Running: go install -v ${FLAGS[@]} ${pkgs}"
go install -v "${FLAGS[@]}" ${pkgs} 2>&1
}
loadEnvDir "${env_dir}"
setGitCredHelper "${env_dir}"
trap clearGitCredHelper INT TERM EXIT
FLAGS=(-tags heroku)
determineTool
ver=$(expandVer $ver)
# If $GO_LINKER_SYMBOL and GO_LINKER_VALUE are set, tell the linker to DTRT
if [ -n "${GO_LINKER_SYMBOL}" -a -n "${GO_LINKER_VALUE}" ]; then
case "${ver}" in
go1|go1.0.*|go1.1|go1.1.*|go1.2|go1.2.*|go1.3|go1.3.*|go1.4|go1.4.*)
xval="${GO_LINKER_SYMBOL} ${GO_LINKER_VALUE}"
;;
*)
xval="${GO_LINKER_SYMBOL}=${GO_LINKER_VALUE}"
;;
esac
FLAGS+=(-ldflags "-X ${xval}")
fi
if [ -e "${build}/bin" -a ! -d "${build}/bin" ]; then
err ""
err "File bin exists and is not a directory."
err ""
exit 1
fi
reportVer "${ver}"
ensureGo "${ver}"
mkdir -p "${build}/bin"
export GOPATH
# GB installation
mcount "pkgmanagement.$TOOL"
mainPackagesInModule() {
# For an explanation of what this is doing, see https://dave.cheney.net/2014/09/14/go-list-your-swiss-army-knife
go list -find "${FLAGS[@]}" -f '{{ if eq .Name "main" }} {{.ImportPath}} {{ end }}' ./...
}
setupProcfile() {
if [ -f "${build}/Procfile" ]; then
return
fi
local pkgs=$(mainPackagesInModule)
if [ -z "${pkgs}" ]; then
return
fi
local pf=()
for pkg in ${pkgs}; do
local bn=$(basename ${pkg})
pf+=("${bn}: bin/${bn}")
done
if [ ${#pf} -eq 0 ]; then
return
fi
info ""
info "Created a Procfile with the following entries:"
if [ ${#pf[@]} -eq 1 ]; then
local line="web: bin/$(echo ${pf[0]} | cut -d / -f 2)"
echo "${line}" > ${build}/Procfile
info "\t\t${line}"
elif [ ${#pf[@]} -gt 1 ]; then
for pfl in "${pf[@]}"; do
echo "${pfl}" >> ${build}/Procfile
info "\t\t${pfl}"
done
fi
info ""
info "If these entries look incomplete or incorrect please create a Procfile with the required entries."
info "See https://devcenter.heroku.com/articles/procfile for more details about Procfiles"
info ""
}
case "${TOOL}" in
gomodules)
cd ${build}
export GOBIN="${build}/bin"
if [ "${GO_SETUP_GOPATH_FOR_MODULE_CACHE}" = "true" ]; then
export GOPATH="${build}/.heroku/go-path"
mkdir -p $GOPATH # ensure that it's created
else
export GOPATH="${cache}/go-path"
fi
# TODO: Check the desired language version (eg `go mod edit -json | jq -r '.Go'`)
# and only do this if it's <1.14. The 1.14 release and beyond handles this automatically
# when the desired language version is 1.14+ and vendoring should be used.
# https://golang.org/doc/go1.14#vendor
if [ -d "${build}/vendor" -a -s "${build}/go.sum" ]; then
FLAGS+=(-mod=vendor)
fi
set -x
step "Determining packages to install"
pkgs=${GO_INSTALL_PACKAGE_SPEC:-$(awk '{ if ($1 == "//" && $2 == "+heroku" && $3 == "install" ) { print substr($0, index($0,$4)); exit }}' ${goMOD})}
if [ -z "${pkgs}" ]; then
pkgs=$(mainPackagesInModule)
if [ -z "${pkgs}" ]; then
warn ""
warn "No main packages found in module ${name}"
warn ""
warn "Unsure what package(s) to install, defaulting to '.'"
warn ""
pkgs="default"
mcount "pkginstall.detect-failed"
else
info ""
info "Detected the following main packages to install:"
for pkg in ${pkgs}; do
info "\t\t${pkg}"
done
info ""
mcount "pkginstall.auto"
fi
else
if [ -z "${GO_INSTALL_PACKAGE_SPEC}" ]; then
mcount "pkginstall.go_install_package_spec"
else
mcount "pkginstall.heroku_install"
fi
fi
if [[ "${pkgs}" =~ "..." ]]; then
mcount "package_spec.contains.dot_dot_dot"
fi
warnPackageSpecOverride
handleDefaultPkgSpec
massagePkgSpecForVendor
unset GIT_DIR # unset git dir or it will mess with goinstall
if [[ -f bin/go-pre-compile ]]; then
step "Running bin/go-pre-compile hook"
chmod a+x bin/go-pre-compile
bin/go-pre-compile
fi
installPkgs
if [[ -f bin/go-post-compile ]]; then
step "Running bin/go-post-compile hook"
chmod a+x bin/go-post-compile
bin/go-post-compile
fi
;;
dep)
eval "$(setupGOPATH ${name})"
depTOML="${src}/Gopkg.toml"
pkgs=${GO_INSTALL_PACKAGE_SPEC:-$(<${depTOML} tq '$.metadata.heroku["install"]')}
if [ -z "${pkgs}" ]; then
pkgs="default"
fi
warnPackageSpecOverride
handleDefaultPkgSpec
unset GIT_DIR # unset git dir or it will mess with goinstall
cd "${src}"
if [ "$(< ${depTOML} tq '$.metadata.heroku["ensure"]')" != "false" ]; then
ensureInPath "dep-${DepVersion}-linux-amd64" "${cache}/dep/bin"
step "Fetching any unsaved dependencies (dep ensure)"
dep ensure
fi
massagePkgSpecForVendor
installPkgs
;;
godep)
eval "$(setupGOPATH ${name})"
godepsJSON="${src}/Godeps/Godeps.json"
pkgs=${GO_INSTALL_PACKAGE_SPEC:-$(<${godepsJSON} jq -r 'if .Packages then .Packages | join(" ") else "default" end')}
warnPackageSpecOverride
handleDefaultPkgSpec
UseGodepCommand="false" # Default to not wrapping go install with godep (vendor)
if [ -d "${src}/Godeps/_workspace/src" ]; then
UseGodepCommand="true"
if [ -d "${src}/vendor" ]; then
warn ""
warn "Godeps/_workspace/src and vendor/ exist"
warn "code may not compile. Please convert all deps to vendor/"
warn ""
fi
fi
# Warn that GO15VENDOREXPIERMENT is set, but the go version does not support it.
if ! <"${DataJSON}" jq -e '.Go.SupportsVendorExperiment | any(. == "'${ver}'")' &> /dev/null; then
if [ -n "${GO15VENDOREXPERIMENT}" ]; then
warn ""
warn "GO15VENDOREXPERIMENT is set, but is not supported by ${ver}"
warn "run \`heroku config:unset GO15VENDOREXPERIMENT\` to unset."
warn ""
fi
fi
unset GIT_DIR # unset git dir or it will mess with goinstall
cd "${src}"
if [ "${UseGodepCommand}" = "true" ]; then
ensureInPath "godep_linux_amd64" "${cache}/godep/bin"
step "Running: godep go install -v ${FLAGS[@]} ${pkgs}"
godep go install -v "${FLAGS[@]}" ${pkgs} 2>&1
else
massagePkgSpecForVendor
installPkgs
fi
;;
govendor)
ensureInPath "govendor_linux_amd64" "${cache}/govendor/bin"
eval "$(setupGOPATH ${name})"
vendorJSON="${src}/vendor/vendor.json"
pkgs=${GO_INSTALL_PACKAGE_SPEC:-$(<${vendorJSON} jq -r 'if .heroku.install then .heroku.install | join(" ") else "default" end')}
warnPackageSpecOverride
handleDefaultPkgSpec
unset GIT_DIR # unset git dir or it will mess with goinstall
cd "${src}"
if [ "$(<${vendorJSON} jq -r '.heroku.sync')" != "false" ]; then
step "Fetching any unsaved dependencies (govendor sync)"
govendor sync
fi
massagePkgSpecForVendor
installPkgs
;;
glide)
ensureGlide "${GlideVersion}"
# Do this before setupGOPATH as we need ${name} set first
cd "${build}"
name=$(glide name 2>/dev/null)
eval "$(setupGOPATH ${name})"
pkgs=${GO_INSTALL_PACKAGE_SPEC:-"default"}
handleDefaultPkgSpec
unset GIT_DIR
cd "${src}"
if [[ "${GLIDE_SKIP_INSTALL}" != "true" ]]; then
step "Fetching any unsaved dependencies (glide install)"
glide install 2>&1
fi
massagePkgSpecForVendor
installPkgs
;;
gb)
ensureGB "${GBVersion}"
cd $build
step "Running: gb build ${FLAGS[@]}"
gb build "${FLAGS[@]}" 2>&1
step "Post Compile Cleanup"
for f in bin/*-heroku; do
mv "$f" "${f/-heroku}"
done
rm -rf pkg
;;
esac
installGolangMigrateIfWanted
installMattesMigrateIfWanted
_newOrUpdatedBinFiles=$(binDiff)
info ""
info "Installed the following binaries:"
for fyle in ${_newOrUpdatedBinFiles}; do
info "\t\t${fyle}"
done
if [ -n "${src}" -a "${src}" != "${build}" -a -e "${src}/Procfile" ]; then
mv -t "${build}" "${src}/Procfile"
fi
if [ "${TOOL}" = "gomodules" ]; then
setupProcfile
fi
if [ ! -e $build/Procfile -a -n "${name}" ]; then
echo -e "web: $(basename $name)" >> $build/Procfile
fi
cd $build
mkdir -p $build/.profile.d
echo 'PATH=$PATH:$HOME/bin' > $build/.profile.d/go.sh
if [ "${GO_INSTALL_TOOLS_IN_IMAGE}" = "true" ]; then
start "Copying go tool chain to \$GOROOT=\$HOME/.heroku/go"
mkdir -p "${build}/.heroku/go"
cp -a "${GOROOT}/"* "${build}/.heroku/go"
echo 'export GOROOT=$HOME/.heroku/go' > "${build}/.profile.d/goroot.sh"
echo 'PATH=$PATH:$GOROOT/bin' >> "${build}/.profile.d/goroot.sh"
finished
if which ${TOOL} &> /dev/null; then
step "Copying ${TOOL} binary"
cp $(which ${TOOL}) "${build}/bin"
fi
fi
if [ "${GO_SETUP_GOPATH_IN_IMAGE}" = "true" ]; then
start "Cleaning up \$GOPATH/pkg"
rm -rf $GOPATH/pkg
finished
echo 'export GOPATH=$HOME' > "${build}/.profile.d/zzgopath.sh" #Try to make sure it's down in towards the end
echo 'cd $GOPATH/src/'${name} >> "${build}/.profile.d/zzgopath.sh" # because of this
fi
if needConcurrency ${ver}; then
cp $buildpack/vendor/concurrency.sh $build/.profile.d/
fi
t="${build}/.heroku/go"
mkdir -p "${t}"
t="${t}/.meta"
echo "TOOL=${TOOL}" > "${t}"
if [ "${TOOL}" != "gb" ]; then
echo "NAME=${name}" >> "${t}"
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment