Skip to content

Instantly share code, notes, and snippets.

@dyfer
Last active October 20, 2023 07:52
Show Gist options
  • Save dyfer/6c83905d4593750105897e51e87ec345 to your computer and use it in GitHub Desktop.
Save dyfer/6c83905d4593750105897e51e87ec345 to your computer and use it in GitHub Desktop.
brew install universal
.DS_Store
brew-universal/*
# /bin/bash
# Copyright (c) 2023, Marcin Pączkowski
# All rights reserved.
# this script is designed to create "universal" x86_64 + arm64 binaries of homebrew packages
# to enable building universal binaries of applications depending on these libraries
# it is more of a hack or a workaround, than a production-ready code
# while it seems to work in some cases, it is rather fragile and doesn't handle errors like missing files etc well
# the checks whether to re-download a package are based only on the presence and SHA of the downloaded archive, not extracted/combined files
# if extracted/combined files are removed or altered, but the downloaded archives are kept, the script will fail
# PLEASE NOTE:
# this script overwrites packages in their original homebrew location
# you can use the combined libraries the same way you use any other homebrew libraries
# specified packages will be automatically installed in homebrew (before creating universal libraries), but only if not installed already
# the work directory can be cached for use with CI systems like GitHub Actions
# the combined binaries will be signed ad-hoc, unless signing is skipped (see "environment variables" below)
# usage:
# brew-install-universal.sh <packagename> <packagename> <packagename> ...
# environment variables:
# BREW_UNIVERSAL_WORKDIR: defines work directory for downloads, extracted files, and combined universal binaries (can be cached in CI); it will be created if it doesn't exist
# SKIP_SIGNING: if set to 1, skip ad-hoc signing of combined binaries; please note that this will result in combined binaries with invalid signatures, which will prevent them from loading
# SKIP_COPY: (for testing) if set to 1, skip copying of combined binaries to the homebrew location
# HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: this is a homebrew variable, but it might be a good idea to set it to 1, to prevent post-installation dependency updates from overwriting the combined libraries created with this script
# -----------------
# echo $(pwd)
# echo "$@" # all arguments
if [[ "$@" == "-h" ]] || [[ "$@" == "--help" ]] || [[ -z "$@" ]]; then
printf "usage:\nbrew-install-universal.sh <packagename> <packagename> <packagename> ...\n"
if [[ -z "$@" ]]; then
exit 1
else
exit 0
fi
fi
# OS checks
OS_NAME="`uname -s`"
if [[ "$OS_NAME" != "Darwin" ]]; then
echo "This script is intended to run on macOS (Darwin), but detected $OS_NAME instead"
exit 1
fi
BOTTLE_ID_ARCH_PREFIX_NATIVE=
BOTTLE_ID_ARCH_PREFIX_OTHER= # for the other architecture
BOTTLE_ID_SYSTEM_NAME=
BOTTLE_ID_NATIVE=
BOTTLE_ID_OTHER=
ARCH="`uname -m`" # arm64 or x86_64
# echo $ARCH
if [[ "$ARCH" == "x86_64" ]]; then
BOTTLE_ID_ARCH_PREFIX_NATIVE=""
BOTTLE_ID_ARCH_PREFIX_OTHER="arm64_"
elif [[ "$ARCH" == "arm64" ]]; then
BOTTLE_ID_ARCH_PREFIX_NATIVE="arm64_"
BOTTLE_ID_ARCH_PREFIX_OTHER=""
else
echo "Can't handle archtecture $ARCH"
exit 1
fi
DARWIN_VERSION_FULL=`uname -r` # 19: Catalina, 20: Big Sur, 21: Monterey, 22: Ventura
DARWIN_VERSION=${DARWIN_VERSION_FULL:0:2}
# echo $DARWIN_VERSION
declare VERSION_TO_OS_NAME=( ["18"]="mojave" ["19"]="catalina" ["20"]="big_sur" ["21"]="monterey" ["22"]="ventura" ["23"]="sonoma")
BOTTLE_ID_SYSTEM_NAME="${VERSION_TO_OS_NAME[$DARWIN_VERSION]}"
if [[ -z "$BOTTLE_ID_SYSTEM_NAME" ]]; then
echo "Can't map Darwin version $DARWIN_VERSION to macOS name"
exit 1
fi
if [[ DARWIN_VERSION < 20 ]]; then
echo "This script requires at least macOS 11 (big_sur), but detected $BOTTLE_ID_SYSTEM_NAME"
exit 1
fi
echo "Detected macOS $BOTTLE_ID_SYSTEM_NAME, architecture $ARCH"
BOTTLE_ID_NATIVE="$BOTTLE_ID_ARCH_PREFIX_NATIVE$BOTTLE_ID_SYSTEM_NAME"
BOTTLE_ID_OTHER="$BOTTLE_ID_ARCH_PREFIX_OTHER$BOTTLE_ID_SYSTEM_NAME"
# echo $BOTTLE_ID_NATIVE
# echo $BOTTLE_ID_OTHER
HOMEBREW_PREFIX=`brew --prefix`
# echo "homebrew prefix: "$HOMEBREW_PREFIX
HOMEBREW_CELLAR=`brew --cellar`
# echo "homebrew cellar: "$HOMEBREW_CELLAR
# select formulae path... are these correct?
if [[ $DARWIN_VERSION < 21 ]]; then
FORMULAE_PATH=$HOMEBREW_PREFIX"/Homebrew/Library/Taps/homebrew/homebrew-core/Formula" # big sur
else
FORMULAE_PATH=$HOMEBREW_PREFIX"/Library/Taps/homebrew/homebrew-core/Formula" # monterey
fi
# echo "FORMULAE_PATH: $FORMULAE_PATH"
# resolve $BREW_UNIVERSAL_WORKDIR
if [[ -z "$BREW_UNIVERSAL_WORKDIR" ]]; then
BREW_UNIVERSAL_WORKDIR=$(pwd)"/brew-universal"
echo "BREW_UNIVERSAL_WORKDIR not set, using $BREW_UNIVERSAL_WORKDIR"
else
if [ ! -d $BREW_UNIVERSAL_WORKDIR ]; then
mkdir -p $BREW_UNIVERSAL_WORKDIR
fi
BREW_UNIVERSAL_WORKDIR=`cd $BREW_UNIVERSAL_WORKDIR; pwd`
echo "BREW_UNIVERSAL_WORKDIR set: $BREW_UNIVERSAL_WORKDIR"
fi
DOWNLOAD_PATH="$BREW_UNIVERSAL_WORKDIR/downloads"
EXTRACTED_NATIVE_PATH="$BREW_UNIVERSAL_WORKDIR/extracted_native"
EXTRACTED_OTHER_PATH="$BREW_UNIVERSAL_WORKDIR/extracted_other"
COMBINED_PATH="$BREW_UNIVERSAL_WORKDIR/combined"
# create directories
mkdir -p "$DOWNLOAD_PATH"
mkdir -p "$EXTRACTED_NATIVE_PATH"
mkdir -p "$EXTRACTED_OTHER_PATH"
mkdir -p "$COMBINED_PATH"
# --- functions ---
brew_install_if_needed () { # takes name
# previously we only installed packages if the formula file was not present
# however, formula file might exist even if the package is not installed
# we'll always install the package if it's not installed, without checking for the formula file
brew list ${1} &>/dev/null || (echo "installing ${1}"; brew install ${1})
}
get_bottle_sha () { # takes name and OS_ID (e.g. "monterey or arm64_monterey")
# previously we were getting the sha by introspecting the formula file
# this was not returning sha's for packages that didn't provide platform-specific binaries, like ca-certificates
# currently, we are getting sha for such packages as well... should be ok? we just end up with no binaries to process
res=`brew info --json=v1 ${1} | grep -w "${2}" -A 3 | grep -w "sha256\":" | cut -d ":" -f2- | xargs | tr -d ","`
if [[ -z "$res" ]]; then
return 1
else
echo $res
fi
}
get_bottle_url () { # takes name and OS_ID (e.g. "monterey or arm64_monterey")
res=`brew info --json=v1 ${1} | grep -w "${2}" -A 3 | grep -w "url\":" | cut -d ":" -f2- | xargs | tr -d ","`
if [[ -z "$res" ]]; then
return 1
else
echo $res
fi
}
get_file_sha () { # takes full path
shasum -a 256 ${1} | awk '{ print $1 }'
}
download_bottle () { #takes name and OS_ID; returns 0 if the file was downloaded, 1 if the file was already there and sha matched, 2 if sha is not found for a given platform (e.g. if package is not architecture-specific)
name=$1
os_id=$2
formula_sha=$(get_bottle_sha $name $os_id)
# echo "formula_sha: $formula_sha"
if [[ -z "$formula_sha" ]]; then
echo "no bottle found for $os_id"
return 2
fi
destination=$DOWNLOAD_PATH
filename="${name}.${os_id}.bottle.tar.gz"
dest_full_path="${destination}/${filename}"
if [[ -f "$dest_full_path" ]]; then
file_sha=$(get_file_sha $dest_full_path)
if [[ $file_sha == $formula_sha ]]; then
# echo "sha matches: $file_sha"
echo "$filename already exists and SHA matches, not downloading"
return 1
else
echo "file sha $file_sha doesn't match sha from 'brew info' $formula_sha"
fi
fi
# if the file existed and sha matched, we won't get here
url=$(get_bottle_url $name $os_id)
echo "downloading ${filename} from ${url}"
curl -f -L -H "Authorization: Bearer QQ==" -o "$dest_full_path" ""$url""
return 0
# fi
}
decompress_bottle () { # takes name, OS_ID, destination_path
name=$1
os_id=$2
source=$DOWNLOAD_PATH
destination=$3
filename="${name}.${os_id}.bottle.tar.gz"
rm -rf "$destination/$name" # delete destination folder, in case we have previously extracted files there
echo "decompressing ${filename}"
tar xf "${source}/${filename}" --directory "$destination"
}
set_loader_paths () { # takes the full source path ${EXTRACTED_PATH}/${1}
find "${1}" -print0 | while IFS= read -r -d '' file
do
if [ -f "$file" ]; then
# "$@" "$file"
file_type_string=`file -bh "${file}"`
if [[ $file_type_string == *"shared library"* ]] || [[ $file_type_string == "Mach-O"*"executable"* ]]; then
loader_paths=`otool -XL "$file" | awk '{print $1 }'`
install_name=`otool -XD "$file"`
echo "--- ${file} ---"
basename=$(basename -- "$file")
# echo $basename
if [[ $install_name == *"@@HOMEBREW_PREFIX@@"* ]]; then
corrected_path=`echo "$install_name" | sed s~@@HOMEBREW_PREFIX@@~${HOMEBREW_PREFIX}~`
echo "${basename}: setting install name to ${corrected_path}"
install_name_tool -id "$corrected_path" "$file" 2>/dev/null
fi
if [[ $install_name == *"@@HOMEBREW_CELLAR@@"* ]]; then
corrected_path=`echo "$install_name" | sed s~@@HOMEBREW_CELLAR@@~${HOMEBREW_CELLAR}~`
echo "${basename}: setting install name to ${corrected_path}"
install_name_tool -id "$corrected_path" "$file" 2>/dev/null
fi
for this_path in $loader_paths
do
# echo "this_path:" "$this_path"
if [[ $this_path == *"@@HOMEBREW_PREFIX@@"* ]]; then
# echo $this_path
corrected_path=`echo "$this_path" | sed s~@@HOMEBREW_PREFIX@@~${HOMEBREW_PREFIX}~`
# echo $corrected_path
echo "${this_path}: changing path to ${corrected_path}"
install_name_tool -change "$this_path" "$corrected_path" "$file" 2>/dev/null
fi
if [[ $this_path == *"@@HOMEBREW_CELLAR@@"* ]]; then
# echo $this_path
corrected_path=`echo "$this_path" | sed s~@@HOMEBREW_CELLAR@@~${HOMEBREW_CELLAR}~`
# echo $corrected_path
echo "${this_path}: changing path to ${corrected_path}"
install_name_tool -change "$this_path" "$corrected_path" "$file" 2>/dev/null
fi
# also, try to replace the non-fully qualified paths, to avoid problems when using the libraries with macdeployqt
if [[ $this_path == *"@loader_path/../../../../opt"* ]]; then
# echo $this_path
corrected_path=`echo "$this_path" | sed s~@loader_path/../../../../opt~${HOMEBREW_PREFIX}/opt~`
# echo $corrected_path
echo "${this_path}: changing path to ${corrected_path}"
install_name_tool -change "$this_path" "$corrected_path" "$file" 2>/dev/null
fi
done
fi
# echo "file:" "$@" "$file"
fi
done
}
combine_libraries () { # takes name; saves them to ${COMBINED_PATH}/${name}/<version>
name=$1
version_dir=`ls -1 "${EXTRACTED_OTHER_PATH}/${name}/" | head -1` # we assume there's only one subdirectory
base_src_path="${EXTRACTED_OTHER_PATH}/${name}/${version_dir}"
base_native_path="${EXTRACTED_NATIVE_PATH}/${name}/${version_dir}"
base_combined_path="${COMBINED_PATH}/${name}/${version_dir}"
rm -rf "${COMBINED_PATH}/${name}/" # remove existing contents, in case there's an old version there already
find "$base_src_path" -print0 | while IFS= read -r -d '' file
do
# echo "file: $file"
if [ -f "$file" ] && [ ! -h "$file" ]; then
file_type_string=`file -bh "${file}"`
if [[ $file_type_string == *"shared library"* ]] || [[ $file_type_string == "Mach-O"*"executable"* ]]; then
echo "--- ${file} ---"
basename=$(basename -- "$file")
base_relative_path=${file##$base_src_path}
# echo "base_relative_path: $base_relative_path"
native_full_path="${base_native_path}${base_relative_path}"
# echo "native_full_path: $native_full_path"
combined_full_path="${base_combined_path}${base_relative_path}"
if [ -f "$native_full_path" ]; then # we don't handle a situation when this file is not present (possible reasons: deleted, issue with decompressing, discrepancy between packages, ?)
echo "combining ${file} and ${native_full_path}"
mkdir -p "$(dirname "${combined_full_path}")"
lipo "$file" "$native_full_path" -create -output "$combined_full_path"
if [[ $SKIP_SIGNING != 1 ]]; then
echo "signing ad-hoc ${basename}"
codesign -s - --force "$combined_full_path"
fi;
fi
fi
fi
done
}
copy_combined_libraries () { # takes name; returns 1 if the combined file doesn't exist
name=$1
base_installed_path=`brew --prefix ${name}`
version_dir=`ls -1 "${COMBINED_PATH}/${name}/" | head -1` # we assume there's only one subdirectory
base_combined_path="${COMBINED_PATH}/${name}/${version_dir}"
find "$base_combined_path" -print0 | while IFS= read -r -d '' file
do
# echo "file: $file"
if [ -f "$file" ] && [ ! -h "$file" ]; then
file_type_string=`file -bh "${file}"`
if [[ $file_type_string == *"shared library"* ]] || [[ $file_type_string == "Mach-O"*"executable"* ]]; then
echo "--- ${file} ---"
basename=$(basename -- "$file")
base_relative_path=${file##$base_combined_path}
installed_full_path="${base_installed_path}${base_relative_path}"
# echo "installed_full_path: $installed_full_path"
combined_full_path="${base_combined_path}${base_relative_path}"
# echo "combined_full_path: $combined_full_path"
if [ -f "$installed_full_path" ]; then
if [ -f "$combined_full_path" ]; then
echo "copying ${combined_full_path} to ${installed_full_path}"
cp -f ${combined_full_path} ${installed_full_path}
else
echo "combined file ${combined_full_path} doesn't exist!"
return 1
fi
fi
fi
fi
done
return 0
}
# --- process all ---
# colors
MSG_COL='\033[1;37m' # white
PKG_COL='\033[1;32m' # light green
WARN='\033[1;33m' # yellow
RES='\033[0m' # white
# sanitize provided package names (e.g. takes care of qt5 -> qt@5)
all_packages=
for package in "$@"
do
# select the line with package name, print second column, remove colon, strip color escape sequence
info=`brew info $package`
info_result=$?
if [[ $info_result == 0 ]]; then
sanitized=`echo "$info" | grep "==>" | head -1 | awk '{print $2}' | tr -d ':' | sed -E 's/[[:cntrl:]]\[([0-9]{1,3};)*[0-9]{1,3}m//g'`
# echo "package: $package, sanitized: $sanitized"
all_packages+=$sanitized
else
# echo "package not found: $package"
exit $info_result
fi
done
all_packages+=' '
# add all dependencies
for package in "$@"
do
deps=`brew deps $package | tr '\n' ' '`
# echo "deps: $deps"
all_packages+=$deps
done
# remove duplicates
all_packages_no_dup=`echo $all_packages | tr ' ' '\n' | sort | uniq | tr '\n' ' '`
# echo "all packages: $all_packages"
# echo "all all_packages_no_dup: $all_packages_no_dup"
echo "all packages to be processed: $all_packages_no_dup"
# download, combine, and copy libraries
for package in $all_packages_no_dup
do
printf "\n${MSG_COL}package: ${PKG_COL}${package}${RES}\n"
printf "${MSG_COL}${package}: downloading native package${RES}\n"
brew_install_if_needed $package # so that homebrew knows the package is installed
download_bottle $package $BOTTLE_ID_NATIVE
dl_result_native=$?
# echo "dl_result_native: $dl_result_native"
if [[ $dl_result_native == 0 ]]; then
printf "${MSG_COL}${package}: processing native package${RES}\n"
decompress_bottle $package $BOTTLE_ID_NATIVE $EXTRACTED_NATIVE_PATH
set_loader_paths "$EXTRACTED_NATIVE_PATH/$package"
elif [[ $dl_result_native == 2 ]]; then
printf "${WARN}${package}: package appears to not provide an architecture-specific bottle${RES}\n"
continue # go to the next package
else
printf "${WARN}${package}: package should be already extracted and processed${RES}\n"
fi
printf "${MSG_COL}${package}: downloading other arch package${RES}\n"
download_bottle $package $BOTTLE_ID_OTHER
dl_result_other=$?
# echo "dl_result_other: $dl_result_other"
if [[ $dl_result_other == 0 ]]; then
printf "${MSG_COL}${package}: processing other package${RES}\n"
decompress_bottle $package $BOTTLE_ID_OTHER $EXTRACTED_OTHER_PATH
set_loader_paths "$EXTRACTED_OTHER_PATH/$package"
else
printf "${WARN}${package}: package should be already extracted and processed${RES}\n"
fi
if [[ $dl_result_native == 0 ]] || [[ $dl_result_other == 0 ]]; then
printf "${MSG_COL}${package}: combining libraries${RES}\n"
combine_libraries $package
else
printf "${WARN}${package}: not combining binaries - both native and other packages were not modified${RES}\n"
fi
if [[ $SKIP_COPY != 1 ]]; then
printf "${MSG_COL}copying libraries${RES}\n"
copy_combined_libraries $package
else
printf "${WARN}${package}: skipping copy libraries${RES}\n"
fi
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment