Last active
October 20, 2023 07:52
-
-
Save dyfer/6c83905d4593750105897e51e87ec345 to your computer and use it in GitHub Desktop.
brew install universal
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
.DS_Store | |
brew-universal/* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# /bin/bash | |
# Copyright (c) 2023, Marcin Pączkowski | |
# All rights reserved. | |
# this script is designed to create "universal" x86_64 + arm64 binaries of homebrew packages | |
# to enable building universal binaries of applications depending on these libraries | |
# it is more of a hack or a workaround, than a production-ready code | |
# while it seems to work in some cases, it is rather fragile and doesn't handle errors like missing files etc well | |
# the checks whether to re-download a package are based only on the presence and SHA of the downloaded archive, not extracted/combined files | |
# if extracted/combined files are removed or altered, but the downloaded archives are kept, the script will fail | |
# PLEASE NOTE: | |
# this script overwrites packages in their original homebrew location | |
# you can use the combined libraries the same way you use any other homebrew libraries | |
# specified packages will be automatically installed in homebrew (before creating universal libraries), but only if not installed already | |
# the work directory can be cached for use with CI systems like GitHub Actions | |
# the combined binaries will be signed ad-hoc, unless signing is skipped (see "environment variables" below) | |
# usage: | |
# brew-install-universal.sh <packagename> <packagename> <packagename> ... | |
# environment variables: | |
# BREW_UNIVERSAL_WORKDIR: defines work directory for downloads, extracted files, and combined universal binaries (can be cached in CI); it will be created if it doesn't exist | |
# SKIP_SIGNING: if set to 1, skip ad-hoc signing of combined binaries; please note that this will result in combined binaries with invalid signatures, which will prevent them from loading | |
# SKIP_COPY: (for testing) if set to 1, skip copying of combined binaries to the homebrew location | |
# HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: this is a homebrew variable, but it might be a good idea to set it to 1, to prevent post-installation dependency updates from overwriting the combined libraries created with this script | |
# ----------------- | |
# echo $(pwd) | |
# echo "$@" # all arguments | |
if [[ "$@" == "-h" ]] || [[ "$@" == "--help" ]] || [[ -z "$@" ]]; then | |
printf "usage:\nbrew-install-universal.sh <packagename> <packagename> <packagename> ...\n" | |
if [[ -z "$@" ]]; then | |
exit 1 | |
else | |
exit 0 | |
fi | |
fi | |
# OS checks | |
OS_NAME="`uname -s`" | |
if [[ "$OS_NAME" != "Darwin" ]]; then | |
echo "This script is intended to run on macOS (Darwin), but detected $OS_NAME instead" | |
exit 1 | |
fi | |
BOTTLE_ID_ARCH_PREFIX_NATIVE= | |
BOTTLE_ID_ARCH_PREFIX_OTHER= # for the other architecture | |
BOTTLE_ID_SYSTEM_NAME= | |
BOTTLE_ID_NATIVE= | |
BOTTLE_ID_OTHER= | |
ARCH="`uname -m`" # arm64 or x86_64 | |
# echo $ARCH | |
if [[ "$ARCH" == "x86_64" ]]; then | |
BOTTLE_ID_ARCH_PREFIX_NATIVE="" | |
BOTTLE_ID_ARCH_PREFIX_OTHER="arm64_" | |
elif [[ "$ARCH" == "arm64" ]]; then | |
BOTTLE_ID_ARCH_PREFIX_NATIVE="arm64_" | |
BOTTLE_ID_ARCH_PREFIX_OTHER="" | |
else | |
echo "Can't handle archtecture $ARCH" | |
exit 1 | |
fi | |
DARWIN_VERSION_FULL=`uname -r` # 19: Catalina, 20: Big Sur, 21: Monterey, 22: Ventura | |
DARWIN_VERSION=${DARWIN_VERSION_FULL:0:2} | |
# echo $DARWIN_VERSION | |
declare VERSION_TO_OS_NAME=( ["18"]="mojave" ["19"]="catalina" ["20"]="big_sur" ["21"]="monterey" ["22"]="ventura" ["23"]="sonoma") | |
BOTTLE_ID_SYSTEM_NAME="${VERSION_TO_OS_NAME[$DARWIN_VERSION]}" | |
if [[ -z "$BOTTLE_ID_SYSTEM_NAME" ]]; then | |
echo "Can't map Darwin version $DARWIN_VERSION to macOS name" | |
exit 1 | |
fi | |
if [[ DARWIN_VERSION < 20 ]]; then | |
echo "This script requires at least macOS 11 (big_sur), but detected $BOTTLE_ID_SYSTEM_NAME" | |
exit 1 | |
fi | |
echo "Detected macOS $BOTTLE_ID_SYSTEM_NAME, architecture $ARCH" | |
BOTTLE_ID_NATIVE="$BOTTLE_ID_ARCH_PREFIX_NATIVE$BOTTLE_ID_SYSTEM_NAME" | |
BOTTLE_ID_OTHER="$BOTTLE_ID_ARCH_PREFIX_OTHER$BOTTLE_ID_SYSTEM_NAME" | |
# echo $BOTTLE_ID_NATIVE | |
# echo $BOTTLE_ID_OTHER | |
HOMEBREW_PREFIX=`brew --prefix` | |
# echo "homebrew prefix: "$HOMEBREW_PREFIX | |
HOMEBREW_CELLAR=`brew --cellar` | |
# echo "homebrew cellar: "$HOMEBREW_CELLAR | |
# select formulae path... are these correct? | |
if [[ $DARWIN_VERSION < 21 ]]; then | |
FORMULAE_PATH=$HOMEBREW_PREFIX"/Homebrew/Library/Taps/homebrew/homebrew-core/Formula" # big sur | |
else | |
FORMULAE_PATH=$HOMEBREW_PREFIX"/Library/Taps/homebrew/homebrew-core/Formula" # monterey | |
fi | |
# echo "FORMULAE_PATH: $FORMULAE_PATH" | |
# resolve $BREW_UNIVERSAL_WORKDIR | |
if [[ -z "$BREW_UNIVERSAL_WORKDIR" ]]; then | |
BREW_UNIVERSAL_WORKDIR=$(pwd)"/brew-universal" | |
echo "BREW_UNIVERSAL_WORKDIR not set, using $BREW_UNIVERSAL_WORKDIR" | |
else | |
if [ ! -d $BREW_UNIVERSAL_WORKDIR ]; then | |
mkdir -p $BREW_UNIVERSAL_WORKDIR | |
fi | |
BREW_UNIVERSAL_WORKDIR=`cd $BREW_UNIVERSAL_WORKDIR; pwd` | |
echo "BREW_UNIVERSAL_WORKDIR set: $BREW_UNIVERSAL_WORKDIR" | |
fi | |
DOWNLOAD_PATH="$BREW_UNIVERSAL_WORKDIR/downloads" | |
EXTRACTED_NATIVE_PATH="$BREW_UNIVERSAL_WORKDIR/extracted_native" | |
EXTRACTED_OTHER_PATH="$BREW_UNIVERSAL_WORKDIR/extracted_other" | |
COMBINED_PATH="$BREW_UNIVERSAL_WORKDIR/combined" | |
# create directories | |
mkdir -p "$DOWNLOAD_PATH" | |
mkdir -p "$EXTRACTED_NATIVE_PATH" | |
mkdir -p "$EXTRACTED_OTHER_PATH" | |
mkdir -p "$COMBINED_PATH" | |
# --- functions --- | |
brew_install_if_needed () { # takes name | |
# previously we only installed packages if the formula file was not present | |
# however, formula file might exist even if the package is not installed | |
# we'll always install the package if it's not installed, without checking for the formula file | |
brew list ${1} &>/dev/null || (echo "installing ${1}"; brew install ${1}) | |
} | |
get_bottle_sha () { # takes name and OS_ID (e.g. "monterey or arm64_monterey") | |
# previously we were getting the sha by introspecting the formula file | |
# this was not returning sha's for packages that didn't provide platform-specific binaries, like ca-certificates | |
# currently, we are getting sha for such packages as well... should be ok? we just end up with no binaries to process | |
res=`brew info --json=v1 ${1} | grep -w "${2}" -A 3 | grep -w "sha256\":" | cut -d ":" -f2- | xargs | tr -d ","` | |
if [[ -z "$res" ]]; then | |
return 1 | |
else | |
echo $res | |
fi | |
} | |
get_bottle_url () { # takes name and OS_ID (e.g. "monterey or arm64_monterey") | |
res=`brew info --json=v1 ${1} | grep -w "${2}" -A 3 | grep -w "url\":" | cut -d ":" -f2- | xargs | tr -d ","` | |
if [[ -z "$res" ]]; then | |
return 1 | |
else | |
echo $res | |
fi | |
} | |
get_file_sha () { # takes full path | |
shasum -a 256 ${1} | awk '{ print $1 }' | |
} | |
download_bottle () { #takes name and OS_ID; returns 0 if the file was downloaded, 1 if the file was already there and sha matched, 2 if sha is not found for a given platform (e.g. if package is not architecture-specific) | |
name=$1 | |
os_id=$2 | |
formula_sha=$(get_bottle_sha $name $os_id) | |
# echo "formula_sha: $formula_sha" | |
if [[ -z "$formula_sha" ]]; then | |
echo "no bottle found for $os_id" | |
return 2 | |
fi | |
destination=$DOWNLOAD_PATH | |
filename="${name}.${os_id}.bottle.tar.gz" | |
dest_full_path="${destination}/${filename}" | |
if [[ -f "$dest_full_path" ]]; then | |
file_sha=$(get_file_sha $dest_full_path) | |
if [[ $file_sha == $formula_sha ]]; then | |
# echo "sha matches: $file_sha" | |
echo "$filename already exists and SHA matches, not downloading" | |
return 1 | |
else | |
echo "file sha $file_sha doesn't match sha from 'brew info' $formula_sha" | |
fi | |
fi | |
# if the file existed and sha matched, we won't get here | |
url=$(get_bottle_url $name $os_id) | |
echo "downloading ${filename} from ${url}" | |
curl -f -L -H "Authorization: Bearer QQ==" -o "$dest_full_path" ""$url"" | |
return 0 | |
# fi | |
} | |
decompress_bottle () { # takes name, OS_ID, destination_path | |
name=$1 | |
os_id=$2 | |
source=$DOWNLOAD_PATH | |
destination=$3 | |
filename="${name}.${os_id}.bottle.tar.gz" | |
rm -rf "$destination/$name" # delete destination folder, in case we have previously extracted files there | |
echo "decompressing ${filename}" | |
tar xf "${source}/${filename}" --directory "$destination" | |
} | |
set_loader_paths () { # takes the full source path ${EXTRACTED_PATH}/${1} | |
find "${1}" -print0 | while IFS= read -r -d '' file | |
do | |
if [ -f "$file" ]; then | |
# "$@" "$file" | |
file_type_string=`file -bh "${file}"` | |
if [[ $file_type_string == *"shared library"* ]] || [[ $file_type_string == "Mach-O"*"executable"* ]]; then | |
loader_paths=`otool -XL "$file" | awk '{print $1 }'` | |
install_name=`otool -XD "$file"` | |
echo "--- ${file} ---" | |
basename=$(basename -- "$file") | |
# echo $basename | |
if [[ $install_name == *"@@HOMEBREW_PREFIX@@"* ]]; then | |
corrected_path=`echo "$install_name" | sed s~@@HOMEBREW_PREFIX@@~${HOMEBREW_PREFIX}~` | |
echo "${basename}: setting install name to ${corrected_path}" | |
install_name_tool -id "$corrected_path" "$file" 2>/dev/null | |
fi | |
if [[ $install_name == *"@@HOMEBREW_CELLAR@@"* ]]; then | |
corrected_path=`echo "$install_name" | sed s~@@HOMEBREW_CELLAR@@~${HOMEBREW_CELLAR}~` | |
echo "${basename}: setting install name to ${corrected_path}" | |
install_name_tool -id "$corrected_path" "$file" 2>/dev/null | |
fi | |
for this_path in $loader_paths | |
do | |
# echo "this_path:" "$this_path" | |
if [[ $this_path == *"@@HOMEBREW_PREFIX@@"* ]]; then | |
# echo $this_path | |
corrected_path=`echo "$this_path" | sed s~@@HOMEBREW_PREFIX@@~${HOMEBREW_PREFIX}~` | |
# echo $corrected_path | |
echo "${this_path}: changing path to ${corrected_path}" | |
install_name_tool -change "$this_path" "$corrected_path" "$file" 2>/dev/null | |
fi | |
if [[ $this_path == *"@@HOMEBREW_CELLAR@@"* ]]; then | |
# echo $this_path | |
corrected_path=`echo "$this_path" | sed s~@@HOMEBREW_CELLAR@@~${HOMEBREW_CELLAR}~` | |
# echo $corrected_path | |
echo "${this_path}: changing path to ${corrected_path}" | |
install_name_tool -change "$this_path" "$corrected_path" "$file" 2>/dev/null | |
fi | |
# also, try to replace the non-fully qualified paths, to avoid problems when using the libraries with macdeployqt | |
if [[ $this_path == *"@loader_path/../../../../opt"* ]]; then | |
# echo $this_path | |
corrected_path=`echo "$this_path" | sed s~@loader_path/../../../../opt~${HOMEBREW_PREFIX}/opt~` | |
# echo $corrected_path | |
echo "${this_path}: changing path to ${corrected_path}" | |
install_name_tool -change "$this_path" "$corrected_path" "$file" 2>/dev/null | |
fi | |
done | |
fi | |
# echo "file:" "$@" "$file" | |
fi | |
done | |
} | |
combine_libraries () { # takes name; saves them to ${COMBINED_PATH}/${name}/<version> | |
name=$1 | |
version_dir=`ls -1 "${EXTRACTED_OTHER_PATH}/${name}/" | head -1` # we assume there's only one subdirectory | |
base_src_path="${EXTRACTED_OTHER_PATH}/${name}/${version_dir}" | |
base_native_path="${EXTRACTED_NATIVE_PATH}/${name}/${version_dir}" | |
base_combined_path="${COMBINED_PATH}/${name}/${version_dir}" | |
rm -rf "${COMBINED_PATH}/${name}/" # remove existing contents, in case there's an old version there already | |
find "$base_src_path" -print0 | while IFS= read -r -d '' file | |
do | |
# echo "file: $file" | |
if [ -f "$file" ] && [ ! -h "$file" ]; then | |
file_type_string=`file -bh "${file}"` | |
if [[ $file_type_string == *"shared library"* ]] || [[ $file_type_string == "Mach-O"*"executable"* ]]; then | |
echo "--- ${file} ---" | |
basename=$(basename -- "$file") | |
base_relative_path=${file##$base_src_path} | |
# echo "base_relative_path: $base_relative_path" | |
native_full_path="${base_native_path}${base_relative_path}" | |
# echo "native_full_path: $native_full_path" | |
combined_full_path="${base_combined_path}${base_relative_path}" | |
if [ -f "$native_full_path" ]; then # we don't handle a situation when this file is not present (possible reasons: deleted, issue with decompressing, discrepancy between packages, ?) | |
echo "combining ${file} and ${native_full_path}" | |
mkdir -p "$(dirname "${combined_full_path}")" | |
lipo "$file" "$native_full_path" -create -output "$combined_full_path" | |
if [[ $SKIP_SIGNING != 1 ]]; then | |
echo "signing ad-hoc ${basename}" | |
codesign -s - --force "$combined_full_path" | |
fi; | |
fi | |
fi | |
fi | |
done | |
} | |
copy_combined_libraries () { # takes name; returns 1 if the combined file doesn't exist | |
name=$1 | |
base_installed_path=`brew --prefix ${name}` | |
version_dir=`ls -1 "${COMBINED_PATH}/${name}/" | head -1` # we assume there's only one subdirectory | |
base_combined_path="${COMBINED_PATH}/${name}/${version_dir}" | |
find "$base_combined_path" -print0 | while IFS= read -r -d '' file | |
do | |
# echo "file: $file" | |
if [ -f "$file" ] && [ ! -h "$file" ]; then | |
file_type_string=`file -bh "${file}"` | |
if [[ $file_type_string == *"shared library"* ]] || [[ $file_type_string == "Mach-O"*"executable"* ]]; then | |
echo "--- ${file} ---" | |
basename=$(basename -- "$file") | |
base_relative_path=${file##$base_combined_path} | |
installed_full_path="${base_installed_path}${base_relative_path}" | |
# echo "installed_full_path: $installed_full_path" | |
combined_full_path="${base_combined_path}${base_relative_path}" | |
# echo "combined_full_path: $combined_full_path" | |
if [ -f "$installed_full_path" ]; then | |
if [ -f "$combined_full_path" ]; then | |
echo "copying ${combined_full_path} to ${installed_full_path}" | |
cp -f ${combined_full_path} ${installed_full_path} | |
else | |
echo "combined file ${combined_full_path} doesn't exist!" | |
return 1 | |
fi | |
fi | |
fi | |
fi | |
done | |
return 0 | |
} | |
# --- process all --- | |
# colors | |
MSG_COL='\033[1;37m' # white | |
PKG_COL='\033[1;32m' # light green | |
WARN='\033[1;33m' # yellow | |
RES='\033[0m' # white | |
# sanitize provided package names (e.g. takes care of qt5 -> qt@5) | |
all_packages= | |
for package in "$@" | |
do | |
# select the line with package name, print second column, remove colon, strip color escape sequence | |
info=`brew info $package` | |
info_result=$? | |
if [[ $info_result == 0 ]]; then | |
sanitized=`echo "$info" | grep "==>" | head -1 | awk '{print $2}' | tr -d ':' | sed -E 's/[[:cntrl:]]\[([0-9]{1,3};)*[0-9]{1,3}m//g'` | |
# echo "package: $package, sanitized: $sanitized" | |
all_packages+=$sanitized | |
else | |
# echo "package not found: $package" | |
exit $info_result | |
fi | |
done | |
all_packages+=' ' | |
# add all dependencies | |
for package in "$@" | |
do | |
deps=`brew deps $package | tr '\n' ' '` | |
# echo "deps: $deps" | |
all_packages+=$deps | |
done | |
# remove duplicates | |
all_packages_no_dup=`echo $all_packages | tr ' ' '\n' | sort | uniq | tr '\n' ' '` | |
# echo "all packages: $all_packages" | |
# echo "all all_packages_no_dup: $all_packages_no_dup" | |
echo "all packages to be processed: $all_packages_no_dup" | |
# download, combine, and copy libraries | |
for package in $all_packages_no_dup | |
do | |
printf "\n${MSG_COL}package: ${PKG_COL}${package}${RES}\n" | |
printf "${MSG_COL}${package}: downloading native package${RES}\n" | |
brew_install_if_needed $package # so that homebrew knows the package is installed | |
download_bottle $package $BOTTLE_ID_NATIVE | |
dl_result_native=$? | |
# echo "dl_result_native: $dl_result_native" | |
if [[ $dl_result_native == 0 ]]; then | |
printf "${MSG_COL}${package}: processing native package${RES}\n" | |
decompress_bottle $package $BOTTLE_ID_NATIVE $EXTRACTED_NATIVE_PATH | |
set_loader_paths "$EXTRACTED_NATIVE_PATH/$package" | |
elif [[ $dl_result_native == 2 ]]; then | |
printf "${WARN}${package}: package appears to not provide an architecture-specific bottle${RES}\n" | |
continue # go to the next package | |
else | |
printf "${WARN}${package}: package should be already extracted and processed${RES}\n" | |
fi | |
printf "${MSG_COL}${package}: downloading other arch package${RES}\n" | |
download_bottle $package $BOTTLE_ID_OTHER | |
dl_result_other=$? | |
# echo "dl_result_other: $dl_result_other" | |
if [[ $dl_result_other == 0 ]]; then | |
printf "${MSG_COL}${package}: processing other package${RES}\n" | |
decompress_bottle $package $BOTTLE_ID_OTHER $EXTRACTED_OTHER_PATH | |
set_loader_paths "$EXTRACTED_OTHER_PATH/$package" | |
else | |
printf "${WARN}${package}: package should be already extracted and processed${RES}\n" | |
fi | |
if [[ $dl_result_native == 0 ]] || [[ $dl_result_other == 0 ]]; then | |
printf "${MSG_COL}${package}: combining libraries${RES}\n" | |
combine_libraries $package | |
else | |
printf "${WARN}${package}: not combining binaries - both native and other packages were not modified${RES}\n" | |
fi | |
if [[ $SKIP_COPY != 1 ]]; then | |
printf "${MSG_COL}copying libraries${RES}\n" | |
copy_combined_libraries $package | |
else | |
printf "${WARN}${package}: skipping copy libraries${RES}\n" | |
fi | |
done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment