Created
December 16, 2022 15:22
-
-
Save Jason-CKY/7d7056ce261c6d606585f05218230037 to your computer and use it in GitHub Desktop.
Script to extract all kubeflow images given a kubeflow version
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# Usage: `KUBEFLOW_VERSION=1.6.1 bash ./extract.sh` | |
install_kustomize() | |
{ | |
# install_kustomize $TARGET_KUSTOMIZE_VERSION | |
echo "Installing Kustomize version v$1" | |
# Supported values of 'arch': amd64, arm64, ppc64le, s390x | |
case $(uname -m) in | |
x86_64) | |
arch=amd64 | |
;; | |
arm64|aarch64) | |
arch=arm64 | |
;; | |
ppc64le) | |
arch=ppc64le | |
;; | |
s390x) | |
arch=s390x | |
;; | |
*) | |
arch=amd64 | |
;; | |
esac | |
if [[ "$OSTYPE" == linux* ]]; then | |
opsys=linux | |
elif [[ "$OSTYPE" == darwin* ]]; then | |
opsys=darwin | |
fi | |
RELEASE_URL=https://github.com/kubernetes-sigs/kustomize/releases/download/v${1}/kustomize_${1}_${opsys}_${arch} | |
curl -sLO "$RELEASE_URL" | |
mv ./kustomize_${TARGET_KUSTOMIZE_VERSION}_${opsys}_${arch} /usr/local/bin/kustomize | |
chmod +x /usr/local/bin/kustomize | |
echo "kustomize installed to $(which kustomize)" | |
} | |
if [[ -z "${KUBEFLOW_VERSION}" ]]; then | |
echo "KUBEFLOW_VERSION environment variable not set, exiting." | |
exit 1 | |
fi | |
echo "Pulling kubeflow version: v$KUBEFLOW_VERSION" | |
wget -q https://github.com/kubeflow/manifests/archive/refs/tags/v$KUBEFLOW_VERSION.zip -O ./$KUBEFLOW_VERSION.zip | |
if [ $? -ne 0 ]; then | |
echo "Failed downloading kubeflow release version v$KUBEFLOW_VERSION" | |
rm ./$KUBEFLOW_VERSION.zip | |
exit 1 | |
fi | |
echo "Unzipping..." | |
unzip -q ./$KUBEFLOW_VERSION.zip | |
rm ./$KUBEFLOW_VERSION.zip | |
KUBEFLOW_DIRECTORY=./manifests-$KUBEFLOW_VERSION | |
KUSTOMIZE_URL=$(cat ./manifests-$KUBEFLOW_VERSION/README.md | grep "### Prerequisites" -A 10 | grep kustomize | grep -Eo "(http|https)://[a-zA-Z0-9./?=_%:-]*" | head -1) | |
TARGET_KUSTOMIZE_VERSION=$(echo $KUSTOMIZE_URL | grep -Eo "[0-9][0-9.]*") | |
which kustomize | |
if [ $? -ne 0 ]; then | |
echo "No kustomize detected, downloading kustomize version $TARGET_KUSTOMIZE_VERSION from $KUSTOMIZE_URL" | |
install_kustomize $TARGET_KUSTOMIZE_VERSION | |
else | |
CURRENT_KUSTOMIZE_VERSION=$(kustomize version | grep -Eo "[0-9]\.[0-9]\.[0-9]") | |
if [ $CURRENT_KUSTOMIZE_VERSION = $TARGET_KUSTOMIZE_VERSION ]; then | |
echo "Correct kustomize version installed, skipping installation..." | |
else | |
echo "Incorrect kustomize version installed, deleting current installation of kustomize..." | |
rm $(which kustomize) | |
install_kustomize $TARGET_KUSTOMIZE_VERSION | |
fi | |
fi | |
MANIFESTS_DIR=./manifests | |
IMAGE_TXT_FILE=./images.md | |
mkdir -p $MANIFESTS_DIR | |
echo "Building images..." | |
echo -e "# Images used by kubeflow\n" > $IMAGE_TXT_FILE | |
for component in cert-manager istio dex "oidc authservice" knative "kubeflow namespace" "kubeflow roles" "kubeflow istio resources" "kubeflow pipelines" kserve katib "central dashboard" "admission webhook" notebooks profiles volumes tensorboard "training operator" "user namespace" | |
do | |
echo "Kustomize building $component..." | |
echo -e "## $component\n" >> $IMAGE_TXT_FILE | |
component_without_spaces=$(echo $component | sed 's/ /-/g') | |
mkdir -p $MANIFESTS_DIR/$component_without_spaces | |
# Multi-line grep search from # $(component) all the way until the next # character | |
cat $KUBEFLOW_DIRECTORY/README.md | grep -iPzo "# $component[.*\na-zA-z0-9()@',:/| \-+\[\]]*#[.*\na-zA-z0-9()@',:/| \-+\[\]]*" | grep -a "kustomize build" | while read -r command | |
do | |
if [ `echo $command | grep -c "pns"` -gt 0 ] | |
then | |
echo "Skipping pns kubeflow pipelines build" | |
continue | |
fi | |
KUSTOMIZE_DIR=$(echo $command | cut -d "|" -f 1 | cut -d " " -f 3) | |
MANIFEST_NAME=$(echo $KUSTOMIZE_DIR | sed 's/\//_/g') | |
MANIFEST_FILEPATH=$MANIFESTS_DIR/$component_without_spaces/$MANIFEST_NAME.yaml | |
kustomize build $KUBEFLOW_DIRECTORY/$KUSTOMIZE_DIR > $MANIFEST_FILEPATH | |
echo -e "### $MANIFEST_FILEPATH\n" >> $IMAGE_TXT_FILE | |
cat $MANIFEST_FILEPATH | grep -Eo "(quay\.io|docker\.io|ghcr\.io|gcr\.io|mysql:|python:|kserve\/|mcr\.microsoft\.com|nvcr\.io|tensorflow\/|kubeflownotebookswg\/|kubeflow\/training-operator)(\/|\w+|-|\.)*(:|\w+|\.|-)*(@|\w+|:)*" | while read -r image | |
do | |
echo -e "* $image" >> $IMAGE_TXT_FILE | |
done | |
echo "" >> $IMAGE_TXT_FILE | |
done | |
echo "" >> $IMAGE_TXT_FILE | |
done | |
rm -rf $KUBEFLOW_DIRECTORY |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment