Created
October 16, 2018 17:40
-
-
Save timfoster/ea45c06baa4f6dbdd185757eb431a7fc to your computer and use it in GitHub Desktop.
doing imgadm upload and saving an artifacts manifest file
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
timf@iorangi-eth0 (master) git diff | |
diff --git a/tools/bits-upload.sh b/tools/bits-upload.sh | |
index 8f0d77c..ca14a9d 100755 | |
--- a/tools/bits-upload.sh | |
+++ b/tools/bits-upload.sh | |
@@ -57,6 +57,7 @@ function usage { | |
echo " -n <name> the name of component to upload" | |
echo " -N use NFS rather than manta to store artifacts" | |
echo " -t <timestamp> the timestamp (optional, derived otherwise)" | |
+ echo " -p publish bits to updates.joyent.com experimental" | |
echo "" | |
echo "Upload bits to Manta or NFS destination from \$BITS_DIR" | |
echo "which (defaults to <component>/bits)" | |
@@ -68,6 +69,12 @@ function usage { | |
exit 2 | |
} | |
+# | |
+# Maintain a global associative array mapping uploaded file basenames | |
+# to their corresponding Manta paths. This assumes basenames are unique. | |
+# | |
+declare -A STORED_MANTA_PATHS | |
+ | |
# | |
# Upload build artifacts to Manta. There's some duplication in the logic | |
# here and nfs_upload. | |
@@ -85,7 +92,6 @@ function manta_upload { | |
if [[ -z "$MANTA_USER" ]]; then | |
export MANTA_USER="Joyent_Dev"; | |
fi | |
- | |
if [[ ${UPLOAD_BASE_DIR:0:1} != '/' ]]; then | |
# if it starts with a / we assume it's /stor/<something> or | |
# /public/<something> if not, we prepend /stor | |
@@ -122,21 +128,40 @@ function manta_upload { | |
echo "Skipping upload." | |
fi | |
md5sums="${md5sums}${local_md5_line}\n" | |
+ | |
+ # save the file to our global assoc-array of {filename: manta path} | |
+ # used later when mapping manifests to image file URLs. | |
+ MANTA_PATHS[$(basename $file)]=${manta_object} | |
+ | |
+ # Store a full URL if it appears to be a public resource, otherwise | |
+ # just save the manta path. | |
+ echo $manta_object | grep -q /public/ | |
+ if [[ $? -eq 0 ]]; then | |
+ echo ${MANTA_URL}${manta_object} >> ${BITS_DIR}/artifacts.txt | |
+ else | |
+ echo $manta_object >> ${BITS_DIR}/artifacts.txt | |
+ fi | |
done | |
# upload the md5sums | |
echo -e $md5sums | \ | |
mput -v -H 'content-type: text/plain' ${MANTA_DESTDIR}/md5sums.txt | |
+ echo ${MANTA_DESTDIR}/md5sums.txt >> ${BITS_DIR}/artifacts.txt | |
# now update the branch latest link | |
echo "${MANTA_DESTDIR}" | \ | |
mput -v -H 'content-type: text/plain' \ | |
/${MANTA_USER}${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest | |
+ echo /${MANTA_USER}${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest >> \ | |
+ ${BITS_DIR}/artifacts.txt | |
+ | |
# If this is a bi-weekly release branch, also update latest-release link | |
if [[ $UPLOAD_BRANCH =~ ^release- ]]; then | |
echo "${MANTA_DESTDIR}" | \ | |
mput -v -H 'content-type: text/plain' \ | |
/${MANTA_USER}${UPLOAD_BASE_DIR}/latest-release | |
+ echo /${MANTA_USER}${UPLOAD_BASE_DIR}/latest-release >> \ | |
+ ${BITS_DIR}/artifacts.txt | |
fi | |
echo "Uploaded to ${MANTA_DESTDIR}" | |
@@ -151,12 +176,12 @@ function manta_upload { | |
function nfs_upload { | |
NFS_DESTDIR=${UPLOAD_BASE_DIR}/${UPLOAD_SUBDIR} | |
- for sub in $SUBS; do | |
+ for sub in ${SUBS}; do | |
mkdir -p ${NFS_DESTDIR}/${sub#${BITS_DIR}} | |
done | |
md5sums="" | |
- for file in $FILES; do | |
+ for file in ${FILES}; do | |
remote_object=${NFS_DESTDIR}/${file#$BITS_DIR} | |
local_md5_line=$(md5sum ${file}) | |
@@ -181,6 +206,7 @@ function nfs_upload { | |
echo "skipping upload." | |
fi | |
md5sums="${md5sums}${local_md5_line}\n" | |
+ echo $remote_object >> ${BITS_DIR}/artifacts.txt | |
done | |
# upload the md5sums | |
@@ -190,7 +216,7 @@ function nfs_upload { | |
if [[ -L ${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest ]]; then | |
unlink ${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest | |
fi | |
- (cd $UPLOAD_BASE_DIR ; ln -s ${UPLOAD_SUBDIR} ${UPLOAD_BRANCH}-latest) | |
+ (cd ${UPLOAD_BASE_DIR} ; ln -s ${UPLOAD_SUBDIR} ${UPLOAD_BRANCH}-latest) | |
# If this is a bi-weekly release branch, also update latest-release link | |
if [[ $UPLOAD_BRANCH =~ ^release- ]]; then | |
@@ -201,12 +227,15 @@ function nfs_upload { | |
fi | |
} | |
+# | |
+# Look for build artifacts to operate on. | |
+# | |
function find_upload_bits { | |
if [[ -z "$SUBDIRS" ]]; then | |
SUBS=$(find $BITS_DIR -type d) | |
FILES=$(find $BITS_DIR -type f) | |
else | |
- for subdir in $SUBDIRS; do | |
+ for subdir in ${SUBDIRS}; do | |
if [[ -d $BITS_DIR/$subdir ]]; then | |
SUBS="$SUBS $(find $BITS_DIR/$subdir -type d)" | |
FILES="$FILES $(find $BITS_DIR/$subdir -type f)" | |
@@ -215,6 +244,44 @@ function find_upload_bits { | |
fi | |
} | |
+# | |
+# Publish build artifacts to updates.joyent.com. | |
+# | |
+function publish_to_updates { | |
+ echo "Publishing updates to updates.joyent.com" | |
+ for file in ${FILES}; do | |
+ echo ${file} | grep -q '.imgmanifest$' | |
+ if [[ $? -ne 0 ]]; then | |
+ continue | |
+ fi | |
+ | |
+ MF=${file} | |
+ IMAGEFILE=$(echo ${file} | sed -e 's/.imgmanifest$/.zfs.gz/g') | |
+ | |
+ if [[ ! -f ${ZFSFILE} ]]; then | |
+ echo "Unable to determine ZFS image file for ${MF}." | |
+ echo "Not publishing to updates.joyent.com" | |
+ continue | |
+ fi | |
+ | |
+ UUID=$(json -f ${MF} uuid) | |
+ if [[ -z "${UUID}" ]]; then | |
+ echo "Unable to determine UUID of $MF." | |
+ echo "Not publishing to updates.joyent.com" | |
+ continue | |
+ fi | |
+ | |
+ # The default 1hr expiry for msign is sufficient, since we're going | |
+ # to be accessing this URL almost immediately. | |
+ MANTA_PATH=${MANTA_PATHS[$(basename $IMAGEFILE])} | |
+ MANTA_URL=$(msign $MANTA_PATH) | |
+ FILE_SHA=$(digest -a sha1 $IMAGEFILE) | |
+ echo updates-imageadm import -m $MF --channel experimental | |
+ echo updates-imgadm add-file --url -s $FILE_SHA -f $MANTA_URL $UUID | |
+ fi | |
+} | |
+ | |
+ | |
# | |
# Main | |
# | |
@@ -235,6 +302,9 @@ while getopts "B:b:d:n:Nt:" opt; do | |
n) | |
NAME=$OPTARG | |
;; | |
+ p) | |
+ PUBLISH_UPDATES=true | |
+ ;; | |
t) | |
TIMESTAMP=$OPTARG | |
;; | |
@@ -269,6 +339,12 @@ fi | |
start_time=$(date +%s) | |
+# we keep a file containing a list of uploads for this | |
+# session, useful to include as part of build artifacts. | |
+if [[ -f $BITS_DIR/artifacts.txt ]]; then | |
+ rm -f $BITS_DIR/artifacts.txt | |
+fi | |
+ | |
find_upload_bits | |
if [[ -z "$TIMESTAMP" ]]; then | |
@@ -284,11 +360,18 @@ fi | |
UPLOAD_SUBDIR=$TIMESTAMP | |
if [[ -n "$USE_NFS" ]]; then | |
+ if [[ -n "$PUBLISH_TO_UPDATES" ]]; then | |
+ fatal "-p requires that we are publishing to Manta" | |
+ fi | |
nfs_upload | |
else | |
manta_upload | |
fi | |
+if [[ -n "$PUBLISH_TO_UPDATES" ]]; then | |
+ publish_to_updates | |
+fi | |
+ | |
end_time=$(date +%s) | |
elapsed=$((${end_time} - ${start_time})) | |
if [[ -n "$USE_NFS" ]]; then | |
timf@iorangi-eth0 (master) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment