doing imgadm upload and saving an artifacts manifest file

bits-upload.sh.patch

timf@iorangi-eth0 (master) git diff
diff --git a/tools/bits-upload.sh b/tools/bits-upload.sh
index 8f0d77c..ca14a9d 100755
--- a/tools/bits-upload.sh
+++ b/tools/bits-upload.sh
@@ -57,6 +57,7 @@ function usage {
     echo "  -n <name>           the name of component to upload"
     echo "  -N                  use NFS rather than manta to store artifacts"
     echo "  -t <timestamp>      the timestamp (optional, derived otherwise)"
+    echo "  -p                  publish bits to updates.joyent.com experimental"
     echo ""
     echo "Upload bits to Manta or NFS destination from \$BITS_DIR"
     echo "which (defaults to <component>/bits)"
@@ -68,6 +69,12 @@ function usage {
     exit 2
 }

+#
+# Maintain a global associative array mapping uploaded file basenames
+# to their corresponding Manta paths. This assumes basenames are unique.
+#
+declare -A STORED_MANTA_PATHS
+
 #
 # Upload build artifacts to Manta. There's some duplication in the logic
 # here and nfs_upload.
@@ -85,7 +92,6 @@ function manta_upload {
     if [[ -z "$MANTA_USER" ]]; then
         export MANTA_USER="Joyent_Dev";
     fi
-
     if [[ ${UPLOAD_BASE_DIR:0:1} != '/' ]]; then
         # if it starts with a / we assume it's /stor/<something> or
         # /public/<something> if not, we prepend /stor
@@ -122,21 +128,40 @@ function manta_upload {
             echo "Skipping upload."
         fi
         md5sums="${md5sums}${local_md5_line}\n"
+
+        # save the file to our global assoc-array of {filename: manta path}
+        # used later when mapping manifests to image file URLs.
+        MANTA_PATHS[$(basename $file)]=${manta_object}
+
+        # Store a full URL if it appears to be a public resource, otherwise
+        # just save the manta path.
+        echo $manta_object | grep -q /public/
+        if [[ $? -eq 0 ]]; then
+            echo ${MANTA_URL}${manta_object} >> ${BITS_DIR}/artifacts.txt
+        else
+            echo $manta_object >> ${BITS_DIR}/artifacts.txt
+        fi
     done

     # upload the md5sums
     echo -e $md5sums | \
         mput -v -H 'content-type: text/plain' ${MANTA_DESTDIR}/md5sums.txt
+        echo ${MANTA_DESTDIR}/md5sums.txt >> ${BITS_DIR}/artifacts.txt

     # now update the branch latest link
     echo "${MANTA_DESTDIR}" | \
         mput -v -H 'content-type: text/plain' \
             /${MANTA_USER}${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest
+    echo /${MANTA_USER}${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest >> \
+        ${BITS_DIR}/artifacts.txt
+
     # If this is a bi-weekly release branch, also update latest-release link
     if [[ $UPLOAD_BRANCH =~ ^release- ]]; then
         echo "${MANTA_DESTDIR}" | \
             mput -v -H 'content-type: text/plain' \
                 /${MANTA_USER}${UPLOAD_BASE_DIR}/latest-release
+        echo /${MANTA_USER}${UPLOAD_BASE_DIR}/latest-release >> \
+            ${BITS_DIR}/artifacts.txt
     fi

     echo "Uploaded to ${MANTA_DESTDIR}"
@@ -151,12 +176,12 @@ function manta_upload {
 function nfs_upload {

     NFS_DESTDIR=${UPLOAD_BASE_DIR}/${UPLOAD_SUBDIR}
-    for sub in $SUBS; do
+    for sub in ${SUBS}; do
         mkdir -p ${NFS_DESTDIR}/${sub#${BITS_DIR}}
     done

     md5sums=""
-    for file in $FILES; do
+    for file in ${FILES}; do
         remote_object=${NFS_DESTDIR}/${file#$BITS_DIR}

         local_md5_line=$(md5sum ${file})
@@ -181,6 +206,7 @@ function nfs_upload {
             echo "skipping upload."
         fi
         md5sums="${md5sums}${local_md5_line}\n"
+        echo $remote_object >> ${BITS_DIR}/artifacts.txt
     done

     # upload the md5sums
@@ -190,7 +216,7 @@ function nfs_upload {
     if [[ -L ${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest ]]; then
         unlink ${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest
     fi
-    (cd $UPLOAD_BASE_DIR ; ln -s ${UPLOAD_SUBDIR} ${UPLOAD_BRANCH}-latest)
+    (cd ${UPLOAD_BASE_DIR} ; ln -s ${UPLOAD_SUBDIR} ${UPLOAD_BRANCH}-latest)

     # If this is a bi-weekly release branch, also update latest-release link
     if [[ $UPLOAD_BRANCH =~ ^release- ]]; then
@@ -201,12 +227,15 @@ function nfs_upload {
     fi
 }

+#
+# Look for build artifacts to operate on.
+#
 function find_upload_bits {
     if [[ -z "$SUBDIRS" ]]; then
         SUBS=$(find $BITS_DIR -type d)
         FILES=$(find $BITS_DIR -type f)
     else
-        for subdir in $SUBDIRS; do
+        for subdir in ${SUBDIRS}; do
             if [[ -d $BITS_DIR/$subdir ]]; then
                 SUBS="$SUBS $(find $BITS_DIR/$subdir -type d)"
                 FILES="$FILES $(find $BITS_DIR/$subdir -type f)"
@@ -215,6 +244,44 @@ function find_upload_bits {
     fi
 }

+#
+# Publish build artifacts to updates.joyent.com.
+#
+function publish_to_updates {
+    echo "Publishing updates to updates.joyent.com"
+    for file in ${FILES}; do
+        echo ${file} | grep -q '.imgmanifest$'
+        if [[ $? -ne 0 ]]; then
+            continue
+        fi
+
+        MF=${file}
+        IMAGEFILE=$(echo ${file} | sed -e 's/.imgmanifest$/.zfs.gz/g')
+
+        if [[ ! -f ${ZFSFILE} ]]; then
+            echo "Unable to determine ZFS image file for ${MF}."
+            echo "Not publishing to updates.joyent.com"
+            continue
+        fi
+
+        UUID=$(json -f ${MF} uuid)
+        if [[ -z "${UUID}" ]]; then
+            echo "Unable to determine UUID of $MF."
+            echo "Not publishing to updates.joyent.com"
+            continue
+        fi
+
+        # The default 1hr expiry for msign is sufficient, since we're going
+        # to be accessing this URL almost immediately.
+        MANTA_PATH=${MANTA_PATHS[$(basename $IMAGEFILE])}
+        MANTA_URL=$(msign $MANTA_PATH)
+        FILE_SHA=$(digest -a sha1 $IMAGEFILE)
+        echo updates-imageadm import -m $MF --channel experimental
+        echo updates-imgadm add-file --url -s $FILE_SHA -f $MANTA_URL $UUID
+    fi
+}
+
+
 #
 # Main
 #
@@ -235,6 +302,9 @@ while getopts "B:b:d:n:Nt:" opt; do
         n)
             NAME=$OPTARG
             ;;
+        p)
+            PUBLISH_UPDATES=true
+            ;;
         t)
             TIMESTAMP=$OPTARG
             ;;
@@ -269,6 +339,12 @@ fi

 start_time=$(date +%s)

+# we keep a file containing a list of uploads for this
+# session, useful to include as part of build artifacts.
+if [[ -f $BITS_DIR/artifacts.txt ]]; then
+    rm -f $BITS_DIR/artifacts.txt
+fi
+
 find_upload_bits

 if [[ -z "$TIMESTAMP" ]]; then
@@ -284,11 +360,18 @@ fi
 UPLOAD_SUBDIR=$TIMESTAMP

 if [[ -n "$USE_NFS" ]]; then
+    if [[ -n "$PUBLISH_TO_UPDATES" ]]; then
+        fatal "-p requires that we are publishing to Manta"
+    fi
     nfs_upload
 else
     manta_upload
 fi

+if [[ -n "$PUBLISH_TO_UPDATES" ]]; then
+    publish_to_updates
+fi
+
 end_time=$(date +%s)
 elapsed=$((${end_time} - ${start_time}))
 if [[ -n "$USE_NFS" ]]; then
timf@iorangi-eth0 (master)