Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
doing imgadm upload and saving an artifacts manifest file
timf@iorangi-eth0 (master) git diff
diff --git a/tools/bits-upload.sh b/tools/bits-upload.sh
index 8f0d77c..ca14a9d 100755
--- a/tools/bits-upload.sh
+++ b/tools/bits-upload.sh
@@ -57,6 +57,7 @@ function usage {
echo " -n <name> the name of component to upload"
echo " -N use NFS rather than manta to store artifacts"
echo " -t <timestamp> the timestamp (optional, derived otherwise)"
+ echo " -p publish bits to updates.joyent.com experimental"
echo ""
echo "Upload bits to Manta or NFS destination from \$BITS_DIR"
echo "which (defaults to <component>/bits)"
@@ -68,6 +69,12 @@ function usage {
exit 2
}
+#
+# Maintain a global associative array mapping uploaded file basenames
+# to their corresponding Manta paths. This assumes basenames are unique.
+#
+declare -A STORED_MANTA_PATHS
+
#
# Upload build artifacts to Manta. There's some duplication in the logic
# here and nfs_upload.
@@ -85,7 +92,6 @@ function manta_upload {
if [[ -z "$MANTA_USER" ]]; then
export MANTA_USER="Joyent_Dev";
fi
-
if [[ ${UPLOAD_BASE_DIR:0:1} != '/' ]]; then
# if it starts with a / we assume it's /stor/<something> or
# /public/<something> if not, we prepend /stor
@@ -122,21 +128,40 @@ function manta_upload {
echo "Skipping upload."
fi
md5sums="${md5sums}${local_md5_line}\n"
+
+ # save the file to our global assoc-array of {filename: manta path}
+ # used later when mapping manifests to image file URLs.
+ MANTA_PATHS[$(basename $file)]=${manta_object}
+
+ # Store a full URL if it appears to be a public resource, otherwise
+ # just save the manta path.
+ echo $manta_object | grep -q /public/
+ if [[ $? -eq 0 ]]; then
+ echo ${MANTA_URL}${manta_object} >> ${BITS_DIR}/artifacts.txt
+ else
+ echo $manta_object >> ${BITS_DIR}/artifacts.txt
+ fi
done
# upload the md5sums
echo -e $md5sums | \
mput -v -H 'content-type: text/plain' ${MANTA_DESTDIR}/md5sums.txt
+ echo ${MANTA_DESTDIR}/md5sums.txt >> ${BITS_DIR}/artifacts.txt
# now update the branch latest link
echo "${MANTA_DESTDIR}" | \
mput -v -H 'content-type: text/plain' \
/${MANTA_USER}${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest
+ echo /${MANTA_USER}${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest >> \
+ ${BITS_DIR}/artifacts.txt
+
# If this is a bi-weekly release branch, also update latest-release link
if [[ $UPLOAD_BRANCH =~ ^release- ]]; then
echo "${MANTA_DESTDIR}" | \
mput -v -H 'content-type: text/plain' \
/${MANTA_USER}${UPLOAD_BASE_DIR}/latest-release
+ echo /${MANTA_USER}${UPLOAD_BASE_DIR}/latest-release >> \
+ ${BITS_DIR}/artifacts.txt
fi
echo "Uploaded to ${MANTA_DESTDIR}"
@@ -151,12 +176,12 @@ function manta_upload {
function nfs_upload {
NFS_DESTDIR=${UPLOAD_BASE_DIR}/${UPLOAD_SUBDIR}
- for sub in $SUBS; do
+ for sub in ${SUBS}; do
mkdir -p ${NFS_DESTDIR}/${sub#${BITS_DIR}}
done
md5sums=""
- for file in $FILES; do
+ for file in ${FILES}; do
remote_object=${NFS_DESTDIR}/${file#$BITS_DIR}
local_md5_line=$(md5sum ${file})
@@ -181,6 +206,7 @@ function nfs_upload {
echo "skipping upload."
fi
md5sums="${md5sums}${local_md5_line}\n"
+ echo $remote_object >> ${BITS_DIR}/artifacts.txt
done
# upload the md5sums
@@ -190,7 +216,7 @@ function nfs_upload {
if [[ -L ${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest ]]; then
unlink ${UPLOAD_BASE_DIR}/${UPLOAD_BRANCH}-latest
fi
- (cd $UPLOAD_BASE_DIR ; ln -s ${UPLOAD_SUBDIR} ${UPLOAD_BRANCH}-latest)
+ (cd ${UPLOAD_BASE_DIR} ; ln -s ${UPLOAD_SUBDIR} ${UPLOAD_BRANCH}-latest)
# If this is a bi-weekly release branch, also update latest-release link
if [[ $UPLOAD_BRANCH =~ ^release- ]]; then
@@ -201,12 +227,15 @@ function nfs_upload {
fi
}
+#
+# Look for build artifacts to operate on.
+#
function find_upload_bits {
if [[ -z "$SUBDIRS" ]]; then
SUBS=$(find $BITS_DIR -type d)
FILES=$(find $BITS_DIR -type f)
else
- for subdir in $SUBDIRS; do
+ for subdir in ${SUBDIRS}; do
if [[ -d $BITS_DIR/$subdir ]]; then
SUBS="$SUBS $(find $BITS_DIR/$subdir -type d)"
FILES="$FILES $(find $BITS_DIR/$subdir -type f)"
@@ -215,6 +244,44 @@ function find_upload_bits {
fi
}
+#
+# Publish build artifacts to updates.joyent.com.
+#
+function publish_to_updates {
+ echo "Publishing updates to updates.joyent.com"
+ for file in ${FILES}; do
+ echo ${file} | grep -q '.imgmanifest$'
+ if [[ $? -ne 0 ]]; then
+ continue
+ fi
+
+ MF=${file}
+ IMAGEFILE=$(echo ${file} | sed -e 's/.imgmanifest$/.zfs.gz/g')
+
+ if [[ ! -f ${ZFSFILE} ]]; then
+ echo "Unable to determine ZFS image file for ${MF}."
+ echo "Not publishing to updates.joyent.com"
+ continue
+ fi
+
+ UUID=$(json -f ${MF} uuid)
+ if [[ -z "${UUID}" ]]; then
+ echo "Unable to determine UUID of $MF."
+ echo "Not publishing to updates.joyent.com"
+ continue
+ fi
+
+ # The default 1hr expiry for msign is sufficient, since we're going
+ # to be accessing this URL almost immediately.
+ MANTA_PATH=${MANTA_PATHS[$(basename $IMAGEFILE])}
+ MANTA_URL=$(msign $MANTA_PATH)
+ FILE_SHA=$(digest -a sha1 $IMAGEFILE)
+ echo updates-imageadm import -m $MF --channel experimental
+ echo updates-imgadm add-file --url -s $FILE_SHA -f $MANTA_URL $UUID
+ fi
+}
+
+
#
# Main
#
@@ -235,6 +302,9 @@ while getopts "B:b:d:n:Nt:" opt; do
n)
NAME=$OPTARG
;;
+ p)
+ PUBLISH_UPDATES=true
+ ;;
t)
TIMESTAMP=$OPTARG
;;
@@ -269,6 +339,12 @@ fi
start_time=$(date +%s)
+# we keep a file containing a list of uploads for this
+# session, useful to include as part of build artifacts.
+if [[ -f $BITS_DIR/artifacts.txt ]]; then
+ rm -f $BITS_DIR/artifacts.txt
+fi
+
find_upload_bits
if [[ -z "$TIMESTAMP" ]]; then
@@ -284,11 +360,18 @@ fi
UPLOAD_SUBDIR=$TIMESTAMP
if [[ -n "$USE_NFS" ]]; then
+ if [[ -n "$PUBLISH_TO_UPDATES" ]]; then
+ fatal "-p requires that we are publishing to Manta"
+ fi
nfs_upload
else
manta_upload
fi
+if [[ -n "$PUBLISH_TO_UPDATES" ]]; then
+ publish_to_updates
+fi
+
end_time=$(date +%s)
elapsed=$((${end_time} - ${start_time}))
if [[ -n "$USE_NFS" ]]; then
timf@iorangi-eth0 (master)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment