generate-query-tree

generate-query-tree

#!/bin/bash
#
# based off ec2-publishing-scripts 'generate-query-tree', but
# updated for my personal publishing habits on a private openstack instance
#
export LC_ALL=C
BUCKET="${PUBLISH_BUCKET:-smoser-cloud-images}"
NUM_ARCHES=2
TEMP_D=""

Usage() {
	cat <<EOF
Usage: ${0##*/} [outdir]

  outdir defaults to ./outd

  update a /query tree based data in ec2
  Note, this would be lossy for past images (dailies that have fallen off)

  this  makes an attempt to save old data. you could populate the
  output dir with:
      rsync cloud-images.ubuntu.com::uec-images/query/ outd -az --stats

EOF
}

error() { echo "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
cleanup() {
	[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}

xnova() { bash -c '. $HOME/creds/novarc && exec nova "$@"' nova "$@"; }

ec2data_sort() {
	# ec2 files are sorted by 
	#  serial (4), region (7), virtualization_type (11), 
	#  then store (5) and arch (6)
	local tab=$'\t'
	sort --uniq "--field-separator=${tab}" -k 4,4 -k 7,7 -k 11,11 -k 5,6
}

printdata() {
	local bdir=${1} region=$2
	local loc=$region oifs=$IFS
	[ "$region" = "us-east-1" ] && loc="us" ||
		{ [ "$region" = "eu-west-1" ] && loc="eu"; }

	local bp="${BUCKET}"
	local mdif="" manif iarch rstore ami aki ari vt
	while IFS="|" read manif iarch rstore ami aki ari vt; do
		[ -n "$vt" ] || vt="paravirtual"
		mdir="${manif}"
		case "$mdir" in
			ubuntu/*) mdir=${mdir#ubuntu/}; mdir="ubuntu/${mdir%%/*}";;
			*) mdir=${mdir%%/*};;
		esac
		mbname=${manif##*/}
		mbname=${mbname%.manifest.xml}

		case "$mbname" in
			*-vmlinu[xz]|*-ramdisk|*-kernel)
				error "skipping $mbname";
				continue;;
		esac
		case "${mdir}" in
			${bp}-testing-$loc|${bp}-testing) rline=testing;;
			${bp}-sandbox-$loc|${bp}-sandbox) continue;;
			${bp}-$loc|${bp}-milestone-$loc|$bp|$bp-milestone) rline=release;;
			*) fail "unknown manifest/name: $manif ($mdir)";
		esac
		[ "$iarch" = "x86_64" ] && arch=amd64 || arch=i386

		# HACK: due to loader kernels, i386 images are registered as amd64
		# trust their name, not their metadata for arch type
		case "$mbname" in
			*-i386-*) arch="i386";;
		esac

		IFS="-"; set -- ${mbname}; IFS=${oifs}
		suite=${2}

		local label=""
		# spin (build name) is always the second to last token
		if [ $# -eq 7 ]; then
			serial=$7
 			spin=$6
			label=$4
		elif [ $# -eq 6 ]; then
			serial=$6
			spin=$5
			case "$3" in
				[0-9]*) label="release";;
				*) label=$3;;
			esac
		else
			fail "confused by name ${mbname}"
		fi

		#error "label=$label spin=$spin mbname=$mbname"

		dir="${bdir}/${suite}/${spin}"
		local print printdl
		print=( printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n"
			"$suite" "$spin" "$label" "$serial"
			"$rstore" "$arch" "$region" "$ami" "$aki" "$ari" "$vt" )
		printdl=( printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n"
			"$suite" "$spin" "$label" "$serial" "$arch" 
			"/see/cloud/images/for/downloads" "$mbname" )
		[ -d "${dir}" ] || mkdir -p "${dir}" || return 1
		if [ "${label}" = "daily" ]; then
			"${print[@]}" >> "${dir}/daily.txt"
			"${printdl[@]}" >> "${dir}/daily-dl.txt"
		else
			"${print[@]}" >> "${dir}/released.txt"
			"${printdl[@]}" >> "${dir}/released-dl.txt"
		fi
		[ $? -eq 0 ] || return 1
	done
}

TEMP_D=$(mktemp -d ${TMPDIR:-/tmp}/${0##*/}.XXXXXX)
trap cleanup EXIT
export XC2_XIMAGES_CACHE_D=${XC2_XIMAGES_CACHE_D:-"$TEMP_D/ximgcache"}

[ $# -eq 0 ] && { Usage 1>&2; exit 1; }
[ "$1" = "-h" -o "$1" = "--help" ] && { Usage; exit 0; }

outd="$1"
[ -d "$outd" ] || mkdir -p "$outd"

#regions=( $(ec2-list-all-regions) ) || fail "failed to get list of regions"
regions=( lcy01 lcy02 us-east-1 )

# save off the originals
for f in "${outd}/query"/*/*/* "${outd}/query-nova"/*/*/*; do
	[ -f "$f" ] || continue
	case "$f" in
		*/released.txt|*/daily.txt) :;;
		*) continue;;
	esac
	cp "$f" "$f.orig" || fail "failed to save $f to $f.orig"
done

name_match="^${BUCKET}"
for region in "${regions[@]}"; do
	f="${TEMP_D}/images.$r.txt"
	error "updating image cache for $region"

	# we cheat, and populate 'us-east-1' as 'lcy01', because
	# ubuntu-cloudimg-query only considers regions formatted as 'xx-xxxxx-0[0-9]'
	xregion=$region
	[ "$region" == "us-east-1" ] && xregion="lcy01"

	OS_REGION_NAME=$xregion euca-describe-images --all > "${f}" ||
		fail "failed to describe-images in region $region"
	ufile="$TEMP_D/ubuntu.images.$region.txt"
	# fields:
	#   2: ami-id                   #  14: virtualization-type
	#   3: name/path                #  10: aki
	#   8: arch                     #  11: ari
	#  13: root-store
	#  Note: LP: #954068 affects root-store and virtualization-type
	awk '-F\t' '$1 == "IMAGE" && $3 ~ ubum && $6 == "public" {
		print($3,$8,$13,$2,$10,$11,$14); }' \
		"ubum=$name_match" OFS="|" "$f" > "$ufile" ||
		fail "failed to get ubuntu images from $ufile"
		printdata "${outd}/query" "$region" < "$ufile" ||
		fail "failed to print ubuntu images in $region"

	error "updating image cache for nova $region"
	f="${TEMP_D}/images.nova.$r.txt"
	OS_REGION_NAME=$xregion xnova image-list > "${f}" ||
		fail "failed to describe-images in region $region"
	ufile="$TEMP_D/nova.ubuntu.images.$region.txt"
	# field are '|' and space in a prettytable so awk tokens are:
	#   2: image-uuid  #  4: image-name  # 6: status
	# need to create:
	#   manif iarch rstore ami aki ari vt
	#  Note: LP: #954068 affects root-store and virtualization-type
	awk '$6 == "ACTIVE" && $4 ~ ubum {
		arch="x86_64"
		if ($4 ~ /i386/) { arch="i386"; } 
		print($4,arch,"instance-store",$2,"","",""); }' \
		"ubum=$name_match" OFS="|" "$f" > "$ufile" ||
		fail "failed to get ubuntu images from $ufile"
		printdata "${outd}/query-nova" "$region" < "$ufile" ||
		fail "failed to print ubuntu images in $region"
done

# now update the .current files from the released.txt
for f in "${outd}/query"/*/*/* "${outd}/query-nova"/*/*/*; do
	case "$f" in
		*/released.txt|*/daily.txt) :;;
		*) continue;;
	esac

	# append to the file any original data.
	if [ -f "$f.orig" ]; then
		 cat "$f.orig" >> "$f" && rm "$f.orig" ||
			fail "failed to update $f with orig data"
	fi

	# sort results
	ec2data_sort < "$f" > "$f.new"
	mv "${f}.new" "${f}"
	error "wrote to $f"
	cur="${f%.txt}.current.txt"

	: > "$cur"
	for region in "${regions[@]}"; do
		# for each region, get the most recent serial
		# from the (sorted) released.txt or daily.txt
		awk '-F\t' '$7 == region { last=$0 }; END { print last }' \
			"region=$region" "$f" > "$TEMP_D/latest" ||
			fail "failed to get latest entry for $region from $f"

		# get the bottom entry, and populate current with stuff like it
		read suite bname label serial other < "$TEMP_D/latest" ||
			fail "failed to read final entry in $f"

		cond='$1 == suite && $2 == bname && $3 == label && $4 == serial && $7 == region'
		awk '-F\t' "$cond { print \$0 }" \
			"suite=$suite" "bname=$bname" "label=$label" \
			"serial=$serial" "region=$region" "$f" >> "$cur" ||
			fail "failed to update current $cur"
	done

	error "wrote to $cur"
done

# this goes through each of the suite/buildname/daily.current.txt 
# files and then writes the first 4 fields of each of those files into
# the current.txt file at the top level.
for r in daily released; do
	for topl in query query-nova; do
		tail -qn 1 "$outd/$topl"/*/*/"$r.current.txt" > "$TEMP_D/$r.current" &&
			awk '-F\t' '{print $1,$2,$3,$4}' 'OFS=\t' \
				"$TEMP_D/$r.current" > "$outd/$topl/$r.latest.txt" ||
			fail "failed to write to $outd/$topl/$r.latest.txt"
		error "updated $topl/$r.latest.txt"
	done
done

# create the bogus -dl.current.txt files to make ubuntu-cloudimg-query happy
for stream in daily released; do
	for dlfile in "$outd/"query/*/*/"${stream}-dl.txt" \
		"$outd/"query-nova/*/*/"${stream}-dl.txt"; do

		[ -f "$dlfile" ] || fail "full dl file $dlfile not a file"
		# dlfiles get a line per region , per arch above. thats busted.  so
		# here, we just fix that and sort -u it.
		sort --unique --reverse "$dlfile" > "$dlfile".new
		mv "${dlfile}.new" "$dlfile"
		head -n "$NUM_ARCHES" "$dlfile" > "${dlfile%.txt}.current.txt"
	done
done

# vi: ts=4 noexpandtab