simplestreams-image-convert.sh

#!/bin/bash -eu
#
# Origin: https://gist.github.com/dosaboy/95372c6ec16c9360dec240a2aa5859bb
#
# Authors:
#  - edward.hope-morley@canonical.com
#  - opentastic@gmail.com
#
# Description:
#   Convert simplestreams images in Glance to disk_format=raw
#
# Usage:
#   Add to /etc/cron.daily/glance_simplestreams_sync or run manually.
#

images_root=/var/cache/simplestreams-image-convert
mkdir -p $images_root
mkdir -p $images_root/actions/delete
mkdir -p $images_root/completed
#lockfile=/var/cache/simplestreams-image-convert/lock
logfile=/var/log/simplestreams-image-convert.log

dpkg -s jq| grep 'Status: install ok installed' || sudo apt install jq -y

#(
(
#flock -e 200
echo "== Conversion started at `date` =="
# Authenticate as if simplestreams agent
fid=/etc/glance-simplestreams-sync/identity.yaml
for s in auth_host admin_password admin_tenant_name admin_user admin_domain_name; do
    eval $s=`sed -r "s/.*${s}:\s+(.+)\s*/\1/g;t;d" $fid`
done
export OS_AUTH_URL=http://${auth_host}:35357/v3
export OS_PROJECT_NAME=$admin_tenant_name
export OS_USERNAME=$admin_user
export OS_PASSWORD=$admin_password
export OS_PROJECT_DOMAIN_NAME=$admin_domain_name
export OS_USER_DOMAIN_NAME=$admin_domain_name
export OS_IDENTITY_API_VERSION=3
export OS_AUTH_VERSION=3

for pkg in python-openstackclient python-swiftclient; do
    dpkg -s $pkg| grep -q "Status: install ok installed" || \
            sudo apt install $pkg -y
done

swift download simplestreams data/streams/v1/auto.sync.json \
                             -o $images_root/auto.sync.json

python << EOF
import json

j = json.loads(open('${images_root}/auto.sync.json').read())
with open('$images_root/properties', 'w') as fd:
    properties = {
        'sha256': {},
        'name': {},
        'size': {},
        'md5': {},
        'product_id': {},
        'product_ver': {},
    }
    image_ids = []
    for product_id, img in j['products'].iteritems():
        for ver, entry in img['versions'].iteritems():
            for item in entry['items'].itervalues():
                id = item['id']
                image_ids.append(id)
                properties['sha256'][id] = item['sha256']
                properties['name'][id] = item['name']
                properties['size'][id] = item['size']
                properties['md5'][id] = item['md5']
                properties['product_id'][id] = product_id
                properties['product_ver'][id] = ver

    fd.write("declare -a images=( \n")
    fd.write("{}".format('\n'.join(image_ids)))
    fd.write(")\n")

    fd.write("declare -A properties=( \n")
    for property in properties:
        for id, val in properties[property].iteritems():
            fd.write("    [{}.{}]={}\n".format(id, property, val))
    fd.write(")\n")
EOF

cat $images_root/properties
. $images_root/properties

echo "${#images[@]} images to process"

for id in ${images[@]}; do
    sha256=${properties[$id.sha256]}
    name=${properties[$id.name]}
    size=${properties[$id.size]}
    md5=${properties[$id.md5]}
    product_id=${properties[$id.product_id]}
    product_ver=${properties[$id.product_ver]}

    echo -e "\n== Processing image '$name' id=$id =="
    if [ -r "$images_root/actions/delete/$id" ]; then
         echo "Deleting image $id from Glance"
         jq "del(.products.\"$product_id\".versions.\"$product_ver\")" $images_root/auto.sync.json > $images_root/actions/delete/$id.auto.sync.json
         openstack image delete $id || continue
         swift upload simplestreams --object-name data/streams/v1/auto.sync.json \
                                    $images_root/actions/delete/$id.auto.sync.json 
         cp $images_root/actions/delete/$id.auto.sync.json $images_root/auto.sync.json
         rm $images_root/actions/delete/$id.*
         mv $images_root/actions/delete/$id $images_root/actions/delete/$id.deleted
         continue
    fi

    if [ -r "$images_root/completed/$id" ] || \
         [ "`openstack image show $id -c disk_format -f value`" = "raw" ]; then
        echo "Image '$name' (id=$id) already has disk_format=raw - skipping"
        touch $images_root/completed/$id
        continue
    fi

    echo "$name" > $images_root/img_${id}.name
    echo "Downloading image from glance"
    glance image-download --progress --file $images_root/img_${id}.qcow2 $id

    echo "De-activating image in glance"
    glance image-deactivate $id

    echo "Converting image to RAW"
    qemu-img convert -p -f qcow2 -O raw $images_root/img_${id}.qcow2 \
                                        $images_root/img_${id}.raw
    echo "Fetching image properties"
    openstack image show $id -c properties -f value > $images_root/img_${id}.properties
    p_simplestreams_metadata=`sed -r "s/.+simplestreams_metadata='(.+})'.+/\1/g" $images_root/img_${id}.properties`
    readarray p_other<<<`sed -r "s/(.+), simplestreams_metadata=.+}'(.+)/\1\2/g" \
                         $images_root/img_${id}.properties| \
                         sed -r "s/(.+), locations='\[.+\]'(.+)/\1\2/g"| \
                         sed -r 's/, /\n/g'| \
                         egrep -v "direct_url|locations|architecture"|tr -d "\'"`

    echo "Creating new glance image"
    newid=`openstack image create --public \
           --container-format=bare --disk-format=raw \
           --file=$images_root/img_${id}.raw \
           --property architecture=x86_64 \
           $name| \
           awk '$2=="id" {print $4}'`

    echo "New image id=$newid"
    if [ -z "$newid" ]; then
        echo "ERROR: image did not create successfully"
        break
    fi

    newsize=`openstack image show $id -c size -f value`
    checksum=`openstack image show $id -c checksum -f value`

    echo "Setting new image properties in Glance"
    openstack image set \
      --property simplestreams_metadata="$p_simplestreams_metadata" $newid
    for p in ${p_other[@]}; do
        [ "${p%%=*}" = "os_hash_value" ] && continue
        [ "${p%%=*}" = "os_hidden" ] && continue
        echo "Setting $p"
        openstack image set --property "$p" $newid
    done
    echo "Deleting original image $id"
    glance image-delete $id
    echo "Updating simplestreams data for image ${id}->${newid}"
    sed -i -e "s/$id/$newid/g" -e "s/$size/$newsize/g" \
           -e "s/$md5/$checksum/g" $images_root/auto.sync.json
    echo "Updating sha256sum (this will take a few seconds)"
    newsha256=`sha256sum -b $images_root/img_${id}.raw| cut -f 1 -d ' '`
    sed -i "s/$sha256/$newsha256/g" $images_root/auto.sync.json
    rm $images_root/img_${id}.*
    echo "Updating swift simplestreams index"
    swift upload simplestreams --object-name data/streams/v1/auto.sync.json \
                           $images_root/auto.sync.json
    touch $images_root/completed/$id
done

echo -e "\n== Conversion finished at `date` =="
) &>> $logfile
#) 200>$lockfile