Skip to content

Instantly share code, notes, and snippets.

@smoser
Last active April 2, 2020 13:08
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 2 You must be signed in to fork a gist
  • Save smoser/5364534 to your computer and use it in GitHub Desktop.
Save smoser/5364534 to your computer and use it in GitHub Desktop.
Utility for converting a disk image in raw format or some format qcow understands into VHD format.
#!/bin/bash
VERBOSITY=0
TEMP_D=""
FORMATS=(
qcow2
qcow2-compressed
raw
azure
azure-dynamic
azure-fixed
)
error() { echo "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
_assert_python() {
[ -n "$_PYTHON" ] && return 0
local out="" c=""
for c in python python3; do
out=$(command -v $c 2>&1) && _PYTHON=$c && break
done
[ -n "$_PYTHON" ] || fail "No python available."
}
json_key() {
# json_key(json_blob, keyname)
_assert_python
local val="" jblob="$1" kname="$2" req="$3"
val=$(
$_PYTHON -c "
import json, sys;
jblob, kname = (sys.argv[1], sys.argv[2])
data = json.loads(jblob)
print(data.get(kname, '_none'))" "$jblob" "$kname") ||
{ error "Failed to read input as json"; return 1; }
[ "$val" = "_none" -a "$req" = "req" ] && return 1
_RET="$val"
}
get_img_info() {
debug 2 "get image info${2:+ -f $2} --output=json $1"
qemu-img info ${2:+-f $2} --output=json "$1" ||
{ debug 1 "failed qemu-img info $img"; return 1; }
}
get_img_info_key() {
local img="$1" key="$2" ifmt=$3 info=""
info=$(qemu-img info ${ifmt:+-f $ifmt} --output=json "$img") ||
{ debug 1 "failed qemu-img info $img"; return 1; }
json_key "$info" "$key" || {
debug 1 "failed to get key '$key' from '$img'"
return 1
}
}
disk_to_azure() {
# https://docs.microsoft.com/en-us/azure/virtual-machines/linux/redhat-create-upload-vhd
local img_in="$1" img_out="$2" subfmt=$3 ifmt="$4" img_tmp=""
local out="" force_size="" fimg_in="" ret=""
fimg_in=$(realpath "$img_in") ||
{ error "failed to get full path to $img_in"; return 1; }
out=$(qemu-img convert -O vpc -o ? 2>&1) ||
{ error "Failed to check qemu-img for vpc out."; return 1; }
echo "$out" | grep -q force_size && force_size="force_size"
get_img_info_key "$img_in" virtual-size "$ifmt" || return
local vsize=$_RET mb=$((1024*1024))
local roundmb=$(( ((vsize+mb-1)/mb)*mb ))
debug 2 "${img_in} is $vsize bytes."
local curifmt="$ifmt"
if [ $vsize -ne $roundmb ]; then
debug 1 "creating $vsize != $roundmb temp image in even mb $roundmb."
img_tmp="$TEMP_D/img.roundmb"
rq 3 create-tmp-1mb-img qemu-img create \
-f qcow2 -F "$curifmt" -b "$fimg_in" "$img_tmp" "$roundmb" ||
{ error "failed to create temp image"; return 1; }
curifmt="qcow2"
else
img_tmp="$img_in"
fi
local opts="subformat=$subfmt"
# force_size is https://bugs.launchpad.net/qemu/+bug/1490611
# and really is probably necessary.
opts="$opts${force_size:+,${force_size}}"
rq 2 convert-vhd qemu-img convert \
-f "$curifmt" -O vpc -o "$opts" \
"$img_tmp" "$img_out"
ret=$?
[ "$img_tmp" = "$img_in" ] || rm -f "$img_tmp"
return $ret
}
rq() {
local verbosity=$1 name=$2
shift 2;
debug 2 "running:" "$@"
if [ ${VERBOSITY} -ge $verbosity ]; then
"$@"
else
local f="${TEMP_D}/${name}.out"
"$@" > "$f" 2>&1
local ret=$?
[ $ret -eq 0 ] && { rm -f "$f"; return 0; }
error "FAILED:" "$@"
cat "$f" 1>&2;
rm -f "${f}"
return $ret
fi
}
bad_Usage() { Usage 1>&2; fail "$@"; }
Usage() {
cat <<EOF
Usage: ${0##*/} [ options ] input format output
Convert a qcow image to another format successfully.
options:
--resize SIZE grow the disk to SIZE during conversion
format is one of:
EOF
local f
for f in "${FORMATS[@]}"; do
echo " $f"
done
}
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }
cleanup() {
[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
debug() {
local level=${1}; shift;
[ "${level}" -gt "${VERBOSITY}" ] && return
error "${@}"
}
is_hidden_vpc() {
# qemu-img info can't identify vpc fixed format from raw.
# https://bugs.launchpad.net/bugs/1819182
local detected="$1" img="$2"
[ "$detected" = "raw" ] || return 1
case "$img" in
*.[vV][hH][dD]|*.[vV][pP][cC]) return 0;;
esac
local foot=""
# footer at 512 bytes from end starts with 'conectix'
tail -c 512 "$img" | head -c 8 | grep -q conectix && return 0
return 1
}
inargs() {
local needle="$1" hay=""
shift
for hay in "$@"; do
[ "$needle" = "$hay" ] && return 0
done
return 1
}
short_opts="hr:v"
long_opts="help,resize:,verbose"
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
bad_Usage
## <<insert default variables here>>
input=""
output=""
resize=""
while [ $# -ne 0 ]; do
cur=$1; next=$2;
case "$cur" in
-h|--help) Usage ; exit 0;;
-v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
-r|--resize) resize=$next;;
--) shift; break;;
esac
shift;
done
[ $# -eq 3 ] ||
bad_Usage "Expect 3 args. (input, format, output). got $#"
input="$1"
oformat="$2"
output="$3"
curimg="$input"
command -v qemu-img >/dev/null || fail "No qemu-img in path"
[ -f "$input" ] || fail "$input: not a file"
[ "$input" -ef "$output" ] && fail "input has to differ from output."
rm -f "$output" || fail "failed to delete output file: $output"
inargs "$oformat" "${FORMATS[@]}" ||
fail "$oformat not valid format: ${FORMATS[*]}"
TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
fail "failed to make tempdir"
trap cleanup EXIT
info=$(get_img_info "$input") ||
fail "failed to get image info for $input"
json_key "$info" "format" req && ifmt="$_RET" ||
fail "failed to get image type of $_RET: $info"
if is_hidden_vpc "$ifmt" "$input"; then
debug 1 "Input looks like vhd, assuming that."
ifmt="vpc"
info=$(get_img_info "$input" "$ifmt") ||
fail "failed to get image info for $input as fmt=$ifmt"
fi
json_key "$info" "virtual-size" req && isize="$_RET" ||
fail "failed to get size of $input"
debug 1 "input: $input [format=$ifmt virtual-size=$isize]"
debug 2 "output: $output [format=$oformat]"
case "$oformat" in
# force_size is https://bugs.launchpad.net/qemu/+bug/1490611
# older qemu-img just can't really work with dynamic size.
azure-dynamic)
out=$(qemu-img convert -O vpc -o ? 2>&1) ||
{ error "Failed to check qemu-img for vpc out."; return 1; }
if ! echo "$out" | grep -q force_size; then
error "Cannot create $oformat with this qemu."
fail "see https://bugs.launchpad.net/qemu/+bug/1490611"
fi
esac
img_raw="${TEMP_D}/img.raw"
curifmt="$ifmt"
if [ -n "$resize" ]; then
debug 1 "resizing via qcow2 backed."
fcurimg=$(realpath "$curimg")
rq 2 resize-with-qcow2 qemu-img create \
-F "$ifmt" -f qcow2 -b "$fcurimg" \
"$TEMP_D/resized.img" "$resize" ||
fail "failed to create qcow2 resized."
curimg="$TEMP_D/resized.img"
curifmt="qcow2"
fi
case "$oformat" in
qcow2-compressed)
rq 2 to-qcow2-comp qemu-image convert \
-f "$curifmt" -O qcow2 -c \
"$curimg" "$output" ||
fail "failed to convert to qcow2"
;;
qcow2)
rq 2 to-qcow2 qemu-image convert \
-f "$curifmt" -O qcow2 \
"$curimg" "$output" ||
fail "failed to convert to qcow2"
;;
azure-dynamic)
disk_to_azure "$curimg" "$output" dynamic $curifmt ||
fail "failed to convert to azure-dynamic type."
;;
azure|azure-fixed)
disk_to_azure "$curimg" "$output" fixed $curifmt ||
fail "failed to convert to azure type."
;;
raw)
rq 2 to-raw qemu-image convert \
-f "$curifmt" -O raw "$curimg" "$output" ||
fail "failed to convert to raw"
;;
esac
debug 1 "created $output in $oformat format"
exit
# vi: ts=4 expandtab
#!/bin/bash
# any 'variant' and format options other than these here end up being reported
# by 'vboxmanage showhdinfo' as 'dynamic-default'
# VDI-Fixed: fixed default
# VMDK-Stream: dynamic streamOptimized
# VHD-Fixed: fixed default
VERBOSITY=0
TEMP_D=""
# see 'vboxmanage clonehd' output
FORMATS=(
qcow2
qcow2-compressed
raw
vdi
vdi-fixed
vhd
vhd-fixed
vmdk
vmdk-esx
vmdk-stream
)
declare -A VBOX_VAR_MAP
declare -A VBOX_FMT_MAP
VBOX_FMT_MAP=( [vdi]=VDI [vhd]=VHD [vmdk]="VMDK" )
VBOX_VAR_MAP=( [fixed]="Fixed" [stream]="Stream" [standard]="Standard" )
error() { echo "$@" 1>&2; }
errorp() { printf "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
failp() { [ $# -eq 0 ] || errorp "$@"; exit 1; }
checkreqs() {
local missing="" missing_pkgs="" reqs="" req="" pkgs="" pkg=""
reqs=( qemu-img vboxmanage )
pkgs=( qemu-kvm virtualbox )
for((i=0;i<${#reqs[@]};i++)); do
req=${reqs[$i]}
pkg=${pkgs[$i]}
command -v "$req" >/dev/null || {
missing="${missing:+${missing} }${req}"
missing_pkgs="${missing_pkgs:+${missing_pkgs} }$pkg"
}
done
[ -z "$missing" ] || {
error "missing prereqs: $missing";
error "apt-get install ${missing_pkgs}";
return 1;
}
}
rq() {
local verbosity=$1 name=$2
shift 2;
debug 2 "running:" "$@"
if [ ${VERBOSITY} -ge $verbosity ]; then
"$@"
else
local f="${TEMP_D}/${name}.out"
"$@" > "$f" 2>&1
local ret=$?
[ $ret -eq 0 ] && { rm -f "$f"; return 0; }
error "FAILED:" "$@"
cat "$f" 1>&2;
rm -f "${f}"
return $ret
fi
}
bad_Usage() { Usage 1>&2; fail "$@"; }
Usage() {
cat <<EOF
Usage: ${0##*/} [ options ] input format output
Convert a qcow image to another format successfully.
options:
--resize SIZE grow the disk to SIZE during conversion
--direct do not use intermediate 'format' file
format is one of:
EOF
local f
for f in "${FORMATS[@]}"; do
echo " $f"
done
}
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }
cleanup() {
[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
debug() {
local level=${1}; shift;
[ "${level}" -gt "${VERBOSITY}" ] && return
error "${@}"
}
inargs() {
local needle="$1" hay=""
shift
for hay in "$@"; do
[ "$needle" = "$hay" ] && return 0
done
return 1
}
short_opts="hr:v"
long_opts="direct,help,resize:,verbose"
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
bad_Usage
## <<insert default variables here>>
input=""
output=""
resize=""
fixed=true
direct=0
while [ $# -ne 0 ]; do
cur=$1; next=$2;
case "$cur" in
--direct) direct=1;;
-h|--help) Usage ; exit 0;;
-v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
-r|--resize) resize=$next;;
--no-fixed) fixed=false;;
--) shift; break;;
esac
shift;
done
[ $# -eq 3 ] ||
bad_Usage "Expect 3 args. (input, format, output). got $#"
input="$1"
oformat="$2"
output="$3"
[ -f "$input" ] || fail "$input: not a file"
rm -f "$output" || fail "failed to delete output file: $output"
inargs "$oformat" "${FORMATS[@]}" ||
fail "$oformat not valid format: ${FORMATS[*]}"
checkreqs || fail
TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
fail "failed to make tempdir"
trap cleanup EXIT
out=$(LANG=C qemu-img info "$input") ||
fail "failed to get image info for $input"
itype=$(echo "$out" | awk '$0 ~ /^file format:/ { print $3 }') &&
[ -n "$itype" ] || fail "failed to get input type"
isize=$(echo "$out" | awk '$0 ~ /^virtual size:/ { print $4 }') &&
[ -n "$isize" ] && isize="${isize#(}" || fail "failed to get input type"
debug 1 "$input: $itype/$isize"
img_raw="${TEMP_D}/img.raw"
if [ "$itype" = "raw" ]; then
if [ -n "$resize" ]; then
debug 1 "copying input to temp for resize to $resize"
img_raw="${TEMP_D}/raw.img"
cp "$itype" "$img_raw" ||
fail "failed copy to temp"
else
ln -s "$(readlink -f "$input")" "$img_raw" ||
fail "failed to link to $input"
fi
else
debug 1 "converting input to raw"
rq 3 convert-to-raw qemu-img convert -O raw "$input" "$img_raw" ||
fail "failed conversion to raw"
fi
if [ -n "$resize" ]; then
truncate --size "$resize" "$img_raw" ||
fail "failed truncate --size $resize"
fi
fmt=${oformat%%-*}
variant=${oformat#*-}
if [ "$variant" = "$fmt" ]; then
variant="standard"
fi
if [ "$fmt" = "qcow2" ]; then
compflag=""
[ "$opts" = "compressed" ] && compflag="-c"
rq 3 raw2qcow2 qemu-image convert -O qcow2 $compflag \
"$input" "$output" ||
fail "failed to convert to qcow2"
else
# first convert to the format default (standard)
tvariant="standard"
target="${TEMP_D}/$fmt-$tvariant.img"
if [ "$variant" = "standard" ]; then
target="$output"
direct="1"
fi
if [ $direct = "1" ]; then
tvariant="$variant"
target="$output"
fi
debug 1 "converting to $fmt-$tvariant"
rq 3 to-$fmt-$tvariant vboxmanage convertfromraw \
"--format=${VBOX_FMT_MAP[$fmt]}" \
"--variant=${VBOX_VAR_MAP[$tvariant]}" \
"$img_raw" "$target" ||
fail "failed conversion to $fmt-${tvariant}"
rm "$img_raw"
if [ "$direct" != "1" ]; then
debug 1 "converting $fmt-$tvariant to $fmt-$variant"
rq 3 to-$fmt-$variant vboxmanage clonehd \
"--format=${VBOX_FMT_MAP[$fmt]}" \
"--variant=${VBOX_VAR_MAP[$variant]}" \
"$target" "$output"
fi
fi
debug 1 "created $output in $fmt-$variant"
exit
# vi: ts=4 noexpandtab
#!/bin/bash
VERBOSITY=0
TEMP_D=""
error() { echo "$@" 1>&2; }
errorp() { printf "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
failp() { [ $# -eq 0 ] || errorp "$@"; exit 1; }
checkreqs() {
local missing="" missing_pkgs="" reqs="" req="" pkgs="" pkg=""
reqs=( qemu-img vboxmanage )
pkgs=( qemu-kvm virtualbox )
for((i=0;i<${#reqs[@]};i++)); do
req=${reqs[$i]}
pkg=${pkgs[$i]}
command -v "$req" >/dev/null || {
missing="${missing:+${missing} }${req}"
missing_pkgs="${missing_pkgs:+${missing_pkgs} }$pkg"
}
done
[ -z "$missing" ] || {
error "missing prereqs: $missing";
error "apt-get install ${missing_pkgs}";
return 1;
}
}
rq() {
local verbosity=$1 name=$2
shift 2;
debug 2 "running:" "$@"
if [ ${VERBOSITY} -ge $verbosity ]; then
"$@"
else
local f="${TEMP_D}/${name}.out"
"$@" > "$f" 2>&1
local ret=$?
[ $ret -eq 0 ] && { rm -f "$f"; return 0; }
error "FAILED:" "$@"
cat "$f" 1>&2;
rm -f "${f}"
return $ret
fi
}
bad_Usage() { Usage 1>&2; fail "$@"; }
Usage() {
cat <<EOF
Usage: ${0##*/} [ options ] input output
Convert a qcow image to vhd successfully.
options:
--resize SIZE grow the disk to SIZE during conversion
--no-fixed output image in 'Dynamic' format
EOF
}
bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }
cleanup() {
[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
debug() {
local level=${1}; shift;
[ "${level}" -gt "${VERBOSITY}" ] && return
error "${@}"
}
short_opts="hr:v"
long_opts="help,no-fixed,resize:,verbose"
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
bad_Usage
## <<insert default variables here>>
input=""
output=""
resize=""
fixed=true
while [ $# -ne 0 ]; do
cur=$1; next=$2;
case "$cur" in
-h|--help) Usage ; exit 0;;
-v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
-r|--resize) resize=$next;;
--no-fixed) fixed=false;;
--) shift; break;;
esac
shift;
done
[ $# -eq 2 ] ||
bad_Usage "Expect 2 arguments only (input, output). recieved $#"
input="$1"
output="$2"
[ -f "$input" ] || fail "$input: not a file"
rm -f "$output" || fail "failed to delete output file: $output"
checkreqs || fail
TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
fail "failed to make tempdir"
trap cleanup EXIT
out=$(LANG=C qemu-img info "$input") ||
fail "failed to get image info for $input"
itype=$(echo "$out" | awk '$0 ~ /^file format:/ { print $3 }') &&
[ -n "$itype" ] || fail "failed to get input type"
isize=$(echo "$out" | awk '$0 ~ /^virtual size:/ { print $4 }') &&
[ -n "$isize" ] && isize="${isize#(}" || fail "failed to get input type"
debug 1 "$input: $itype/$isize"
img_raw="${TEMP_D}/raw.img"
img_vhd="${TEMP_D}/vhd.img"
if [ "$itype" = "raw" ]; then
if [ -n "$resize" ]; then
debug 1 "copying input to temp for resize to $resize"
img_raw="${TEMP_D}/raw.img"
cp "$itype" "$img_raw" ||
fail "failed copy to temp"
else
ln -s "$(readlink -f "$input")" "$img_raw" ||
fail "failed to link to $input"
fi
else
debug 1 "converting input to raw"
rq 3 convert-to-raw qemu-img convert -O raw "$input" "$img_raw" ||
fail "failed conversion to raw"
fi
if [ -n "$resize" ]; then
truncate --size "$resize" "$img_raw" ||
fail "failed truncate --size $resize"
fi
$fixed && target="$img_vhd" || target="$output"
debug 1 "converting raw to VHD-dynamic"
rq 3 to-vhd-dyanmic vboxmanage convertfromraw \
--format VHD "$img_raw" "$target" ||
fail "failed convert from raw to vhd"
if ! $fixed; then
debug 1 "created $output in vhd format"
fi
debug 1 "converting vhd to vhd-fixed"
rq 3 to-vhd-fixed vboxmanage clonehd --format VHD \
--variant Fixed "$img_vhd" "$output" ||
fail "failed convert from vhd to vhd-fixed"
debug 1 "created $output in vhd fixed format"
exit
# vi: ts=4 noexpandtab
#!/bin/sh
input="/tmp/input.img"
output="/tmp/output.img"
outd="out.d"
mkdir -p "$outd"
uuid="b15256c1-70fb-45df-8728-77cf75bffd92"
rm out.info.log
for format in VDI VMDK VHD; do
for variant in Standard Fixed Split2G Stream ESX ""; do
rm -f "$input" "$output"
vboxmanage closemedium disk --delete "$output" >/dev/null 2>&1
[ "$variant" = "" ] && ovariant="none" || ovariant="$variant"
echo "=== $format $variant ==="
truncate --size 50M "$input"
vboxmanage convertfromraw --format=$format ${variant:+--variant=${variant}} --uuid=$uuid $input $output > "$outd/$format.$ovariant.out" 2>&1
ret="$?"
if [ "$ret" -ne 0 ]; then
echo "Failed $outd/$format.$ovariant"
continue
fi
vboxmanage showhdinfo "$output" | tee "$outd/$format.$ovariant.info"
echo "md5sum: $(md5sum "$output")"
info=$(awk -F: '$1 == "Format variant" { print $2 }' "$outd/$format.$ovariant.info")
echo "$format-$ovariant:" $info | tee -a out.info.log
cp "$output" "$outd/$format.$ovariant.img"
vboxmanage closemedium disk --delete "$output" >/dev/null 2>&1
done
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment