Skip to content

Instantly share code, notes, and snippets.

@victortoso
Created May 14, 2024 10:25
Show Gist options
  • Save victortoso/4a54ef0e75a2b917bb21cb33ec4da141 to your computer and use it in GitHub Desktop.
Save victortoso/4a54ef0e75a2b917bb21cb33ec4da141 to your computer and use it in GitHub Desktop.
Example of adding QEMU command line args with sidecar + configmap + python script
apiVersion: v1
kind: ConfigMap
metadata:
name: my-config-map
data:
my_script.sh: |
#!/usr/bin/env python3
import xml.etree.ElementTree as ET
import sys
import json
ANNOTATION_ARGS_KEY = "libvirt.vm.kubevirt.io/qemuArgs"
XMLNS_QEMU = "http://libvirt.org/schemas/domain/qemu/1.0"
def get_annotation(vmijson):
vmi = json.loads(vmijson)
if "metadata" not in vmi:
sys.stderr.write("missing metadata\n")
return None, None
elif "annotations" not in vmi["metadata"]:
sys.stderr.write("missing annotations\n")
return None, None
elif ANNOTATION_ARGS_KEY not in vmi["metadata"]["annotations"]:
sys.stderr.write("missing " + ANNOTATION_ARGS_KEY + "\n")
return None, None
qemuArgs = json.loads(vmi["metadata"]["annotations"][ANNOTATION_ARGS_KEY])
env, args = {}, []
for it in qemuArgs["env"]:
name = it["name"]
value = "" if "value" not in it else it["value"]
env[name] = value
for it in qemuArgs["arg"]:
args.append(it["value"])
sys.stderr.write(
f"qemu-args: Found {len(env)} env vars and {len(args)} args in {ANNOTATION_ARGS_KEY}"
)
return env, args
def current_qemu_env_and_args(domainxml):
root = ET.fromstring(domainxml)
cmdline = root.find("qemu:commandline")
if cmdline is None:
return {}, []
env, args = {}, []
for it in cmdline.findall("qemu:env"):
name = it.attrib.get("name")
value = it.attrib.get("value", "")
env[name] = value
for it in cmdline.findall("qemu:arg"):
value = it.attrib.get("value")
args.append(value)
sys.stderr.write(
f"qemu-args: already set commandline:\n{ET.tostring(cmdline)}\n"
)
return env, args
# Just set annotations to the domainxml if they were not yet set
def handle_qemu_args(domainxml, ann_env, ann_arg):
env, arg = current_qemu_env_and_args(domainxml)
# Add enviroment variables from annotation to existing qemu cmd envs
for name, value in ann_env.items():
if name in env and value != env[name]:
sys.stderr.write(
"env {name} exists: replace '{env[name]}' with '{value}'\n"
)
env[name] = value
# skip if all elements in annotation are already present
if not set(ann_arg).issubset(arg):
# Add arg params from annotation to existing qemu args
for it in ann_arg:
if it in arg and not it.startswith("-"):
sys.stderr.write(f"arg {it.name} exists: skip it'\n")
continue
arg.append(it)
root = ET.fromstring(domainxml)
root.set("xmlns:qemu", XMLNS_QEMU)
# remove it as we are going to add a new one.
cmdline = root.find("qemu:commandline")
if cmdline is not None:
root.remove(cmdline)
cmdline = ET.SubElement(root, "qemu:commandline")
for name, value in env.items():
sub = ET.SubElement(cmdline, "qemu:env", name=name)
if value != "":
sub.set("value", value)
for it in arg:
ET.SubElement(cmdline, "qemu:arg", value=it)
ET.dump(root)
if __name__ == "__main__":
# input: --vmi "json-string" --domain "xml-string"
vmijson, domainxml = sys.argv[2], sys.argv[4]
env, arg = get_annotation(vmijson)
if env is None:
# no annotations, so provide input as result
# stdout is what sidecar-shim is expected
print(domainxml)
else:
handle_qemu_args(domainxml, env, arg)
import xml.etree.ElementTree as ET
import sys
import json
ANNOTATION_ARGS_KEY = "libvirt.vm.kubevirt.io/qemuArgs"
XMLNS_QEMU = "http://libvirt.org/schemas/domain/qemu/1.0"
def get_annotation(vmijson):
vmi = json.loads(vmijson)
if "metadata" not in vmi:
sys.stderr.write("missing metadata\n")
return None, None
elif "annotations" not in vmi["metadata"]:
sys.stderr.write("missing annotations\n")
return None, None
elif ANNOTATION_ARGS_KEY not in vmi["metadata"]["annotations"]:
sys.stderr.write("missing " + ANNOTATION_ARGS_KEY + "\n")
return None, None
qemuArgs = json.loads(vmi["metadata"]["annotations"][ANNOTATION_ARGS_KEY])
env, args = {}, []
for it in qemuArgs["env"]:
name = it["name"]
value = "" if "value" not in it else it["value"]
env[name] = value
for it in qemuArgs["arg"]:
args.append(it["value"])
sys.stderr.write(
f"qemu-args: Found {len(env)} env vars and {len(args)} args in {ANNOTATION_ARGS_KEY}\n"
)
return env, args
def current_qemu_env_and_args(domainxml):
root = ET.fromstring(domainxml)
cmdline = root.find("qemu:commandline")
if cmdline is None:
return {}, []
env, args = {}, []
for it in cmdline.findall("qemu:env"):
name = it.attrib.get("name")
value = it.attrib.get("value", "")
env[name] = value
for it in cmdline.findall("qemu:arg"):
value = it.attrib.get("value")
args.append(value)
sys.stderr.write(
f"qemu-args: already set commandline:\n{ET.tostring(cmdline)}\n"
)
return env, args
# Just set annotations to the domainxml if they were not yet set
def handle_qemu_args(domainxml, ann_env, ann_arg):
env, arg = current_qemu_env_and_args(domainxml)
# Add enviroment variables from annotation to existing qemu cmd envs
for name, value in ann_env.items():
if name in env and value != env[name]:
sys.stderr.write(
"env {name} exists: replace '{env[name]}' with '{value}'\n"
)
env[name] = value
# skip if all elements in annotation are already present
if not set(ann_arg).issubset(arg):
# Add arg params from annotation to existing qemu args
for it in ann_arg:
if it in arg and not it.startswith("-"):
sys.stderr.write(f"arg {it.name} exists: skip it'\n")
continue
arg.append(it)
root = ET.fromstring(domainxml)
root.set("xmlns:qemu", XMLNS_QEMU)
# remove it as we are going to add a new one.
cmdline = root.find("qemu:commandline")
if cmdline is not None:
root.remove(cmdline)
cmdline = ET.SubElement(root, "qemu:commandline")
for name, value in env.items():
sub = ET.SubElement(cmdline, "qemu:env", name=name)
if value != "":
sub.set("value", value)
for it in arg:
ET.SubElement(cmdline, "qemu:arg", value=it)
ET.dump(root)
if __name__ == "__main__":
# input: --vmi "json-string" --domain "xml-string"
vmijson, domainxml = sys.argv[2], sys.argv[4]
env, arg = get_annotation(vmijson)
if env is None:
# no annotations, so provide input as result
# stdout is what sidecar-shim is expected
print(domainxml)
else:
handle_qemu_args(domainxml, env, arg)
Authorization not available. Check if polkit service is running or see debug message for more information.
<domain type='kvm' id='1' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
<name>default_vmi-with-sidecar-hook-configmap</name>
<uuid>51be4432-b6db-4987-bb07-55934e57c000</uuid>
<metadata xmlns:ns0="http://kubevirt.io">
<ns0:kubevirt>
<ns0:uid/>
</ns0:kubevirt>
</metadata>
<memory unit='KiB'>1000448</memory>
<currentMemory unit='KiB'>1000000</currentMemory>
<vcpu placement='static'>1</vcpu>
<iothreads>1</iothreads>
<sysinfo type='smbios'>
<system>
<entry name='manufacturer'>KubeVirt</entry>
<entry name='product'>None</entry>
<entry name='uuid'>51be4432-b6db-4987-bb07-55934e57c000</entry>
<entry name='family'>KubeVirt</entry>
</system>
</sysinfo>
<os>
<type arch='x86_64' machine='pc-q35-rhel9.4.0'>hvm</type>
<boot dev='hd'/>
<smbios mode='sysinfo'/>
</os>
<features>
<acpi/>
</features>
<cpu mode='custom' match='exact' check='full'>
<model fallback='forbid'>Skylake-Client-IBRS</model>
<vendor>Intel</vendor>
<topology sockets='1' dies='1' clusters='1' cores='1' threads='1'/>
<feature policy='require' name='ss'/>
<feature policy='require' name='vmx'/>
<feature policy='require' name='pdcm'/>
<feature policy='require' name='hypervisor'/>
<feature policy='require' name='tsc_adjust'/>
<feature policy='require' name='clflushopt'/>
<feature policy='require' name='umip'/>
<feature policy='require' name='md-clear'/>
<feature policy='require' name='stibp'/>
<feature policy='require' name='flush-l1d'/>
<feature policy='require' name='arch-capabilities'/>
<feature policy='require' name='ssbd'/>
<feature policy='require' name='xsaves'/>
<feature policy='require' name='pdpe1gb'/>
<feature policy='require' name='ibpb'/>
<feature policy='require' name='ibrs'/>
<feature policy='require' name='amd-stibp'/>
<feature policy='require' name='amd-ssbd'/>
<feature policy='require' name='rdctl-no'/>
<feature policy='require' name='ibrs-all'/>
<feature policy='require' name='skip-l1dfl-vmentry'/>
<feature policy='require' name='mds-no'/>
<feature policy='require' name='pschange-mc-no'/>
<feature policy='require' name='tsx-ctrl'/>
<feature policy='require' name='fb-clear'/>
<feature policy='require' name='gds-no'/>
<feature policy='require' name='vmx-ins-outs'/>
<feature policy='require' name='vmx-true-ctls'/>
<feature policy='require' name='vmx-store-lma'/>
<feature policy='require' name='vmx-activity-hlt'/>
<feature policy='require' name='vmx-activity-wait-sipi'/>
<feature policy='require' name='vmx-vmwrite-vmexit-fields'/>
<feature policy='require' name='vmx-apicv-xapic'/>
<feature policy='require' name='vmx-ept'/>
<feature policy='require' name='vmx-desc-exit'/>
<feature policy='require' name='vmx-rdtscp-exit'/>
<feature policy='require' name='vmx-apicv-x2apic'/>
<feature policy='require' name='vmx-vpid'/>
<feature policy='require' name='vmx-wbinvd-exit'/>
<feature policy='require' name='vmx-unrestricted-guest'/>
<feature policy='require' name='vmx-rdrand-exit'/>
<feature policy='require' name='vmx-invpcid-exit'/>
<feature policy='require' name='vmx-vmfunc'/>
<feature policy='require' name='vmx-shadow-vmcs'/>
<feature policy='require' name='vmx-rdseed-exit'/>
<feature policy='require' name='vmx-pml'/>
<feature policy='require' name='vmx-xsaves'/>
<feature policy='require' name='vmx-invvpid'/>
<feature policy='require' name='vmx-invvpid-single-addr'/>
<feature policy='require' name='vmx-invvpid-all-context'/>
<feature policy='require' name='vmx-ept-execonly'/>
<feature policy='require' name='vmx-page-walk-4'/>
<feature policy='require' name='vmx-ept-2mb'/>
<feature policy='require' name='vmx-ept-1gb'/>
<feature policy='require' name='vmx-invept'/>
<feature policy='require' name='vmx-eptad'/>
<feature policy='require' name='vmx-invept-single-context'/>
<feature policy='require' name='vmx-invept-all-context'/>
<feature policy='require' name='vmx-intr-exit'/>
<feature policy='require' name='vmx-nmi-exit'/>
<feature policy='require' name='vmx-vnmi'/>
<feature policy='require' name='vmx-preemption-timer'/>
<feature policy='require' name='vmx-vintr-pending'/>
<feature policy='require' name='vmx-tsc-offset'/>
<feature policy='require' name='vmx-hlt-exit'/>
<feature policy='require' name='vmx-invlpg-exit'/>
<feature policy='require' name='vmx-mwait-exit'/>
<feature policy='require' name='vmx-rdpmc-exit'/>
<feature policy='require' name='vmx-rdtsc-exit'/>
<feature policy='require' name='vmx-cr3-load-noexit'/>
<feature policy='require' name='vmx-cr3-store-noexit'/>
<feature policy='require' name='vmx-cr8-load-exit'/>
<feature policy='require' name='vmx-cr8-store-exit'/>
<feature policy='require' name='vmx-flexpriority'/>
<feature policy='require' name='vmx-vnmi-pending'/>
<feature policy='require' name='vmx-movdr-exit'/>
<feature policy='require' name='vmx-io-exit'/>
<feature policy='require' name='vmx-io-bitmap'/>
<feature policy='require' name='vmx-mtf'/>
<feature policy='require' name='vmx-msr-bitmap'/>
<feature policy='require' name='vmx-monitor-exit'/>
<feature policy='require' name='vmx-pause-exit'/>
<feature policy='require' name='vmx-secondary-ctls'/>
<feature policy='require' name='vmx-exit-nosave-debugctl'/>
<feature policy='require' name='vmx-exit-load-perf-global-ctrl'/>
<feature policy='require' name='vmx-exit-ack-intr'/>
<feature policy='require' name='vmx-exit-save-pat'/>
<feature policy='require' name='vmx-exit-load-pat'/>
<feature policy='require' name='vmx-exit-save-efer'/>
<feature policy='require' name='vmx-exit-load-efer'/>
<feature policy='require' name='vmx-exit-save-preemption-timer'/>
<feature policy='disable' name='vmx-exit-clear-bndcfgs'/>
<feature policy='require' name='vmx-entry-noload-debugctl'/>
<feature policy='require' name='vmx-entry-ia32e-mode'/>
<feature policy='require' name='vmx-entry-load-perf-global-ctrl'/>
<feature policy='require' name='vmx-entry-load-pat'/>
<feature policy='require' name='vmx-entry-load-efer'/>
<feature policy='disable' name='vmx-entry-load-bndcfgs'/>
<feature policy='require' name='vmx-eptp-switching'/>
<feature policy='disable' name='hle'/>
<feature policy='disable' name='rtm'/>
<feature policy='disable' name='mpx'/>
</cpu>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<devices>
<emulator>/usr/libexec/qemu-kvm</emulator>
<disk type='file' device='disk' model='virtio-non-transitional'>
<driver name='qemu' type='qcow2' cache='none' error_policy='stop' discard='unmap'/>
<source file='/var/run/kubevirt-ephemeral-disks/disk-data/containerdisk/disk.qcow2' index='2'/>
<backingStore type='file' index='3'>
<format type='qcow2'/>
<source file='/var/run/kubevirt/container-disks/disk_0.img'/>
</backingStore>
<target dev='vda' bus='virtio'/>
<alias name='ua-containerdisk'/>
<address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
</disk>
<disk type='file' device='disk' model='virtio-non-transitional'>
<driver name='qemu' type='raw' cache='none' error_policy='stop' discard='unmap'/>
<source file='/var/run/kubevirt-ephemeral-disks/cloud-init-data/default/vmi-with-sidecar-hook-configmap/noCloud.iso' index='1'/>
<backingStore/>
<target dev='vdb' bus='virtio'/>
<alias name='ua-cloudinitdisk'/>
<address type='pci' domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
</disk>
<controller type='usb' index='0' model='none'>
<alias name='usb'/>
</controller>
<controller type='scsi' index='0' model='virtio-non-transitional'>
<alias name='scsi0'/>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</controller>
<controller type='virtio-serial' index='0' model='virtio-non-transitional'>
<alias name='virtio-serial0'/>
<address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
</controller>
<controller type='sata' index='0'>
<alias name='ide'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pcie-root'>
<alias name='pcie.0'/>
</controller>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<alias name='pci.1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<alias name='pci.2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x12'/>
<alias name='pci.3'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0x13'/>
<alias name='pci.4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0x14'/>
<alias name='pci.5'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0x15'/>
<alias name='pci.6'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
</controller>
<controller type='pci' index='7' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='7' port='0x16'/>
<alias name='pci.7'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x6'/>
</controller>
<controller type='pci' index='8' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='8' port='0x17'/>
<alias name='pci.8'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x7'/>
</controller>
<controller type='pci' index='9' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='9' port='0x18'/>
<alias name='pci.9'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='10' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='10' port='0x19'/>
<alias name='pci.10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x1'/>
</controller>
<controller type='pci' index='11' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='11' port='0x1a'/>
<alias name='pci.11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x2'/>
</controller>
<interface type='ethernet'>
<mac address='12:bf:52:ff:1f:1b'/>
<target dev='tap0' managed='no'/>
<model type='virtio-non-transitional'/>
<mtu size='1480'/>
<alias name='ua-default'/>
<rom enabled='no'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</interface>
<serial type='unix'>
<source mode='bind' path='/var/run/kubevirt-private/4ead3920-a1e2-40cb-9402-76342cd0e1d6/virt-serial0'/>
<log file='/var/run/kubevirt-private/4ead3920-a1e2-40cb-9402-76342cd0e1d6/virt-serial0-log' append='on'/>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
<alias name='serial0'/>
</serial>
<console type='unix'>
<source mode='bind' path='/var/run/kubevirt-private/4ead3920-a1e2-40cb-9402-76342cd0e1d6/virt-serial0'/>
<log file='/var/run/kubevirt-private/4ead3920-a1e2-40cb-9402-76342cd0e1d6/virt-serial0-log' append='on'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
</console>
<channel type='unix'>
<source mode='bind' path='/var/run/libvirt/qemu/run/channel/1-default_vmi-with-sid/org.qemu.guest_agent.0'/>
<target type='virtio' name='org.qemu.guest_agent.0' state='connected'/>
<alias name='channel0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='mouse' bus='ps2'>
<alias name='input0'/>
</input>
<input type='keyboard' bus='ps2'>
<alias name='input1'/>
</input>
<graphics type='vnc' socket='/var/run/kubevirt-private/4ead3920-a1e2-40cb-9402-76342cd0e1d6/virt-vnc'>
<listen type='socket' socket='/var/run/kubevirt-private/4ead3920-a1e2-40cb-9402-76342cd0e1d6/virt-vnc'/>
</graphics>
<audio id='1' type='none'/>
<video>
<model type='vga' vram='16384' heads='1' primary='yes'/>
<alias name='video0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
</video>
<watchdog model='itco' action='reset'>
<alias name='watchdog0'/>
</watchdog>
<memballoon model='virtio-non-transitional' freePageReporting='on'>
<stats period='10'/>
<alias name='balloon0'/>
<address type='pci' domain='0x0000' bus='0x09' slot='0x00' function='0x0'/>
</memballoon>
<rng model='virtio-non-transitional'>
<backend model='random'>/dev/urandom</backend>
<alias name='rng0'/>
<address type='pci' domain='0x0000' bus='0x0a' slot='0x00' function='0x0'/>
</rng>
</devices>
<qemu:commandline>
<qemu:arg value='-name'/>
<qemu:arg value='guest=sidecarKubevirtVM,debug-threads=on'/>
<qemu:env name='G_MESSAGES_DEBUG' value='all'/>
</qemu:commandline>
</domain>
---
apiVersion: kubevirt.io/v1
kind: VirtualMachineInstance
metadata:
annotations:
libvirt.vm.kubevirt.io/qemuArgs: >
{
"env": [
{"name": "G_MESSAGES_DEBUG", "value": "all"}
],
"arg": [
{"value": "-name"},
{"value": "guest=sidecarKubevirtVM,debug-threads=on"}
]
}
hooks.kubevirt.io/hookSidecars: >
[
{
"args": ["--version", "v1alpha2"],
"image":"registry:5000/kubevirt/sidecar-shim:devel",
"configMap": {
"name": "my-config-map",
"key": "my_script.sh",
"hookPath": "/usr/bin/onDefineDomain"
}
}
]
labels:
special: vmi-with-sidecar-hook-configmap
name: vmi-with-sidecar-hook-configmap
spec:
domain:
devices:
disks:
- disk:
bus: virtio
name: containerdisk
- disk:
bus: virtio
name: cloudinitdisk
rng: {}
resources:
requests:
memory: 1024M
terminationGracePeriodSeconds: 0
volumes:
- containerDisk:
image: registry:5000/kubevirt/fedora-with-test-tooling-container-disk:devel
name: containerdisk
- cloudInitNoCloud:
userData: |-
#cloud-config
password: fedora
chpasswd: { expire: False }
name: cloudinitdisk
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment