fccd_orch_upf_log.err

time="2023-10-18T13:29:03-06:00" level=warning msg="containerd config version `1` has been deprecated and will be removed in containerd v2.0, please switch to version `2`, see https://github.com/containerd/containerd/blob/main/docs/PLUGINS.md#version-header"
time="2023-10-18T13:29:03.250772173-06:00" level=info msg="starting containerd" revision=09e74c6de86fb093077a8aed11ad35b612c35b6a version=1.6.20+unknown
time="2023-10-18T13:29:03.272993881-06:00" level=info msg="loading plugin \"io.containerd.content.v1.content\"..." type=io.containerd.content.v1
time="2023-10-18T13:29:03.273097784-06:00" level=info msg="loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." type=io.containerd.snapshotter.v1
time="2023-10-18T13:29:03.273150551-06:00" level=info msg="initializing pool device \"fc-dev-thinpool\""
time="2023-10-18T13:29:03.274583053-06:00" level=info msg="using dmsetup:\nLibrary version:   1.02.167 (2019-11-30)\nDriver version:    4.41.0"
time="2023-10-18T13:29:03.277705526-06:00" level=info msg="loading plugin \"io.containerd.snapshotter.v1.overlayfs\"..." type=io.containerd.snapshotter.v1
time="2023-10-18T13:29:03.277900499-06:00" level=info msg="loading plugin \"io.containerd.metadata.v1.bolt\"..." type=io.containerd.metadata.v1
time="2023-10-18T13:29:03.277945157-06:00" level=info msg="metadata content store policy set" policy=shared
time="2023-10-18T13:29:03.278865478-06:00" level=info msg="loading plugin \"io.containerd.differ.v1.walking\"..." type=io.containerd.differ.v1
time="2023-10-18T13:29:03.278891295-06:00" level=info msg="loading plugin \"io.containerd.event.v1.exchange\"..." type=io.containerd.event.v1
time="2023-10-18T13:29:03.278905086-06:00" level=info msg="loading plugin \"io.containerd.gc.v1.scheduler\"..." type=io.containerd.gc.v1
time="2023-10-18T13:29:03.278943636-06:00" level=info msg="loading plugin \"io.containerd.service.v1.introspection-service\"..." type=io.containerd.service.v1
time="2023-10-18T13:29:03.278962592-06:00" level=info msg="loading plugin \"io.containerd.service.v1.containers-service\"..." type=io.containerd.service.v1
time="2023-10-18T13:29:03.278977671-06:00" level=info msg="loading plugin \"io.containerd.service.v1.content-service\"..." type=io.containerd.service.v1
time="2023-10-18T13:29:03.278992211-06:00" level=info msg="loading plugin \"io.containerd.service.v1.diff-service\"..." type=io.containerd.service.v1
time="2023-10-18T13:29:03.279017038-06:00" level=info msg="loading plugin \"io.containerd.service.v1.images-service\"..." type=io.containerd.service.v1
time="2023-10-18T13:29:03.279044614-06:00" level=info msg="loading plugin \"io.containerd.service.v1.leases-service\"..." type=io.containerd.service.v1
time="2023-10-18T13:29:03.279071235-06:00" level=info msg="loading plugin \"io.containerd.service.v1.namespaces-service\"..." type=io.containerd.service.v1
time="2023-10-18T13:29:03.279091113-06:00" level=info msg="loading plugin \"io.containerd.service.v1.snapshots-service\"..." type=io.containerd.service.v1
time="2023-10-18T13:29:03.279104297-06:00" level=info msg="loading plugin \"io.containerd.runtime.v1.linux\"..." type=io.containerd.runtime.v1
time="2023-10-18T13:29:03.279184974-06:00" level=info msg="loading plugin \"io.containerd.runtime.v2.task\"..." type=io.containerd.runtime.v2
time="2023-10-18T13:29:03.279252663-06:00" level=info msg="loading plugin \"io.containerd.monitor.v1.cgroups\"..." type=io.containerd.monitor.v1
time="2023-10-18T13:29:03.279588856-06:00" level=info msg="loading plugin \"io.containerd.service.v1.tasks-service\"..." type=io.containerd.service.v1
time="2023-10-18T13:29:03.279612336-06:00" level=debug msg="No RDT config file specified, RDT not configured"
time="2023-10-18T13:29:03.279626865-06:00" level=info msg="loading plugin \"io.containerd.service.v1.fc-control\"..." type=io.containerd.service.v1
time="2023-10-18T13:29:03.279639637-06:00" level=debug msg="initializing fc-control plugin (root: \"/var/lib/firecracker-containerd/containerd/io.containerd.service.v1.fc-control\")"
time="2023-10-18T13:29:03.300195189-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300259377-06:00" level=info msg="loading plugin \"io.containerd.internal.v1.restart\"..." type=io.containerd.internal.v1
time="2023-10-18T13:29:03.300348570-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.containers\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300380347-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.content\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300401863-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.diff\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300421804-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.events\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300443944-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.healthcheck\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300464562-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.images\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300484115-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.leases\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300503360-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.namespaces\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300525209-06:00" level=info msg="loading plugin \"io.containerd.internal.v1.opt\"..." type=io.containerd.internal.v1
time="2023-10-18T13:29:03.300586873-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.snapshots\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300619121-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.tasks\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300640095-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.version\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300659005-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.fc-control-service\"..." type=io.containerd.grpc.v1
time="2023-10-18T13:29:03.300674158-06:00" level=debug msg="initializing fc-control-service plugin"
time="2023-10-18T13:29:03.300891877-06:00" level=info msg=serving... address=/run/firecracker-containerd/containerd.sock.ttrpc
time="2023-10-18T13:29:03.300983058-06:00" level=info msg=serving... address=/run/firecracker-containerd/containerd.sock
time="2023-10-18T13:29:03.301058154-06:00" level=debug msg="sd notification" error="<nil>" notified=false state="READY=1"
time="2023-10-18T13:29:03.301082569-06:00" level=info msg="containerd successfully booted in 0.051325s"
time="2023-10-18T13:29:03.380437238-06:00" level=debug msg="garbage collected" d="341.363µs"
time="2023-10-18T13:29:05.825726291-06:00" level=debug msg="(*service).Write started" expected="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" ref="manifest-sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" total=1998
time="2023-10-18T13:29:06.015188134-06:00" level=debug msg="(*service).Write started" expected="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734" ref="config-sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734" total=8312
time="2023-10-18T13:29:06.249210464-06:00" level=debug msg="stat snapshot" key="sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10"
time="2023-10-18T13:29:06.250684562-06:00" level=debug msg="prepare snapshot" key="extract-249767592-Xw-a sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10" parent=
time="2023-10-18T13:29:06.251221025-06:00" level=debug msg=prepare key="firecracker-containerd/1/extract-249767592-Xw-a sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10" parent=
time="2023-10-18T13:29:06.251905000-06:00" level=debug msg="creating new thin device 'fc-dev-thinpool-snap-1'"
time="2023-10-18T13:29:06.301017747-06:00" level=debug msg="Creating file system of type: ext4 with options: nodiscard,lazy_itable_init=0,lazy_journal_init=0 for thin device \"fc-dev-thinpool-snap-1\""
time="2023-10-18T13:29:06.301092679-06:00" level=debug msg="mkfs.ext4 -E nodiscard,lazy_itable_init=0,lazy_journal_init=0 /dev/mapper/fc-dev-thinpool-snap-1"
time="2023-10-18T13:29:07.165311462-06:00" level=debug msg="mkfs:\nmke2fs 1.45.5 (07-Jan-2020)\nCreating filesystem with 2621440 4k blocks and 655360 inodes\nFilesystem UUID: 2aa2eccf-732f-4820-8545-3810768015e1\nSuperblock backups stored on blocks: \n\t32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632\n\nAllocating group tables:  0/80\b\b\b\b\b     \b\b\b\b\bdone                            \nWriting inode tables:  0/80\b\b\b\b\b     \b\b\b\b\bdone                            \nCreating journal (16384 blocks): done\nWriting superblocks and filesystem accounting information:  0/80\b\b\b\b\b     \b\b\b\b\bdone\n\n"
time="2023-10-18T13:29:07.175844790-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:07.178429912-06:00" level=debug msg="(*service).Write started" expected="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" ref="layer-sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" total=2802957
time="2023-10-18T13:29:07.179244640-06:00" level=debug msg="(*service).Write started" expected="sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" ref="layer-sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" total=2233
time="2023-10-18T13:29:07.179361951-06:00" level=debug msg="(*service).Write started" expected="sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" ref="layer-sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" total=31506523
time="2023-10-18T13:29:07.179438129-06:00" level=debug msg="(*service).Write started" expected="sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" ref="layer-sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" total=301261
time="2023-10-18T13:29:07.179519690-06:00" level=debug msg="(*service).Write started" expected="sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" ref="layer-sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" total=637463
time="2023-10-18T13:29:07.179538671-06:00" level=debug msg="(*service).Write started" expected="sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" ref="layer-sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" total=230
time="2023-10-18T13:29:07.182643627-06:00" level=debug msg="(*service).Write started" expected="sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" ref="layer-sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" total=1887472
time="2023-10-18T13:29:07.183322881-06:00" level=debug msg="(*service).Write started" expected="sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" ref="layer-sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" total=20365161
time="2023-10-18T13:29:07.664532186-06:00" level=debug msg="using pigz for decompression"
time="2023-10-18T13:29:07.836212908-06:00" level=debug msg="diff applied" d=171.802267ms digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" media=application/vnd.docker.image.rootfs.diff.tar.gzip size=2802957
time="2023-10-18T13:29:07.837761687-06:00" level=debug msg="commit snapshot" key="extract-249767592-Xw-a sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10" name="sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10"
time="2023-10-18T13:29:07.837946674-06:00" level=debug msg=commit key="firecracker-containerd/1/extract-249767592-Xw-a sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10" name="firecracker-containerd/2/sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10"
time="2023-10-18T13:29:07.916977350-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/commit type=containerd.events.SnapshotCommit
time="2023-10-18T13:29:07.919596785-06:00" level=debug msg="stat snapshot" key="sha256:5edfa66f961ecdadabac1b15441c567a06631fd4cb8a197a2f0399644a3c18d5"
time="2023-10-18T13:29:07.920626302-06:00" level=debug msg="prepare snapshot" key="extract-920062844-Yh1y sha256:5edfa66f961ecdadabac1b15441c567a06631fd4cb8a197a2f0399644a3c18d5" parent="sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10"
time="2023-10-18T13:29:07.921281633-06:00" level=debug msg=prepare key="firecracker-containerd/3/extract-920062844-Yh1y sha256:5edfa66f961ecdadabac1b15441c567a06631fd4cb8a197a2f0399644a3c18d5" parent="firecracker-containerd/2/sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10"
time="2023-10-18T13:29:07.921497809-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-2' from 'fc-dev-thinpool-snap-1' with fsType: 'ext4'"
time="2023-10-18T13:29:08.004714408-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:08.112951683-06:00" level=debug msg="diff applied" d=107.369458ms digest="sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" media=application/vnd.docker.image.rootfs.diff.tar.gzip size=301261
time="2023-10-18T13:29:08.114777934-06:00" level=debug msg="commit snapshot" key="extract-920062844-Yh1y sha256:5edfa66f961ecdadabac1b15441c567a06631fd4cb8a197a2f0399644a3c18d5" name="sha256:5edfa66f961ecdadabac1b15441c567a06631fd4cb8a197a2f0399644a3c18d5"
time="2023-10-18T13:29:08.114926744-06:00" level=debug msg=commit key="firecracker-containerd/3/extract-920062844-Yh1y sha256:5edfa66f961ecdadabac1b15441c567a06631fd4cb8a197a2f0399644a3c18d5" name="firecracker-containerd/4/sha256:5edfa66f961ecdadabac1b15441c567a06631fd4cb8a197a2f0399644a3c18d5"
time="2023-10-18T13:29:08.215560955-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/commit type=containerd.events.SnapshotCommit
time="2023-10-18T13:29:08.225415568-06:00" level=debug msg="stat snapshot" key="sha256:483d3bedbf62615f629bcfd167c0f2b45df7afe3ecb6b3f7ffa7ebe2dab70faa"
time="2023-10-18T13:29:08.227003004-06:00" level=debug msg="prepare snapshot" key="extract-225986568-lWbH sha256:483d3bedbf62615f629bcfd167c0f2b45df7afe3ecb6b3f7ffa7ebe2dab70faa" parent="sha256:5edfa66f961ecdadabac1b15441c567a06631fd4cb8a197a2f0399644a3c18d5"
time="2023-10-18T13:29:08.227434946-06:00" level=debug msg=prepare key="firecracker-containerd/5/extract-225986568-lWbH sha256:483d3bedbf62615f629bcfd167c0f2b45df7afe3ecb6b3f7ffa7ebe2dab70faa" parent="firecracker-containerd/4/sha256:5edfa66f961ecdadabac1b15441c567a06631fd4cb8a197a2f0399644a3c18d5"
time="2023-10-18T13:29:08.227503245-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-3' from 'fc-dev-thinpool-snap-2' with fsType: 'ext4'"
time="2023-10-18T13:29:08.311633511-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:10.426393951-06:00" level=debug msg="diff applied" d=1.860496697s digest="sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" media=application/vnd.docker.image.rootfs.diff.tar.gzip size=31506523
time="2023-10-18T13:29:10.427369579-06:00" level=debug msg="commit snapshot" key="extract-225986568-lWbH sha256:483d3bedbf62615f629bcfd167c0f2b45df7afe3ecb6b3f7ffa7ebe2dab70faa" name="sha256:483d3bedbf62615f629bcfd167c0f2b45df7afe3ecb6b3f7ffa7ebe2dab70faa"
time="2023-10-18T13:29:10.427493257-06:00" level=debug msg=commit key="firecracker-containerd/5/extract-225986568-lWbH sha256:483d3bedbf62615f629bcfd167c0f2b45df7afe3ecb6b3f7ffa7ebe2dab70faa" name="firecracker-containerd/6/sha256:483d3bedbf62615f629bcfd167c0f2b45df7afe3ecb6b3f7ffa7ebe2dab70faa"
time="2023-10-18T13:29:10.518471886-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/commit type=containerd.events.SnapshotCommit
time="2023-10-18T13:29:10.521398775-06:00" level=debug msg="stat snapshot" key="sha256:3adc9ed0c3e4c8e4fb839cc55b92b0fefb26459b3d03edcbe139e0bd097fca0f"
time="2023-10-18T13:29:10.522681509-06:00" level=debug msg="prepare snapshot" key="extract-522036214-TRAD sha256:3adc9ed0c3e4c8e4fb839cc55b92b0fefb26459b3d03edcbe139e0bd097fca0f" parent="sha256:483d3bedbf62615f629bcfd167c0f2b45df7afe3ecb6b3f7ffa7ebe2dab70faa"
time="2023-10-18T13:29:10.523271797-06:00" level=debug msg=prepare key="firecracker-containerd/7/extract-522036214-TRAD sha256:3adc9ed0c3e4c8e4fb839cc55b92b0fefb26459b3d03edcbe139e0bd097fca0f" parent="firecracker-containerd/6/sha256:483d3bedbf62615f629bcfd167c0f2b45df7afe3ecb6b3f7ffa7ebe2dab70faa"
time="2023-10-18T13:29:10.523400639-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-4' from 'fc-dev-thinpool-snap-3' with fsType: 'ext4'"
time="2023-10-18T13:29:10.580116537-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:10.591557953-06:00" level=debug msg="diff applied" d=10.164359ms digest="sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" media=application/vnd.docker.image.rootfs.diff.tar.gzip size=230
time="2023-10-18T13:29:10.592644528-06:00" level=debug msg="commit snapshot" key="extract-522036214-TRAD sha256:3adc9ed0c3e4c8e4fb839cc55b92b0fefb26459b3d03edcbe139e0bd097fca0f" name="sha256:3adc9ed0c3e4c8e4fb839cc55b92b0fefb26459b3d03edcbe139e0bd097fca0f"
time="2023-10-18T13:29:10.592820407-06:00" level=debug msg=commit key="firecracker-containerd/7/extract-522036214-TRAD sha256:3adc9ed0c3e4c8e4fb839cc55b92b0fefb26459b3d03edcbe139e0bd097fca0f" name="firecracker-containerd/8/sha256:3adc9ed0c3e4c8e4fb839cc55b92b0fefb26459b3d03edcbe139e0bd097fca0f"
time="2023-10-18T13:29:10.666251249-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/commit type=containerd.events.SnapshotCommit
time="2023-10-18T13:29:10.668965042-06:00" level=debug msg="stat snapshot" key="sha256:7893763263404b202f7a20649fdab037cf7200d38d5a5e6fc35ced2c072df270"
time="2023-10-18T13:29:10.670269974-06:00" level=debug msg="prepare snapshot" key="extract-669609120-fE17 sha256:7893763263404b202f7a20649fdab037cf7200d38d5a5e6fc35ced2c072df270" parent="sha256:3adc9ed0c3e4c8e4fb839cc55b92b0fefb26459b3d03edcbe139e0bd097fca0f"
time="2023-10-18T13:29:10.670952422-06:00" level=debug msg=prepare key="firecracker-containerd/9/extract-669609120-fE17 sha256:7893763263404b202f7a20649fdab037cf7200d38d5a5e6fc35ced2c072df270" parent="firecracker-containerd/8/sha256:3adc9ed0c3e4c8e4fb839cc55b92b0fefb26459b3d03edcbe139e0bd097fca0f"
time="2023-10-18T13:29:10.671068480-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-5' from 'fc-dev-thinpool-snap-4' with fsType: 'ext4'"
time="2023-10-18T13:29:10.725300326-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:10.978715553-06:00" level=debug msg="diff applied" d=252.245152ms digest="sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" media=application/vnd.docker.image.rootfs.diff.tar.gzip size=1887472
time="2023-10-18T13:29:10.979915466-06:00" level=debug msg="commit snapshot" key="extract-669609120-fE17 sha256:7893763263404b202f7a20649fdab037cf7200d38d5a5e6fc35ced2c072df270" name="sha256:7893763263404b202f7a20649fdab037cf7200d38d5a5e6fc35ced2c072df270"
time="2023-10-18T13:29:10.980164723-06:00" level=debug msg=commit key="firecracker-containerd/9/extract-669609120-fE17 sha256:7893763263404b202f7a20649fdab037cf7200d38d5a5e6fc35ced2c072df270" name="firecracker-containerd/10/sha256:7893763263404b202f7a20649fdab037cf7200d38d5a5e6fc35ced2c072df270"
time="2023-10-18T13:29:11.051551349-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/commit type=containerd.events.SnapshotCommit
time="2023-10-18T13:29:11.054510099-06:00" level=debug msg="stat snapshot" key="sha256:e4226272b56cdeb334d6c3377374a6760ceae26f704fddedd5ff871e52d19784"
time="2023-10-18T13:29:11.055718826-06:00" level=debug msg="prepare snapshot" key="extract-55100013-uwQH sha256:e4226272b56cdeb334d6c3377374a6760ceae26f704fddedd5ff871e52d19784" parent="sha256:7893763263404b202f7a20649fdab037cf7200d38d5a5e6fc35ced2c072df270"
time="2023-10-18T13:29:11.056859326-06:00" level=debug msg=prepare key="firecracker-containerd/11/extract-55100013-uwQH sha256:e4226272b56cdeb334d6c3377374a6760ceae26f704fddedd5ff871e52d19784" parent="firecracker-containerd/10/sha256:7893763263404b202f7a20649fdab037cf7200d38d5a5e6fc35ced2c072df270"
time="2023-10-18T13:29:11.056981010-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-6' from 'fc-dev-thinpool-snap-5' with fsType: 'ext4'"
time="2023-10-18T13:29:11.116112941-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:11.131019836-06:00" level=debug msg="diff applied" d=13.925687ms digest="sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" media=application/vnd.docker.image.rootfs.diff.tar.gzip size=2233
time="2023-10-18T13:29:11.132186849-06:00" level=debug msg="commit snapshot" key="extract-55100013-uwQH sha256:e4226272b56cdeb334d6c3377374a6760ceae26f704fddedd5ff871e52d19784" name="sha256:e4226272b56cdeb334d6c3377374a6760ceae26f704fddedd5ff871e52d19784"
time="2023-10-18T13:29:11.132351968-06:00" level=debug msg=commit key="firecracker-containerd/11/extract-55100013-uwQH sha256:e4226272b56cdeb334d6c3377374a6760ceae26f704fddedd5ff871e52d19784" name="firecracker-containerd/12/sha256:e4226272b56cdeb334d6c3377374a6760ceae26f704fddedd5ff871e52d19784"
time="2023-10-18T13:29:11.218369061-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/commit type=containerd.events.SnapshotCommit
time="2023-10-18T13:29:11.221498050-06:00" level=debug msg="stat snapshot" key="sha256:046f313f5e6a5160dd5c71cdf40aed78bb816bce9030b7b1617dc499a623dac8"
time="2023-10-18T13:29:11.222626098-06:00" level=debug msg="prepare snapshot" key="extract-222048883-0eLG sha256:046f313f5e6a5160dd5c71cdf40aed78bb816bce9030b7b1617dc499a623dac8" parent="sha256:e4226272b56cdeb334d6c3377374a6760ceae26f704fddedd5ff871e52d19784"
time="2023-10-18T13:29:11.223298088-06:00" level=debug msg=prepare key="firecracker-containerd/13/extract-222048883-0eLG sha256:046f313f5e6a5160dd5c71cdf40aed78bb816bce9030b7b1617dc499a623dac8" parent="firecracker-containerd/12/sha256:e4226272b56cdeb334d6c3377374a6760ceae26f704fddedd5ff871e52d19784"
time="2023-10-18T13:29:11.223427821-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-7' from 'fc-dev-thinpool-snap-6' with fsType: 'ext4'"
time="2023-10-18T13:29:11.276524503-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:12.162649490-06:00" level=debug msg="diff applied" d=885.039704ms digest="sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" media=application/vnd.docker.image.rootfs.diff.tar.gzip size=20365161
time="2023-10-18T13:29:12.164047719-06:00" level=debug msg="commit snapshot" key="extract-222048883-0eLG sha256:046f313f5e6a5160dd5c71cdf40aed78bb816bce9030b7b1617dc499a623dac8" name="sha256:046f313f5e6a5160dd5c71cdf40aed78bb816bce9030b7b1617dc499a623dac8"
time="2023-10-18T13:29:12.164258837-06:00" level=debug msg=commit key="firecracker-containerd/13/extract-222048883-0eLG sha256:046f313f5e6a5160dd5c71cdf40aed78bb816bce9030b7b1617dc499a623dac8" name="firecracker-containerd/14/sha256:046f313f5e6a5160dd5c71cdf40aed78bb816bce9030b7b1617dc499a623dac8"
time="2023-10-18T13:29:12.234555200-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/commit type=containerd.events.SnapshotCommit
time="2023-10-18T13:29:12.237794220-06:00" level=debug msg="stat snapshot" key="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:12.239068360-06:00" level=debug msg="prepare snapshot" key="extract-238363617-SYGF sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253" parent="sha256:046f313f5e6a5160dd5c71cdf40aed78bb816bce9030b7b1617dc499a623dac8"
time="2023-10-18T13:29:12.239669611-06:00" level=debug msg=prepare key="firecracker-containerd/15/extract-238363617-SYGF sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253" parent="firecracker-containerd/14/sha256:046f313f5e6a5160dd5c71cdf40aed78bb816bce9030b7b1617dc499a623dac8"
time="2023-10-18T13:29:12.239821507-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-8' from 'fc-dev-thinpool-snap-7' with fsType: 'ext4'"
time="2023-10-18T13:29:12.296325033-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:12.347519290-06:00" level=debug msg="diff applied" d=49.955385ms digest="sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" media=application/vnd.docker.image.rootfs.diff.tar.gzip size=637463
time="2023-10-18T13:29:12.348672385-06:00" level=debug msg="commit snapshot" key="extract-238363617-SYGF sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253" name="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:12.348898142-06:00" level=debug msg=commit key="firecracker-containerd/15/extract-238363617-SYGF sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253" name="firecracker-containerd/16/sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:12.434624753-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/commit type=containerd.events.SnapshotCommit
time="2023-10-18T13:29:12.442371556-06:00" level=debug msg="create image" name="ghcr.io/ease-lab/helloworld:var_workload" target="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25"
time="2023-10-18T13:29:12.443128315-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/images/create type=containerd.services.images.v1.ImageCreate
time="2023-10-18T13:29:12.453089981-06:00" level=debug msg="garbage collected" d="618.224µs"
time="2023-10-18T13:29:12.512830009-06:00" level=debug msg="create VM request: VMID:\"1-0\" MachineCfg:<MemSizeMib:256 VcpuCount:1 > KernelArgs:\"ro noapic reboot=k panic=1 pci=off nomodules systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init tsc=reliable quiet 8250.nr_uarts=0 ipv6.disable=1\" NetworkInterfaces:<StaticConfig:<MacAddress:\"AA:FC:00:00:00:01\" HostDevName:\"tap0\" IPConfig:<PrimaryAddr:\"172.16.0.2/24\" GatewayAddr:\"172.16.0.1\" Nameservers:\"8.8.8.8\" > > > TimeoutSeconds:100 NetNS:\"/var/run/netns/uvmns87\" "
time="2023-10-18T13:29:12.512896809-06:00" level=debug msg="using namespace: firecracker-containerd"
time="2023-10-18T13:29:12.513289812-06:00" level=debug msg="starting containerd-shim-aws-firecracker" vmID=1-0
time="2023-10-18T13:29:12.553978965-06:00" level=info msg="loading plugin \"io.containerd.service.v1.introspection-service\"..." runtime=aws.firecracker type=io.containerd.service.v1
time="2023-10-18T13:29:12.554146731-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." runtime=aws.firecracker type=io.containerd.grpc.v1
time="2023-10-18T13:29:12.554195088-06:00" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=aws.firecracker type=io.containerd.event.v1
time="2023-10-18T13:29:12.554234925-06:00" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=aws.firecracker type=io.containerd.ttrpc.v1
time="2023-10-18T13:29:12.554322376-06:00" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=aws.firecracker type=io.containerd.internal.v1
time="2023-10-18T13:29:12.554576742-06:00" level=info msg="starting signal loop" namespace=firecracker-containerd path="/var/lib/firecracker-containerd/shim-base/firecracker-containerd#1-0" pid=238429 runtime=aws.firecracker
time="2023-10-18T13:29:12.554846350-06:00" level=info msg="creating new VM" runtime=aws.firecracker vmID=1-0
time="2023-10-18T13:29:12.556205555-06:00" level=info msg="Called startVMM(), setting up a VMM on firecracker.sock" runtime=aws.firecracker
time="2023-10-18T13:29:12.569025558-06:00" level=info msg="refreshMachineConfiguration: [GET /machine-config][200] getMachineConfigurationOK  &{CPUTemplate:T2 MemSizeMib:0xc0004f6958 Smt:0xc0004f6963 TrackDirtyPages:0xc0004f6966 VcpuCount:0xc0004f6950}" runtime=aws.firecracker
time="2023-10-18T13:29:12.569377038-06:00" level=info msg="PutGuestBootSource: [PUT /boot-source][204] putGuestBootSourceNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:12.569414418-06:00" level=info msg="Attaching drive /var/lib/firecracker-containerd/runtime/default-rootfs.img, slot root_drive, root true." runtime=aws.firecracker
time="2023-10-18T13:29:12.570005665-06:00" level=info msg="Attached drive /var/lib/firecracker-containerd/runtime/default-rootfs.img: [PUT /drives/{drive_id}][204] putGuestDriveByIdNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:12.570034613-06:00" level=info msg="Attaching drive /var/lib/firecracker-containerd/shim-base/firecracker-containerd#1-0/ctrstub0, slot MN2HE43UOVRDA, root false." runtime=aws.firecracker
time="2023-10-18T13:29:12.570292574-06:00" level=info msg="Attached drive /var/lib/firecracker-containerd/shim-base/firecracker-containerd#1-0/ctrstub0: [PUT /drives/{drive_id}][204] putGuestDriveByIdNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:12.570319409-06:00" level=info msg="Attaching NIC tap0 (hwaddr AA:FC:00:00:00:01) at index 1" runtime=aws.firecracker
time="2023-10-18T13:29:12.571555359-06:00" level=info msg="No interfaces are allowed to access MMDS, skipping MMDS config" runtime=aws.firecracker
time="2023-10-18T13:29:12.599064618-06:00" level=info msg="startInstance successful: [PUT /actions][204] createSyncActionNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:12.599099890-06:00" level=info msg="calling agent" runtime=aws.firecracker vmID=1-0
time="2023-10-18T13:29:14.300101623-06:00" level=info msg="successfully started the VM" runtime=aws.firecracker vmID=1-0
time="2023-10-18T13:29:14.300864973-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/firecracker-vm/start type=VMStart
time="2023-10-18T13:29:14.316298804-06:00" level=debug msg="prepare snapshot" key=vm1-0-containersnap-75fdc501-1d9c-4d parent="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:14.316920119-06:00" level=debug msg=prepare key=firecracker-containerd/17/vm1-0-containersnap-75fdc501-1d9c-4d parent="firecracker-containerd/16/sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:14.317057096-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-9' from 'fc-dev-thinpool-snap-8' with fsType: 'ext4'"
time="2023-10-18T13:29:14.375551492-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:14.389382830-06:00" level=debug msg="get snapshot mounts" key=vm1-0-containersnap-75fdc501-1d9c-4d
time="2023-10-18T13:29:14.389453526-06:00" level=debug msg=mounts key=firecracker-containerd/17/vm1-0-containersnap-75fdc501-1d9c-4d
time="2023-10-18T13:29:14.389546500-06:00" level=debug msg=stat key=firecracker-containerd/17/vm1-0-containersnap-75fdc501-1d9c-4d
time="2023-10-18T13:29:14.404813536-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/containers/create type=containerd.events.ContainerCreate
time="2023-10-18T13:29:14.409611639-06:00" level=debug msg="get snapshot mounts" key=vm1-0-containersnap-75fdc501-1d9c-4d
time="2023-10-18T13:29:14.409662991-06:00" level=debug msg=mounts key=firecracker-containerd/17/vm1-0-containersnap-75fdc501-1d9c-4d
time="2023-10-18T13:29:14.409744001-06:00" level=debug msg=stat key=firecracker-containerd/17/vm1-0-containersnap-75fdc501-1d9c-4d
time="2023-10-18T13:29:14.416743238-06:00" level=debug msg="garbage collected" d="469.224µs"
time="2023-10-18T13:29:14.452642860-06:00" level=debug msg=StartShim runtime=aws.firecracker task_id=vm1-0-containersnap-75fdc501-1d9c-4d
time="2023-10-18T13:29:14.453461580-06:00" level=debug msg="create VM request: VMID:\"1-0\" "
time="2023-10-18T13:29:14.453513712-06:00" level=debug msg="using namespace: firecracker-containerd"
time="2023-10-18T13:29:14.454043145-06:00" level=info msg="successfully started shim (git commit: 09e74c6de86fb093077a8aed11ad35b612c35b6a)." runtime=aws.firecracker task_id=vm1-0-containersnap-75fdc501-1d9c-4d vmID=1-0
time="2023-10-18T13:29:14.456565511-06:00" level=info msg="PatchGuestDrive successful" runtime=aws.firecracker
time="2023-10-18T13:29:14.565318407-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/tasks/create type=containerd.events.TaskCreate
time="2023-10-18T13:29:14.582998112-06:00" level=info msg="successfully created task" ExecID= TaskID=vm1-0-containersnap-75fdc501-1d9c-4d pid_in_vm=773 runtime=aws.firecracker vmID=1-0
time="2023-10-18T13:29:14.600235469-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/tasks/start type=containerd.events.TaskStart
time="2023-10-18T13:29:15.293047355-06:00" level=debug msg="pause VM request: VMID:\"1-0\" "
time="2023-10-18T13:29:15.295232714-06:00" level=debug msg="create snapshot request: VMID:\"1-0\" MemFilePath:\"/fccd/snapshots/1/mem_file\" SnapshotPath:\"/fccd/snapshots/1/snap_file\" "
time="2023-10-18T13:29:16.241389558-06:00" level=debug msg="get snapshot mounts" key=vm1-0-containersnap-75fdc501-1d9c-4d
time="2023-10-18T13:29:16.241470454-06:00" level=debug msg=mounts key=firecracker-containerd/17/vm1-0-containersnap-75fdc501-1d9c-4d
time="2023-10-18T13:29:16.241561641-06:00" level=debug msg=stat key=firecracker-containerd/17/vm1-0-containersnap-75fdc501-1d9c-4d
time="2023-10-18T13:29:16.253927011-06:00" level=debug msg="prepare snapshot" key=vm1-0-containersnap-75fdc501-1d9c-4d-base-image parent="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:16.254337589-06:00" level=debug msg=prepare key=firecracker-containerd/18/vm1-0-containersnap-75fdc501-1d9c-4d-base-image parent="firecracker-containerd/16/sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:16.254465147-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-10' from 'fc-dev-thinpool-snap-8' with fsType: 'ext4'"
time="2023-10-18T13:29:16.310590230-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:16.485287556-06:00" level=debug msg="remove snapshot" key=vm1-0-containersnap-75fdc501-1d9c-4d-base-image
time="2023-10-18T13:29:16.485803223-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/remove type=containerd.events.SnapshotRemove
time="2023-10-18T13:29:16.488715458-06:00" level=debug msg="resume VM request: VMID:\"1-0\" "
time="2023-10-18T13:29:16.492882698-06:00" level=debug msg="stop VM: VMID:\"1-0\" "
time="2023-10-18T13:29:16.494080616-06:00" level=info msg="GetInstanceInfo successful" runtime=aws.firecracker
time="2023-10-18T13:29:16.494145378-06:00" level=info msg="gracefully shutdown VM" runtime=aws.firecracker vmID=1-0
time="2023-10-18T13:29:16.494754631-06:00" level=error msg="error waiting for exit" ExecID= TaskID=vm1-0-containersnap-75fdc501-1d9c-4d error="ttrpc: closed" runtime=aws.firecracker vmID=1-0
time="2023-10-18T13:29:16.494882223-06:00" level=error msg="failed to call in-VM agent" error="ttrpc: closed" runtime=aws.firecracker vmID=1-0
time="2023-10-18T13:29:16.494922023-06:00" level=error msg="forcefully terminate VM 1-0" runtime=aws.firecracker vmID=1-0
time="2023-10-18T13:29:16.494927191-06:00" level=error msg="error copying io" ExecID= TaskID=vm1-0-containersnap-75fdc501-1d9c-4d error="reading from a closed fifo" runtime=aws.firecracker stream=stdin vmID=1-0
time="2023-10-18T13:29:16.495134022-06:00" level=error msg="error closing io stream" ExecID= TaskID=vm1-0-containersnap-75fdc501-1d9c-4d error="1 error occurred:\n\t* close unix @->firecracker.vsock: use of closed network connection\n\n" runtime=aws.firecracker stream=stdin vmID=1-0
time="2023-10-18T13:29:16.494940369-06:00" level=error msg="error while forwarding events from VM agent" error="ttrpc: closed" runtime=aws.firecracker vmID=1-0
time="2023-10-18T13:29:16.495688444-06:00" level=error msg="error returned from VM wait" error=shutdown runtime=aws.firecracker vmID=1-0
time="2023-10-18T13:29:16.495763770-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/firecracker-vm/stop type=VMStop
time="2023-10-18T13:29:16.498625186-06:00" level=debug msg="schedule snapshotter cleanup" snapshotter=devmapper
time="2023-10-18T13:29:16.498719113-06:00" level=debug msg=walk
time="2023-10-18T13:29:16.498895717-06:00" level=debug msg=remove key=firecracker-containerd/18/vm1-0-containersnap-75fdc501-1d9c-4d-base-image
time="2023-10-18T13:29:16.564651616-06:00" level=warning msg="firecracker exited: signal: killed" runtime=aws.firecracker
time="2023-10-18T13:29:16.610877284-06:00" level=debug msg="removed snapshot" key=firecracker-containerd/18/vm1-0-containersnap-75fdc501-1d9c-4d-base-image snapshotter=devmapper
time="2023-10-18T13:29:16.610920071-06:00" level=debug msg=cleanup
time="2023-10-18T13:29:16.610939931-06:00" level=debug msg="snapshot garbage collected" d=112.252045ms snapshotter=devmapper
time="2023-10-18T13:29:16.610983242-06:00" level=debug msg="garbage collected" d="521.134µs"
time="2023-10-18T13:29:21.495585587-06:00" level=error msg="error closing io stream" ExecID= TaskID=vm1-0-containersnap-75fdc501-1d9c-4d error="1 error occurred:\n\t* close unix @->firecracker.vsock: use of closed network connection\n\n" runtime=aws.firecracker stream=stderr vmID=1-0
time="2023-10-18T13:29:21.495627863-06:00" level=error msg="error closing io stream" ExecID= TaskID=vm1-0-containersnap-75fdc501-1d9c-4d error="1 error occurred:\n\t* close unix @->firecracker.vsock: use of closed network connection\n\n" runtime=aws.firecracker stream=stdout vmID=1-0
time="2023-10-18T13:29:21.498099733-06:00" level=debug msg="shim has been terminated" error="exit status 1" vmID=1-0
time="2023-10-18T13:29:21.498124331-06:00" level=info msg="shim disconnected" id=vm1-0-containersnap-75fdc501-1d9c-4d
time="2023-10-18T13:29:21.498396582-06:00" level=warning msg="cleaning up after shim disconnected" id=vm1-0-containersnap-75fdc501-1d9c-4d namespace=firecracker-containerd
time="2023-10-18T13:29:21.498423608-06:00" level=info msg="cleaning up dead shim"
time="2023-10-18T13:29:21.542509176-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/tasks/exit type=containerd.events.TaskExit
time="2023-10-18T13:29:21.542602705-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/tasks/delete type=containerd.events.TaskDelete
time="2023-10-18T13:29:21.610554745-06:00" level=debug msg="create VM request: VMID:\"2-0\" MachineCfg:<MemSizeMib:256 VcpuCount:1 > KernelArgs:\"ro noapic reboot=k panic=1 pci=off nomodules systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init tsc=reliable quiet 8250.nr_uarts=0 ipv6.disable=1\" NetworkInterfaces:<StaticConfig:<MacAddress:\"AA:FC:00:00:00:01\" HostDevName:\"tap0\" IPConfig:<PrimaryAddr:\"172.16.0.2/24\" GatewayAddr:\"172.16.0.1\" Nameservers:\"8.8.8.8\" > > > TimeoutSeconds:100 NetNS:\"/var/run/netns/uvmns87\" "
time="2023-10-18T13:29:21.610600995-06:00" level=debug msg="using namespace: firecracker-containerd"
time="2023-10-18T13:29:21.610840680-06:00" level=debug msg="starting containerd-shim-aws-firecracker" vmID=2-0
time="2023-10-18T13:29:21.628235996-06:00" level=info msg="loading plugin \"io.containerd.service.v1.introspection-service\"..." runtime=aws.firecracker type=io.containerd.service.v1
time="2023-10-18T13:29:21.628401437-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." runtime=aws.firecracker type=io.containerd.grpc.v1
time="2023-10-18T13:29:21.628445821-06:00" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=aws.firecracker type=io.containerd.event.v1
time="2023-10-18T13:29:21.628506509-06:00" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=aws.firecracker type=io.containerd.ttrpc.v1
time="2023-10-18T13:29:21.628554789-06:00" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=aws.firecracker type=io.containerd.internal.v1
time="2023-10-18T13:29:21.629060954-06:00" level=info msg="starting signal loop" namespace=firecracker-containerd path="/var/lib/firecracker-containerd/shim-base/firecracker-containerd#2-0" pid=238616 runtime=aws.firecracker
time="2023-10-18T13:29:21.629961593-06:00" level=info msg="creating new VM" runtime=aws.firecracker vmID=2-0
time="2023-10-18T13:29:21.630367289-06:00" level=info msg="Called startVMM(), setting up a VMM on firecracker.sock" runtime=aws.firecracker
time="2023-10-18T13:29:21.643000563-06:00" level=info msg="refreshMachineConfiguration: [GET /machine-config][200] getMachineConfigurationOK  &{CPUTemplate:T2 MemSizeMib:0xc0004307a8 Smt:0xc0004307b3 TrackDirtyPages:0xc0004307b6 VcpuCount:0xc0004307a0}" runtime=aws.firecracker
time="2023-10-18T13:29:21.643283138-06:00" level=info msg="PutGuestBootSource: [PUT /boot-source][204] putGuestBootSourceNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:21.643314034-06:00" level=info msg="Attaching drive /var/lib/firecracker-containerd/runtime/default-rootfs.img, slot root_drive, root true." runtime=aws.firecracker
time="2023-10-18T13:29:21.643835802-06:00" level=info msg="Attached drive /var/lib/firecracker-containerd/runtime/default-rootfs.img: [PUT /drives/{drive_id}][204] putGuestDriveByIdNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:21.643869080-06:00" level=info msg="Attaching drive /var/lib/firecracker-containerd/shim-base/firecracker-containerd#2-0/ctrstub0, slot MN2HE43UOVRDA, root false." runtime=aws.firecracker
time="2023-10-18T13:29:21.644128080-06:00" level=info msg="Attached drive /var/lib/firecracker-containerd/shim-base/firecracker-containerd#2-0/ctrstub0: [PUT /drives/{drive_id}][204] putGuestDriveByIdNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:21.644156674-06:00" level=info msg="Attaching NIC tap0 (hwaddr AA:FC:00:00:00:01) at index 1" runtime=aws.firecracker
time="2023-10-18T13:29:21.645278510-06:00" level=info msg="No interfaces are allowed to access MMDS, skipping MMDS config" runtime=aws.firecracker
time="2023-10-18T13:29:21.670107350-06:00" level=info msg="startInstance successful: [PUT /actions][204] createSyncActionNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:21.670177758-06:00" level=info msg="calling agent" runtime=aws.firecracker vmID=2-0
time="2023-10-18T13:29:23.471196539-06:00" level=info msg="successfully started the VM" runtime=aws.firecracker vmID=2-0
time="2023-10-18T13:29:23.471825311-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/firecracker-vm/start type=VMStart
time="2023-10-18T13:29:23.483000679-06:00" level=error msg="aws.firecracker: publisher not closed" shim_stream=stderr vmID=1-0
time="2023-10-18T13:29:23.487312457-06:00" level=debug msg="prepare snapshot" key=vm2-0-containersnap-9b8d4677-4c16-48 parent="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:23.487780172-06:00" level=debug msg=prepare key=firecracker-containerd/19/vm2-0-containersnap-9b8d4677-4c16-48 parent="firecracker-containerd/16/sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:23.487916090-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-11' from 'fc-dev-thinpool-snap-8' with fsType: 'ext4'"
time="2023-10-18T13:29:23.540964893-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:23.553820066-06:00" level=debug msg="get snapshot mounts" key=vm2-0-containersnap-9b8d4677-4c16-48
time="2023-10-18T13:29:23.553879577-06:00" level=debug msg=mounts key=firecracker-containerd/19/vm2-0-containersnap-9b8d4677-4c16-48
time="2023-10-18T13:29:23.554043429-06:00" level=debug msg=stat key=firecracker-containerd/19/vm2-0-containersnap-9b8d4677-4c16-48
time="2023-10-18T13:29:23.567302907-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/containers/create type=containerd.events.ContainerCreate
time="2023-10-18T13:29:23.572964778-06:00" level=debug msg="get snapshot mounts" key=vm2-0-containersnap-9b8d4677-4c16-48
time="2023-10-18T13:29:23.573014853-06:00" level=debug msg=mounts key=firecracker-containerd/19/vm2-0-containersnap-9b8d4677-4c16-48
time="2023-10-18T13:29:23.573081444-06:00" level=debug msg=stat key=firecracker-containerd/19/vm2-0-containersnap-9b8d4677-4c16-48
time="2023-10-18T13:29:23.578478906-06:00" level=debug msg="garbage collected" d="581.599µs"
time="2023-10-18T13:29:23.616530260-06:00" level=debug msg=StartShim runtime=aws.firecracker task_id=vm2-0-containersnap-9b8d4677-4c16-48
time="2023-10-18T13:29:23.617312133-06:00" level=debug msg="create VM request: VMID:\"2-0\" "
time="2023-10-18T13:29:23.617355100-06:00" level=debug msg="using namespace: firecracker-containerd"
time="2023-10-18T13:29:23.617731024-06:00" level=info msg="successfully started shim (git commit: 09e74c6de86fb093077a8aed11ad35b612c35b6a)." runtime=aws.firecracker task_id=vm2-0-containersnap-9b8d4677-4c16-48 vmID=2-0
time="2023-10-18T13:29:23.620028640-06:00" level=info msg="PatchGuestDrive successful" runtime=aws.firecracker
time="2023-10-18T13:29:23.729478735-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/tasks/create type=containerd.events.TaskCreate
time="2023-10-18T13:29:23.739776471-06:00" level=info msg="successfully created task" ExecID= TaskID=vm2-0-containersnap-9b8d4677-4c16-48 pid_in_vm=789 runtime=aws.firecracker vmID=2-0
time="2023-10-18T13:29:23.755391679-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/tasks/start type=containerd.events.TaskStart
time="2023-10-18T13:29:24.468715715-06:00" level=debug msg="pause VM request: VMID:\"2-0\" "
time="2023-10-18T13:29:24.470115857-06:00" level=debug msg="create snapshot request: VMID:\"2-0\" MemFilePath:\"/fccd/snapshots/2/mem_file\" SnapshotPath:\"/fccd/snapshots/2/snap_file\" "
time="2023-10-18T13:29:25.407066408-06:00" level=debug msg="get snapshot mounts" key=vm2-0-containersnap-9b8d4677-4c16-48
time="2023-10-18T13:29:25.407148400-06:00" level=debug msg=mounts key=firecracker-containerd/19/vm2-0-containersnap-9b8d4677-4c16-48
time="2023-10-18T13:29:25.407260546-06:00" level=debug msg=stat key=firecracker-containerd/19/vm2-0-containersnap-9b8d4677-4c16-48
time="2023-10-18T13:29:25.421240052-06:00" level=debug msg="prepare snapshot" key=vm2-0-containersnap-9b8d4677-4c16-48-base-image parent="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:25.421653333-06:00" level=debug msg=prepare key=firecracker-containerd/20/vm2-0-containersnap-9b8d4677-4c16-48-base-image parent="firecracker-containerd/16/sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:25.421786722-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-12' from 'fc-dev-thinpool-snap-8' with fsType: 'ext4'"
time="2023-10-18T13:29:25.469766565-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:25.640823848-06:00" level=debug msg="remove snapshot" key=vm2-0-containersnap-9b8d4677-4c16-48-base-image
time="2023-10-18T13:29:25.641340792-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/remove type=containerd.events.SnapshotRemove
time="2023-10-18T13:29:25.644494943-06:00" level=debug msg="resume VM request: VMID:\"2-0\" "
time="2023-10-18T13:29:25.650717335-06:00" level=debug msg="schedule snapshotter cleanup" snapshotter=devmapper
time="2023-10-18T13:29:25.650812243-06:00" level=debug msg=walk
time="2023-10-18T13:29:25.651004129-06:00" level=debug msg=remove key=firecracker-containerd/20/vm2-0-containersnap-9b8d4677-4c16-48-base-image
time="2023-10-18T13:29:25.684910046-06:00" level=debug msg="stop VM: VMID:\"2-0\" "
time="2023-10-18T13:29:25.686290381-06:00" level=info msg="GetInstanceInfo successful" runtime=aws.firecracker
time="2023-10-18T13:29:25.686321990-06:00" level=info msg="gracefully shutdown VM" runtime=aws.firecracker vmID=2-0
time="2023-10-18T13:29:25.686929135-06:00" level=error msg="error waiting for exit" ExecID= TaskID=vm2-0-containersnap-9b8d4677-4c16-48 error="ttrpc: closed" runtime=aws.firecracker vmID=2-0
time="2023-10-18T13:29:25.687014383-06:00" level=error msg="failed to call in-VM agent" error="ttrpc: closed" runtime=aws.firecracker vmID=2-0
time="2023-10-18T13:29:25.687034318-06:00" level=error msg="forcefully terminate VM 2-0" runtime=aws.firecracker vmID=2-0
time="2023-10-18T13:29:25.687167015-06:00" level=error msg="error copying io" ExecID= TaskID=vm2-0-containersnap-9b8d4677-4c16-48 error="reading from a closed fifo" runtime=aws.firecracker stream=stdin vmID=2-0
time="2023-10-18T13:29:25.687243508-06:00" level=error msg="error closing io stream" ExecID= TaskID=vm2-0-containersnap-9b8d4677-4c16-48 error="1 error occurred:\n\t* close unix @->firecracker.vsock: use of closed network connection\n\n" runtime=aws.firecracker stream=stdin vmID=2-0
time="2023-10-18T13:29:25.687277993-06:00" level=error msg="error while forwarding events from VM agent" error="ttrpc: closed" runtime=aws.firecracker vmID=2-0
time="2023-10-18T13:29:25.687352688-06:00" level=error msg="error returned from VM wait" error=shutdown runtime=aws.firecracker vmID=2-0
time="2023-10-18T13:29:25.687506654-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/firecracker-vm/stop type=VMStop
time="2023-10-18T13:29:25.732431720-06:00" level=warning msg="firecracker exited: signal: killed" runtime=aws.firecracker
time="2023-10-18T13:29:25.787681673-06:00" level=debug msg="removed snapshot" key=firecracker-containerd/20/vm2-0-containersnap-9b8d4677-4c16-48-base-image snapshotter=devmapper
time="2023-10-18T13:29:25.787726247-06:00" level=debug msg=cleanup
time="2023-10-18T13:29:25.787747320-06:00" level=debug msg="snapshot garbage collected" d=136.958946ms snapshotter=devmapper
time="2023-10-18T13:29:25.787804591-06:00" level=debug msg="garbage collected" d="850.167µs"
time="2023-10-18T13:29:30.690297759-06:00" level=debug msg="shim has been terminated" error="exit status 1" vmID=2-0
time="2023-10-18T13:29:30.690304104-06:00" level=info msg="shim disconnected" id=vm2-0-containersnap-9b8d4677-4c16-48
time="2023-10-18T13:29:30.690480467-06:00" level=warning msg="cleaning up after shim disconnected" id=vm2-0-containersnap-9b8d4677-4c16-48 namespace=firecracker-containerd
time="2023-10-18T13:29:30.690503952-06:00" level=info msg="cleaning up dead shim"
time="2023-10-18T13:29:30.734498286-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/tasks/exit type=containerd.events.TaskExit
time="2023-10-18T13:29:30.734599623-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/tasks/delete type=containerd.events.TaskDelete
time="2023-10-18T13:29:30.797074693-06:00" level=debug msg="create VM request: VMID:\"3-0\" MachineCfg:<MemSizeMib:256 VcpuCount:1 > KernelArgs:\"ro noapic reboot=k panic=1 pci=off nomodules systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init tsc=reliable quiet 8250.nr_uarts=0 ipv6.disable=1\" NetworkInterfaces:<StaticConfig:<MacAddress:\"AA:FC:00:00:00:01\" HostDevName:\"tap0\" IPConfig:<PrimaryAddr:\"172.16.0.2/24\" GatewayAddr:\"172.16.0.1\" Nameservers:\"8.8.8.8\" > > > TimeoutSeconds:100 NetNS:\"/var/run/netns/uvmns87\" "
time="2023-10-18T13:29:30.797119721-06:00" level=debug msg="using namespace: firecracker-containerd"
time="2023-10-18T13:29:30.797336012-06:00" level=debug msg="starting containerd-shim-aws-firecracker" vmID=3-0
time="2023-10-18T13:29:30.818668852-06:00" level=info msg="loading plugin \"io.containerd.service.v1.introspection-service\"..." runtime=aws.firecracker type=io.containerd.service.v1
time="2023-10-18T13:29:30.818829796-06:00" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." runtime=aws.firecracker type=io.containerd.grpc.v1
time="2023-10-18T13:29:30.818862566-06:00" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=aws.firecracker type=io.containerd.event.v1
time="2023-10-18T13:29:30.818888029-06:00" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=aws.firecracker type=io.containerd.ttrpc.v1
time="2023-10-18T13:29:30.818923866-06:00" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=aws.firecracker type=io.containerd.internal.v1
time="2023-10-18T13:29:30.819088795-06:00" level=info msg="starting signal loop" namespace=firecracker-containerd path="/var/lib/firecracker-containerd/shim-base/firecracker-containerd#3-0" pid=238771 runtime=aws.firecracker
time="2023-10-18T13:29:30.819251256-06:00" level=info msg="creating new VM" runtime=aws.firecracker vmID=3-0
time="2023-10-18T13:29:30.819708028-06:00" level=info msg="Called startVMM(), setting up a VMM on firecracker.sock" runtime=aws.firecracker
time="2023-10-18T13:29:30.832784280-06:00" level=info msg="refreshMachineConfiguration: [GET /machine-config][200] getMachineConfigurationOK  &{CPUTemplate:T2 MemSizeMib:0xc0004f7d18 Smt:0xc0004f7d23 TrackDirtyPages:0xc0004f7d26 VcpuCount:0xc0004f7d10}" runtime=aws.firecracker
time="2023-10-18T13:29:30.833186298-06:00" level=info msg="PutGuestBootSource: [PUT /boot-source][204] putGuestBootSourceNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:30.833218540-06:00" level=info msg="Attaching drive /var/lib/firecracker-containerd/runtime/default-rootfs.img, slot root_drive, root true." runtime=aws.firecracker
time="2023-10-18T13:29:30.833805440-06:00" level=info msg="Attached drive /var/lib/firecracker-containerd/runtime/default-rootfs.img: [PUT /drives/{drive_id}][204] putGuestDriveByIdNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:30.833839330-06:00" level=info msg="Attaching drive /var/lib/firecracker-containerd/shim-base/firecracker-containerd#3-0/ctrstub0, slot MN2HE43UOVRDA, root false." runtime=aws.firecracker
time="2023-10-18T13:29:30.834199146-06:00" level=info msg="Attached drive /var/lib/firecracker-containerd/shim-base/firecracker-containerd#3-0/ctrstub0: [PUT /drives/{drive_id}][204] putGuestDriveByIdNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:30.834229951-06:00" level=info msg="Attaching NIC tap0 (hwaddr AA:FC:00:00:00:01) at index 1" runtime=aws.firecracker
time="2023-10-18T13:29:30.835491204-06:00" level=info msg="No interfaces are allowed to access MMDS, skipping MMDS config" runtime=aws.firecracker
time="2023-10-18T13:29:30.862236618-06:00" level=info msg="startInstance successful: [PUT /actions][204] createSyncActionNoContent " runtime=aws.firecracker
time="2023-10-18T13:29:30.862297042-06:00" level=info msg="calling agent" runtime=aws.firecracker vmID=3-0
time="2023-10-18T13:29:32.562753300-06:00" level=info msg="successfully started the VM" runtime=aws.firecracker vmID=3-0
time="2023-10-18T13:29:32.563409406-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/firecracker-vm/start type=VMStart
time="2023-10-18T13:29:32.575501653-06:00" level=error msg="aws.firecracker: publisher not closed" shim_stream=stderr vmID=2-0
time="2023-10-18T13:29:32.580961072-06:00" level=debug msg="prepare snapshot" key=vm3-0-containersnap-9b0ae024-3806-43 parent="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:32.581433588-06:00" level=debug msg=prepare key=firecracker-containerd/21/vm3-0-containersnap-9b0ae024-3806-43 parent="firecracker-containerd/16/sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:32.581565310-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-13' from 'fc-dev-thinpool-snap-8' with fsType: 'ext4'"
time="2023-10-18T13:29:32.642242332-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:32.653658410-06:00" level=debug msg="get snapshot mounts" key=vm3-0-containersnap-9b0ae024-3806-43
time="2023-10-18T13:29:32.653724047-06:00" level=debug msg=mounts key=firecracker-containerd/21/vm3-0-containersnap-9b0ae024-3806-43
time="2023-10-18T13:29:32.653805431-06:00" level=debug msg=stat key=firecracker-containerd/21/vm3-0-containersnap-9b0ae024-3806-43
time="2023-10-18T13:29:32.664836668-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/containers/create type=containerd.events.ContainerCreate
time="2023-10-18T13:29:32.669716930-06:00" level=debug msg="get snapshot mounts" key=vm3-0-containersnap-9b0ae024-3806-43
time="2023-10-18T13:29:32.669774722-06:00" level=debug msg=mounts key=firecracker-containerd/21/vm3-0-containersnap-9b0ae024-3806-43
time="2023-10-18T13:29:32.669862203-06:00" level=debug msg=stat key=firecracker-containerd/21/vm3-0-containersnap-9b0ae024-3806-43
time="2023-10-18T13:29:32.686838281-06:00" level=debug msg="garbage collected" d="693.914µs"
time="2023-10-18T13:29:32.716507868-06:00" level=debug msg=StartShim runtime=aws.firecracker task_id=vm3-0-containersnap-9b0ae024-3806-43
time="2023-10-18T13:29:32.717344945-06:00" level=debug msg="create VM request: VMID:\"3-0\" "
time="2023-10-18T13:29:32.717387003-06:00" level=debug msg="using namespace: firecracker-containerd"
time="2023-10-18T13:29:32.717741313-06:00" level=info msg="successfully started shim (git commit: 09e74c6de86fb093077a8aed11ad35b612c35b6a)." runtime=aws.firecracker task_id=vm3-0-containersnap-9b0ae024-3806-43 vmID=3-0
time="2023-10-18T13:29:32.719936963-06:00" level=info msg="PatchGuestDrive successful" runtime=aws.firecracker
time="2023-10-18T13:29:32.847326274-06:00" level=info msg="successfully created task" ExecID= TaskID=vm3-0-containersnap-9b0ae024-3806-43 pid_in_vm=789 runtime=aws.firecracker vmID=3-0
time="2023-10-18T13:29:32.848618343-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/tasks/create type=containerd.events.TaskCreate
time="2023-10-18T13:29:32.861530423-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/tasks/start type=containerd.events.TaskStart
time="2023-10-18T13:29:33.581943826-06:00" level=debug msg="pause VM request: VMID:\"3-0\" "
time="2023-10-18T13:29:33.584015199-06:00" level=debug msg="create snapshot request: VMID:\"3-0\" MemFilePath:\"/fccd/snapshots/3/mem_file\" SnapshotPath:\"/fccd/snapshots/3/snap_file\" "
time="2023-10-18T13:29:34.520103600-06:00" level=debug msg="get snapshot mounts" key=vm3-0-containersnap-9b0ae024-3806-43
time="2023-10-18T13:29:34.520192131-06:00" level=debug msg=mounts key=firecracker-containerd/21/vm3-0-containersnap-9b0ae024-3806-43
time="2023-10-18T13:29:34.520283351-06:00" level=debug msg=stat key=firecracker-containerd/21/vm3-0-containersnap-9b0ae024-3806-43
time="2023-10-18T13:29:34.534984309-06:00" level=debug msg="prepare snapshot" key=vm3-0-containersnap-9b0ae024-3806-43-base-image parent="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:34.535379927-06:00" level=debug msg=prepare key=firecracker-containerd/22/vm3-0-containersnap-9b0ae024-3806-43-base-image parent="firecracker-containerd/16/sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:34.535488862-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-14' from 'fc-dev-thinpool-snap-8' with fsType: 'ext4'"
time="2023-10-18T13:29:34.599391345-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
time="2023-10-18T13:29:34.773622403-06:00" level=debug msg="remove snapshot" key=vm3-0-containersnap-9b0ae024-3806-43-base-image
time="2023-10-18T13:29:34.774125157-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/remove type=containerd.events.SnapshotRemove
time="2023-10-18T13:29:34.777156819-06:00" level=debug msg="resume VM request: VMID:\"3-0\" "
time="2023-10-18T13:29:34.782340368-06:00" level=debug msg="stop VM: VMID:\"3-0\" "
time="2023-10-18T13:29:34.783123506-06:00" level=info msg="GetInstanceInfo successful" runtime=aws.firecracker
time="2023-10-18T13:29:34.783162495-06:00" level=info msg="gracefully shutdown VM" runtime=aws.firecracker vmID=3-0
time="2023-10-18T13:29:34.783434701-06:00" level=error msg="failed to call in-VM agent" error="ttrpc: closed" runtime=aws.firecracker vmID=3-0
time="2023-10-18T13:29:34.783464055-06:00" level=error msg="forcefully terminate VM 3-0" runtime=aws.firecracker vmID=3-0
time="2023-10-18T13:29:34.783486107-06:00" level=error msg="error while forwarding events from VM agent" error="ttrpc: closed" runtime=aws.firecracker vmID=3-0
time="2023-10-18T13:29:34.783468516-06:00" level=error msg="error waiting for exit" ExecID= TaskID=vm3-0-containersnap-9b0ae024-3806-43 error="ttrpc: closed" runtime=aws.firecracker vmID=3-0
time="2023-10-18T13:29:34.783798606-06:00" level=error msg="error copying io" ExecID= TaskID=vm3-0-containersnap-9b0ae024-3806-43 error="reading from a closed fifo" runtime=aws.firecracker stream=stdin vmID=3-0
time="2023-10-18T13:29:34.783841802-06:00" level=error msg="error closing io stream" ExecID= TaskID=vm3-0-containersnap-9b0ae024-3806-43 error="1 error occurred:\n\t* close unix @->firecracker.vsock: use of closed network connection\n\n" runtime=aws.firecracker stream=stdin vmID=3-0
time="2023-10-18T13:29:34.783921085-06:00" level=error msg="error returned from VM wait" error=shutdown runtime=aws.firecracker vmID=3-0
time="2023-10-18T13:29:34.784085871-06:00" level=debug msg="event forwarded" ns=firecracker-containerd topic=/firecracker-vm/stop type=VMStop
time="2023-10-18T13:29:34.792020485-06:00" level=debug msg="schedule snapshotter cleanup" snapshotter=devmapper
time="2023-10-18T13:29:34.792101808-06:00" level=debug msg=walk
time="2023-10-18T13:29:34.792293554-06:00" level=debug msg=remove key=firecracker-containerd/22/vm3-0-containersnap-9b0ae024-3806-43-base-image
time="2023-10-18T13:29:34.854794266-06:00" level=warning msg="firecracker exited: signal: killed" runtime=aws.firecracker
time="2023-10-18T13:29:34.911680032-06:00" level=debug msg="removed snapshot" key=firecracker-containerd/22/vm3-0-containersnap-9b0ae024-3806-43-base-image snapshotter=devmapper
time="2023-10-18T13:29:34.911731228-06:00" level=debug msg=cleanup
time="2023-10-18T13:29:34.911751760-06:00" level=debug msg="snapshot garbage collected" d=119.673132ms snapshotter=devmapper
time="2023-10-18T13:29:34.911798045-06:00" level=debug msg="garbage collected" d="629µs"
time="2023-10-18T13:29:39.786825489-06:00" level=debug msg="shim has been terminated" error="exit status 1" vmID=3-0
time="2023-10-18T13:29:39.786909579-06:00" level=info msg="shim disconnected" id=vm3-0-containersnap-9b0ae024-3806-43
time="2023-10-18T13:29:39.787004167-06:00" level=warning msg="cleaning up after shim disconnected" id=vm3-0-containersnap-9b0ae024-3806-43 namespace=firecracker-containerd
time="2023-10-18T13:29:39.787027569-06:00" level=info msg="cleaning up dead shim"
time="2023-10-18T13:29:39.808382312-06:00" level=error msg="aws.firecracker: publisher not closed" shim_stream=stderr vmID=3-0
time="2023-10-18T13:29:39.814582709-06:00" level=debug msg="prepare snapshot" key=vm3-1-containersnap-adb17be5-3768-48 parent="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:39.814902775-06:00" level=debug msg=prepare key=firecracker-containerd/23/vm3-1-containersnap-adb17be5-3768-48 parent="firecracker-containerd/16/sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253"
time="2023-10-18T13:29:39.814998611-06:00" level=debug msg="creating snapshot device 'fc-dev-thinpool-snap-15' from 'fc-dev-thinpool-snap-8' with fsType: 'ext4'"
time="2023-10-18T13:29:39.834166120-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/tasks/exit type=containerd.events.TaskExit
time="2023-10-18T13:29:39.834219287-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/tasks/delete type=containerd.events.TaskDelete
time="2023-10-18T13:29:39.885868238-06:00" level=debug msg="event published" ns=firecracker-containerd topic=/snapshot/prepare type=containerd.events.SnapshotPrepare

upd=true_lazy=false_info.log

sudo mkdir -m777 -p /tmp/ctrd-logs && sudo env "PATH=/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/go/bin" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>/tmp/ctrd-logs/fccd_orch_noupf_log.out 2>/tmp/ctrd-logs/fccd_orch_noupf_log.err &
sudo env "PATH=/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/go/bin" go test vhive_test.go stats.go vhive.go functions.go -short -v -race -cover
time="2023-10-18T08:27:31.653069933-06:00" level=info msg="Orchestrator snapshots enabled: false"
time="2023-10-18T08:27:31.653311246-06:00" level=info msg="Orchestrator UPF enabled: false"
time="2023-10-18T08:27:31.653352441-06:00" level=info msg="Orchestrator lazy serving mode enabled: false"
time="2023-10-18T08:27:31.653430772-06:00" level=info msg="Orchestrator UPF metrics enabled: false"
time="2023-10-18T08:27:31.653493354-06:00" level=info msg="Drop cache: true"
time="2023-10-18T08:27:31.653539096-06:00" level=info msg="Bench dir: bench_results"
time="2023-10-18T08:27:31.694392404-06:00" level=info msg="Creating containerd client"
time="2023-10-18T08:27:31.695633383-06:00" level=info msg="Created containerd client"
time="2023-10-18T08:27:31.695753634-06:00" level=info msg="Creating firecracker client"
time="2023-10-18T08:27:31.696189925-06:00" level=info msg="Created firecracker client"
time="2023-10-18T08:27:31.696289381-06:00" level=info msg="Creating image manager"
=== RUN   TestSendToFunctionSerial
time="2023-10-18T08:27:31.696627110-06:00" level=info msg="New function added" fID=1 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T08:27:39.676268748-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:27:39.676503221-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:27:52.483298124-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 1-0" vmID=1-0
--- PASS: TestSendToFunctionSerial (20.79s)
=== RUN   TestSendToFunctionParallel
time="2023-10-18T08:27:52.484716040-06:00" level=info msg="New function added" fID=2 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T08:27:52.555244180-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:27:52.555483547-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:28:05.375666681-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 2-0" vmID=2-0
--- PASS: TestSendToFunctionParallel (12.89s)
=== RUN   TestStartSendStopTwice
time="2023-10-18T08:28:05.377002168-06:00" level=info msg="New function added" fID=3 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T08:28:05.451186227-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:28:05.451441057-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:28:18.295707253-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 3-0" vmID=3-0
time="2023-10-18T08:28:18.363141807-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:28:18.363363983-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:28:31.143472709-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 3-1" vmID=3-1
--- PASS: TestStartSendStopTwice (25.77s)
=== RUN   TestStatsNotNumericFunction
time="2023-10-18T08:28:31.144777692-06:00" level=info msg="New function added" fID=not-cld image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T08:28:31.217913729-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:28:31.218182752-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:28:44.075419402-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM not-cld-0" vmID=not-cld-0
--- PASS: TestStatsNotNumericFunction (12.93s)
=== RUN   TestStatsNotColdFunction
time="2023-10-18T08:28:44.076697966-06:00" level=info msg="New function added" fID=4 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T08:28:44.147106640-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:28:44.147340631-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:28:56.915480316-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 4-0" vmID=4-0
--- PASS: TestStatsNotColdFunction (12.84s)
=== RUN   TestSaveMemorySerial
time="2023-10-18T08:28:56.916715018-06:00" level=info msg="New function added" fID=5 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=false servedTh=40
time="2023-10-18T08:28:56.984603025-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:28:56.984898367-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:28:59.973331921-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:28:59.973560506-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:29:02.845635353-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:29:02.845874756-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:29:09.930669543-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-0" vmID=5-0
time="2023-10-18T08:29:12.802460077-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-1" vmID=5-1
time="2023-10-18T08:29:15.743403270-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-2" vmID=5-2
--- PASS: TestSaveMemorySerial (18.83s)
=== RUN   TestSaveMemoryParallel
time="2023-10-18T08:29:15.745069455-06:00" level=info msg="New function added" fID=6 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=false servedTh=40
time="2023-10-18T08:29:15.811201648-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:29:15.811480463-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:29:18.738491129-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:29:18.738752624-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:29:21.651830988-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:29:21.652094152-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:29:28.692190763-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-0" vmID=6-0
time="2023-10-18T08:29:31.603199735-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-1" vmID=6-1
time="2023-10-18T08:29:34.524630377-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-2" vmID=6-2
--- PASS: TestSaveMemoryParallel (18.78s)
=== RUN   TestDirectStartStopVM
time="2023-10-18T08:29:34.526487459-06:00" level=info msg="New function added" fID=7 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T08:29:34.591035297-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:29:34.591232783-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:29:47.449232397-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 7-0" vmID=7-0
--- PASS: TestDirectStartStopVM (12.92s)
=== RUN   TestAllFunctions
    vhive_test.go:290: skipping TestAllFunctions in non-nightly runs.
--- SKIP: TestAllFunctions (0.00s)
PASS
coverage: 44.4% of statements
time="2023-10-18T08:29:47.450649522-06:00" level=info msg="waiting for goroutines"
time="2023-10-18T08:29:47.450783696-06:00" level=info msg="waiting done"
time="2023-10-18T08:29:47.450916096-06:00" level=info msg="Closing fcClient"
time="2023-10-18T08:29:47.451149151-06:00" level=info msg="Closing containerd client"
time="2023-10-18T08:29:47.451459498-06:00" level=info msg="Cleaning up network manager"
ok      command-line-arguments  136.085s        coverage: 44.4% of statements
./scripts/clean_fcctr.sh
Killing firecracker agents and VMs
Resetting nftables
Deleting veth* devices created by CNI
Cannot find device "br0"
Cannot find device "br1"
Cleaning in /var/lib/cni/ non-network
find: ‘/var/lib/cni/’: No such file or directory
Removing devmapper devices
Removed fc-dev-thinpool-snap-13
device-mapper: remove ioctl on fc-dev-thinpool  failed: Device or resource busy
Command failed.
Removed fc-dev-thinpool-snap-12
Removed fc-dev-thinpool-snap-11
Removed fc-dev-thinpool-snap-10
Removed fc-dev-thinpool-snap-19
Removed fc-dev-thinpool-snap-21
Removed fc-dev-thinpool-snap-18
Removed fc-dev-thinpool-snap-20
Removed fc-dev-thinpool-snap-17
Removed fc-dev-thinpool-snap-16
Removed fc-dev-thinpool-snap-15
Removed fc-dev-thinpool-snap-14
Removed fc-dev-thinpool-snap-9
Cleaning /var/lib/firecracker-containerd/containerd /var/lib/firecracker-containerd/runtime /var/lib/firecracker-containerd/shim-base /var/lib/firecracker-containerd/snapshotter
Cleaning /run/firecracker-containerd/*
Cleaning CNI state, e.g., allocated addresses
rm: cannot remove '/var/lib/cni/networks/fcnet*/last_reserved_ip.0': No such file or directory
clean already
rm: cannot remove '/var/lib/cni/networks/fcnet*/19*': No such file or directory
clean already
Cleaning snapshots
Creating a fresh devmapper
0 209715200 thin-pool /dev/loop15 /dev/loop14 128 32768 1 skip_block_zeroing
device-mapper: reload ioctl on fc-dev-thinpool  failed: No such device or address
Command failed.
sudo mkdir -m777 -p /tmp/ctrd-logs && sudo env "PATH=/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/go/bin" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>/tmp/ctrd-logs/fccd_orch_noupf_log.out 2>/tmp/ctrd-logs/fccd_orch_noupf_log.err &
sudo env "PATH=/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/go/bin" go test vhive_test.go stats.go vhive.go functions.go -short -v -race -cover -args -snapshotsTest
time="2023-10-18T08:29:53.048841273-06:00" level=info msg="Orchestrator snapshots enabled: true"
time="2023-10-18T08:29:53.049107358-06:00" level=info msg="Orchestrator UPF enabled: false"
time="2023-10-18T08:29:53.049168713-06:00" level=info msg="Orchestrator lazy serving mode enabled: false"
time="2023-10-18T08:29:53.049266754-06:00" level=info msg="Orchestrator UPF metrics enabled: false"
time="2023-10-18T08:29:53.049322493-06:00" level=info msg="Drop cache: true"
time="2023-10-18T08:29:53.049368964-06:00" level=info msg="Bench dir: bench_results"
time="2023-10-18T08:29:53.093924084-06:00" level=info msg="Creating containerd client"
time="2023-10-18T08:29:53.095719919-06:00" level=info msg="Created containerd client"
time="2023-10-18T08:29:53.095805225-06:00" level=info msg="Creating firecracker client"
time="2023-10-18T08:29:53.096262308-06:00" level=info msg="Created firecracker client"
time="2023-10-18T08:29:53.096356469-06:00" level=info msg="Creating image manager"
=== RUN   TestSendToFunctionSerial
time="2023-10-18T08:29:53.096717441-06:00" level=info msg="New function added" fID=1 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T08:30:04.414255665-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:30:04.414508139-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:30:13.465540195-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 1-0" vmID=1-0
--- PASS: TestSendToFunctionSerial (20.37s)
=== RUN   TestSendToFunctionParallel
time="2023-10-18T08:30:13.501403996-06:00" level=info msg="New function added" fID=2 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T08:30:13.566413174-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:30:13.566670898-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:30:22.530439077-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 2-0" vmID=2-0
--- PASS: TestSendToFunctionParallel (9.06s)
=== RUN   TestStartSendStopTwice
time="2023-10-18T08:30:22.566098744-06:00" level=info msg="New function added" fID=3 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T08:30:22.637419986-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:30:22.637660369-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:30:31.671553136-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 3-0" vmID=3-0
time="2023-10-18T08:30:31.894009528-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:30:31.894237816-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:30:42.060791606-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 3-1" vmID=3-1
--- PASS: TestStartSendStopTwice (19.53s)
=== RUN   TestStatsNotNumericFunction
time="2023-10-18T08:30:42.095837550-06:00" level=info msg="New function added" fID=not-cld image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T08:30:42.159495018-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:30:42.159712586-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:30:51.116821691-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM not-cld-0" vmID=not-cld-0
--- PASS: TestStatsNotNumericFunction (9.05s)
=== RUN   TestStatsNotColdFunction
time="2023-10-18T08:30:51.152782910-06:00" level=info msg="New function added" fID=4 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T08:30:51.223935100-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:30:51.224223563-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:31:00.289092157-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 4-0" vmID=4-0
--- PASS: TestStatsNotColdFunction (9.17s)
=== RUN   TestSaveMemorySerial
time="2023-10-18T08:31:00.325360726-06:00" level=info msg="New function added" fID=5 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=false servedTh=40
time="2023-10-18T08:31:00.392640686-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:31:00.392912018-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:31:05.972590360-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:31:05.972831144-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:31:10.720710780-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-0" vmID=5-0
time="2023-10-18T08:31:11.014209893-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:31:11.014465699-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:31:20.810490172-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-1" vmID=5-1
time="2023-10-18T08:31:21.211494033-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-2" vmID=5-2
--- PASS: TestSaveMemorySerial (20.92s)
=== RUN   TestSaveMemoryParallel
time="2023-10-18T08:31:21.246445515-06:00" level=info msg="New function added" fID=6 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=false servedTh=40
time="2023-10-18T08:31:21.309663785-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:31:21.309911960-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:31:25.624189182-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:31:25.624440892-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:31:30.299264240-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-0" vmID=6-0
time="2023-10-18T08:31:30.549763864-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:31:30.550020404-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:31:40.351448560-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-1" vmID=6-1
time="2023-10-18T08:31:40.722893635-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-2" vmID=6-2
--- PASS: TestSaveMemoryParallel (19.51s)
=== RUN   TestDirectStartStopVM
time="2023-10-18T08:31:40.758660023-06:00" level=info msg="New function added" fID=7 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T08:31:40.826049046-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:31:40.826194556-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:31:51.265371611-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 7-0" vmID=7-0
--- PASS: TestDirectStartStopVM (10.54s)
=== RUN   TestAllFunctions
    vhive_test.go:290: skipping TestAllFunctions in non-nightly runs.
--- SKIP: TestAllFunctions (0.00s)
PASS
coverage: 56.8% of statements
time="2023-10-18T08:31:51.266774016-06:00" level=info msg="waiting for goroutines"
time="2023-10-18T08:31:51.266923880-06:00" level=info msg="waiting done"
time="2023-10-18T08:31:51.267038094-06:00" level=info msg="Closing fcClient"
time="2023-10-18T08:31:51.267165623-06:00" level=info msg="Closing containerd client"
time="2023-10-18T08:31:51.267477624-06:00" level=info msg="Cleaning up network manager"
time="2023-10-18T08:31:51.313161990-06:00" level=error msg="failed to remove network deleting route: no such process:"
ok      command-line-arguments  118.561s        coverage: 56.8% of statements
./scripts/clean_fcctr.sh
Killing firecracker agents and VMs
Resetting nftables
Deleting veth* devices created by CNI
Cannot find device "br0"
Cannot find device "br1"
Cleaning in /var/lib/cni/ non-network
find: ‘/var/lib/cni/’: No such file or directory
Removing devmapper devices
Removed fc-dev-thinpool-snap-28
Removed fc-dev-thinpool-snap-13
device-mapper: remove ioctl on fc-dev-thinpool  failed: Device or resource busy
Command failed.
Removed fc-dev-thinpool-snap-11
Removed fc-dev-thinpool-snap-24
Removed fc-dev-thinpool-snap-18
Removed fc-dev-thinpool-snap-20
Removed fc-dev-thinpool-snap-16
Removed fc-dev-thinpool-snap-9
Cleaning /var/lib/firecracker-containerd/containerd /var/lib/firecracker-containerd/runtime /var/lib/firecracker-containerd/shim-base /var/lib/firecracker-containerd/snapshotter
Cleaning /run/firecracker-containerd/*
Cleaning CNI state, e.g., allocated addresses
rm: cannot remove '/var/lib/cni/networks/fcnet*/last_reserved_ip.0': No such file or directory
clean already
rm: cannot remove '/var/lib/cni/networks/fcnet*/19*': No such file or directory
clean already
Cleaning snapshots
Creating a fresh devmapper
0 209715200 thin-pool /dev/loop17 /dev/loop16 128 32768 1 skip_block_zeroing
device-mapper: reload ioctl on fc-dev-thinpool  failed: No such device or address
Command failed.
sudo mkdir -m777 -p /tmp/ctrd-logs && sudo env "PATH=/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/go/bin" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>/tmp/ctrd-logs/fccd_orch_upf_log.out 2>/tmp/ctrd-logs/fccd_orch_upf_log.err &
sudo env "PATH=/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/go/bin" go test vhive_test.go stats.go vhive.go functions.go -short -v -race -cover -args -snapshotsTest -upfTest
time="2023-10-18T08:31:56.732274299-06:00" level=info msg="Orchestrator snapshots enabled: true"
time="2023-10-18T08:31:56.732536113-06:00" level=info msg="Orchestrator UPF enabled: true"
time="2023-10-18T08:31:56.732616705-06:00" level=info msg="Orchestrator lazy serving mode enabled: false"
time="2023-10-18T08:31:56.732712718-06:00" level=info msg="Orchestrator UPF metrics enabled: false"
time="2023-10-18T08:31:56.732801021-06:00" level=info msg="Drop cache: true"
time="2023-10-18T08:31:56.732940726-06:00" level=info msg="Bench dir: bench_results"
time="2023-10-18T08:31:56.772907213-06:00" level=info msg="Creating containerd client"
time="2023-10-18T08:31:56.776161274-06:00" level=info msg="Created containerd client"
time="2023-10-18T08:31:56.776307716-06:00" level=info msg="Creating firecracker client"
time="2023-10-18T08:31:56.776805767-06:00" level=info msg="Created firecracker client"
time="2023-10-18T08:31:56.776948606-06:00" level=info msg="Creating image manager"
=== RUN   TestSendToFunctionSerial
time="2023-10-18T08:31:56.777367950-06:00" level=info msg="New function added" fID=1 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T08:32:06.804666763-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:32:06.804896111-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:32:15.917989408-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 1-0" vmID=1-0
--- PASS: TestSendToFunctionSerial (19.14s)
=== RUN   TestSendToFunctionParallel
time="2023-10-18T08:32:15.953927739-06:00" level=info msg="New function added" fID=2 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T08:32:16.023342032-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:32:16.023608490-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:32:25.032920591-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 2-0" vmID=2-0
--- PASS: TestSendToFunctionParallel (9.11s)
=== RUN   TestStartSendStopTwice
time="2023-10-18T08:32:25.069230764-06:00" level=info msg="New function added" fID=3 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T08:32:25.134327245-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:32:25.134583793-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:32:35.495338201-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 3-0" vmID=3-0
time="2023-10-18T08:32:35.714445555-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T08:32:35.714608256-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T08:32:35.714812668-06:00" level=error msg="VM not registered with the memory manager" vmID=3-1
time="2023-10-18T08:32:35.714975550-06:00" level=panic msg="VM not registered with the memory manager"
--- FAIL: TestStartSendStopTwice (10.68s)
panic: (*logrus.Entry) 0xc0004288c0 [recovered]
        panic: (*logrus.Entry) 0xc0004288c0

goroutine 247 [running]:
testing.tRunner.func1.2({0x1af08c0, 0xc0004288c0})
        /usr/local/go/src/testing/testing.go:1389 +0x366
testing.tRunner.func1()
        /usr/local/go/src/testing/testing.go:1392 +0x5d2
panic({0x1af08c0, 0xc0004288c0})
        /usr/local/go/src/runtime/panic.go:844 +0x258
github.com/sirupsen/logrus.(*Entry).log(0xc000428690, 0x0, {0xc00057b080, 0x29})
        /users/char/go/pkg/mod/github.com/sirupsen/logrus@v1.9.0/entry.go:260 +0xa07
github.com/sirupsen/logrus.(*Entry).Log(0xc000428690, 0x0, {0xc0006cb218, 0x1, 0x1})
        /users/char/go/pkg/mod/github.com/sirupsen/logrus@v1.9.0/entry.go:304 +0x8c
github.com/sirupsen/logrus.(*Logger).Log(0xc0000f8000, 0x0, {0xc0006cb218, 0x1, 0x1})
        /users/char/go/pkg/mod/github.com/sirupsen/logrus@v1.9.0/logger.go:204 +0x85
github.com/sirupsen/logrus.(*Logger).Panic(...)
        /users/char/go/pkg/mod/github.com/sirupsen/logrus@v1.9.0/logger.go:253
github.com/sirupsen/logrus.Panic(...)
        /users/char/go/pkg/mod/github.com/sirupsen/logrus@v1.9.0/exported.go:129
command-line-arguments.(*Function).LoadInstance(0xc000196240, {0xc000694f3d, 0x3})
        /users/char/vHive/functions.go:497 +0x44f
command-line-arguments.(*Function).AddInstance(0xc000196240)
        /users/char/vHive/functions.go:365 +0x2d8
command-line-arguments.(*Function).Serve.func1()
        /users/char/vHive/functions.go:263 +0x150
sync.(*Once).doSlow(0xc000694eb4, 0xc0006cbc10)
        /usr/local/go/src/sync/once.go:68 +0x102
sync.(*Once).Do(0xc000694eb4, 0x1afba4b?)
        /usr/local/go/src/sync/once.go:59 +0x47
command-line-arguments.(*Function).Serve(0xc000196240, {0x1afba4b?, 0x1?}, {0x1b25fa8?, 0x28?}, {0x886fc5?, 0x18c1b0e?}, {0x1afd81d, 0x5})
        /users/char/vHive/functions.go:257 +0x455
command-line-arguments.(*FuncPool).Serve(0x1cd9950?, {0x1cdff80, 0xc000038128}, {0x1afba4b, 0x1}, {0x1b25fa8, 0x28}, {0x1afd81d, 0x5})
        /users/char/vHive/functions.go:125 +0xea
command-line-arguments.TestStartSendStopTwice(0x0?)
        /users/char/vHive/vhive_test.go:160 +0x3dc
testing.tRunner(0xc000c9f860, 0x1b61f48)
        /usr/local/go/src/testing/testing.go:1439 +0x214
created by testing.(*T).Run
        /usr/local/go/src/testing/testing.go:1486 +0x725
FAIL    command-line-arguments  39.034s
FAIL
make: *** [Makefile:57: test] Error 1

upf=true_lazy=false_debug.log

./scripts/clean_fcctr.sh
Killing firecracker agents and VMs
Resetting nftables
Deleting veth* devices created by CNI
Cannot find device "br0"
Cannot find device "br1"
Cleaning in /var/lib/cni/ non-network
find: ‘/var/lib/cni/’: No such file or directory
Removing devmapper devices
Removed fc-dev-thinpool
device-mapper: remove ioctl on fc-dev-thinpool  failed: No such device or address
Command failed.
Cleaning /var/lib/firecracker-containerd/runtime /var/lib/firecracker-containerd/snapshotter
Cleaning /run/firecracker-containerd/*
Cleaning CNI state, e.g., allocated addresses
rm: cannot remove '/var/lib/cni/networks/fcnet*/last_reserved_ip.0': No such file or directory
clean already
rm: cannot remove '/var/lib/cni/networks/fcnet*/19*': No such file or directory
clean already
Cleaning snapshots
Creating a fresh devmapper
0 209715200 thin-pool /dev/loop29 /dev/loop28 128 32768 1 skip_block_zeroing
device-mapper: reload ioctl on fc-dev-thinpool  failed: No such device or address
Command failed.
sudo mkdir -m777 -p /tmp/ctrd-logs && sudo env "PATH=/usr/local/go/bin:/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>/tmp/ctrd-logs/fccd_orch_noupf_log.out 2>/tmp/ctrd-logs/fccd_orch_noupf_log.err &
sudo env "PATH=/usr/local/go/bin:/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin" go test vhive_test.go stats.go vhive.go functions.go -short -v -race -cover
time="2023-10-18T13:24:44.732430579-06:00" level=info msg="Orchestrator snapshots enabled: false"
time="2023-10-18T13:24:44.732702690-06:00" level=info msg="Orchestrator UPF enabled: false"
time="2023-10-18T13:24:44.732749409-06:00" level=info msg="Orchestrator lazy serving mode enabled: false"
time="2023-10-18T13:24:44.732814438-06:00" level=info msg="Orchestrator UPF metrics enabled: false"
time="2023-10-18T13:24:44.732929043-06:00" level=info msg="Drop cache: true"
time="2023-10-18T13:24:44.732978899-06:00" level=info msg="Bench dir: bench_results"
time="2023-10-18T13:24:44.742188679-06:00" level=debug msg="Initializing network pool" poolSize=10
time="2023-10-18T13:24:44.749689555-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns59 tap=tap0
time="2023-10-18T13:24:44.754687732-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns60 tap=tap0
time="2023-10-18T13:24:44.756133445-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns63 tap=tap0
time="2023-10-18T13:24:44.757937378-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns61 tap=tap0
time="2023-10-18T13:24:44.760240213-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns62 tap=tap0
time="2023-10-18T13:24:44.761815489-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns67 tap=tap0
time="2023-10-18T13:24:44.763428803-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns65 tap=tap0
time="2023-10-18T13:24:44.764915418-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns64 tap=tap0
time="2023-10-18T13:24:44.766049783-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns68 tap=tap0
time="2023-10-18T13:24:44.766659454-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns66 tap=tap0
time="2023-10-18T13:24:44.780943183-06:00" level=info msg="Creating containerd client"
time="2023-10-18T13:24:44.782698333-06:00" level=info msg="Created containerd client"
time="2023-10-18T13:24:44.782811698-06:00" level=info msg="Creating firecracker client"
time="2023-10-18T13:24:44.783263762-06:00" level=info msg="Created firecracker client"
time="2023-10-18T13:24:44.783407193-06:00" level=info msg="Creating image manager"
=== RUN   TestSendToFunctionSerial
time="2023-10-18T13:24:44.783893959-06:00" level=debug msg="fIDint=1, err=<nil>, pinnedFuncNum=0"
time="2023-10-18T13:24:44.784054193-06:00" level=debug msg="Created function, pinned=true, shut down after 0 requests" fID=1 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:24:44.784275131-06:00" level=info msg="New function added" fID=1 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T13:24:44.784448733-06:00" level=debug msg="Function is inactive, starting the instance..." fID=1
time="2023-10-18T13:24:44.784588521-06:00" level=debug msg="Adding instance" fID=1
time="2023-10-18T13:24:44.784714125-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:24:44.784965445-06:00" level=debug msg="Allocating a VM instance" vmID=1-0
time="2023-10-18T13:24:44.785116561-06:00" level=debug msg="Creating network config for function instance" funcID=1-0
time="2023-10-18T13:24:44.785346950-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=1-0
time="2023-10-18T13:24:44.785638936-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=1-0
time="2023-10-18T13:24:44.789119733-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns69 tap=tap0
time="2023-10-18T13:24:44.884249311-06:00" level=debug msg=resolving host=ghcr.io
time="2023-10-18T13:24:44.884444906-06:00" level=debug msg="do request" host=ghcr.io request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=HEAD url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:24:45.322342687-06:00" level=debug msg="fetch response received" host=ghcr.io response.header.content-length=73 response.header.content-type=application/json response.header.date="Wed, 18 Oct 2023 19:24:45 GMT" response.header.www-authenticate="Bearer realm=\"https://ghcr.io/token\",service=\"ghcr.io\",scope=\"repository:ease-lab/helloworld:pull\"" response.header.x-github-request-id="DF0D:5CA3:6B95:902D:653030FD" response.status="401 Unauthorized" url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:24:45.322710789-06:00" level=debug msg=Unauthorized header="Bearer realm=\"https://ghcr.io/token\",service=\"ghcr.io\",scope=\"repository:ease-lab/helloworld:pull\"" host=ghcr.io
time="2023-10-18T13:24:45.323040107-06:00" level=debug msg="do request" host=ghcr.io request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=HEAD url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:24:45.533602982-06:00" level=debug msg="fetch response received" host=ghcr.io response.header.content-length=1998 response.header.content-type=application/vnd.docker.distribution.manifest.v2+json response.header.date="Wed, 18 Oct 2023 19:24:45 GMT" response.header.docker-content-digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" response.header.docker-distribution-api-version=registry/2.0 response.header.etag="\"sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25\"" response.header.x-github-request-id="DF0D:5CA3:6BA6:9044:653030FD" response.status="200 OK" url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:24:45.534092013-06:00" level=debug msg=resolved desc.digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" host=ghcr.io
time="2023-10-18T13:24:45.534847175-06:00" level=debug msg=fetch digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" mediatype=application/vnd.docker.distribution.manifest.v2+json size=1998
time="2023-10-18T13:24:45.545414194-06:00" level=debug msg="do request" digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" mediatype=application/vnd.docker.distribution.manifest.v2+json request.header.accept="application/vnd.docker.distribution.manifest.v2+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=1998 url="https://ghcr.io/v2/ease-lab/helloworld/manifests/sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25"
time="2023-10-18T13:24:45.652227966-06:00" level=debug msg="fetch response received" digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" mediatype=application/vnd.docker.distribution.manifest.v2+json response.header.content-length=1998 response.header.content-type=application/vnd.docker.distribution.manifest.v2+json response.header.date="Wed, 18 Oct 2023 19:24:45 GMT" response.header.docker-content-digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" response.header.docker-distribution-api-version=registry/2.0 response.header.etag="\"sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25\"" response.header.x-github-request-id="DF0D:5CA3:6BB0:904F:653030FD" response.status="200 OK" size=1998 url="https://ghcr.io/v2/ease-lab/helloworld/manifests/sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25"
time="2023-10-18T13:24:45.667080183-06:00" level=debug msg=fetch digest="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734" mediatype=application/vnd.docker.container.image.v1+json size=8312
time="2023-10-18T13:24:45.671732870-06:00" level=debug msg="do request" digest="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734" mediatype=application/vnd.docker.container.image.v1+json request.header.accept="application/vnd.docker.container.image.v1+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=8312 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734"
time="2023-10-18T13:24:45.981595890-06:00" level=debug msg="fetch response received" digest="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734" mediatype=application/vnd.docker.container.image.v1+json response.header.accept-ranges=bytes response.header.age=0 response.header.content-disposition= response.header.content-length=8312 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:24:45 GMT" response.header.etag="\"0x8D950DD42AD3ECF\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:28 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=cd638d33ed11f25a2168a1874a8f7469d7d5e02a response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:28 GMT" response.header.x-ms-copy-id=32f6a8de-4aab-4519-b15c-21acf3d16ee7 response.header.x-ms-copy-progress=8312/8312 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:28 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=b5ee528a-c01e-00b4-4ff8-01d7d4000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kcgs7200124-IAD, cache-bur-kbur8200082-BUR" response.status="200 OK" size=8312 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734"
time="2023-10-18T13:24:46.903181445-06:00" level=debug msg=fetch digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=2802957
time="2023-10-18T13:24:46.903449492-06:00" level=debug msg=fetch digest="sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=31506523
time="2023-10-18T13:24:46.903348833-06:00" level=debug msg=fetch digest="sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=301261
time="2023-10-18T13:24:46.903530953-06:00" level=debug msg=fetch digest="sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=230
time="2023-10-18T13:24:46.903952864-06:00" level=debug msg=fetch digest="sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=2233
time="2023-10-18T13:24:46.903750534-06:00" level=debug msg=fetch digest="sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=1887472
time="2023-10-18T13:24:46.904163765-06:00" level=debug msg=fetch digest="sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=20365161
time="2023-10-18T13:24:46.904366741-06:00" level=debug msg=fetch digest="sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=637463
time="2023-10-18T13:24:46.911361734-06:00" level=debug msg="do request" digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=2802957 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9"
time="2023-10-18T13:24:46.912569168-06:00" level=debug msg="do request" digest="sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=31506523 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e"
time="2023-10-18T13:24:46.913188070-06:00" level=debug msg="do request" digest="sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=301261 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a"
time="2023-10-18T13:24:46.913915386-06:00" level=debug msg="do request" digest="sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=230 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd"
time="2023-10-18T13:24:46.915771758-06:00" level=debug msg="do request" digest="sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=2233 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa"
time="2023-10-18T13:24:46.917252665-06:00" level=debug msg="do request" digest="sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=1887472 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d"
time="2023-10-18T13:24:46.918016435-06:00" level=debug msg="do request" digest="sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=20365161 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a"
time="2023-10-18T13:24:46.919436311-06:00" level=debug msg="do request" digest="sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=637463 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d"
time="2023-10-18T13:24:47.101294751-06:00" level=debug msg="fetch response received" digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=173 response.header.content-length=2802957 response.header.content-md5="Xv6UEe5ym/lkcH1POk+jEQ==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:24:47 GMT" response.header.etag="\"0x8D8DEA430F3018D\"" response.header.last-modified="Thu, 04 Mar 2021 00:26:45 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="HIT, HIT" response.header.x-cache-hits="38, 1" response.header.x-fastly-request-id=fe5239527012aa53ef42b8b3f4b97362a2c67e26 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 00:26:45 GMT" response.header.x-ms-copy-id=5eaa9616-c230-429e-a0df-0f62ee445e79 response.header.x-ms-copy-progress=2802957/2802957 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 00:26:45 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=4e7e08ae-001e-0068-0df8-0117ab000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kcgs7200078-IAD, cache-bur-kbur8200082-BUR" response.status="200 OK" size=2802957 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9"
time="2023-10-18T13:24:47.150401565-06:00" level=debug msg="fetch response received" digest="sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-length=230 response.header.content-md5="OBaH1satt139FxF1ozF59w==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:24:47 GMT" response.header.etag="\"0x8D8DECD29DC1039\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=012aed95d8fdcc5a1f5c6383da1487638a931c4b response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=a359c581-3a46-4cf8-a872-9e5325a9976a response.header.x-ms-copy-progress=230/230 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=cddcba6a-701e-0067-4df8-019eb4000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kcgs7200160-IAD, cache-bur-kbur8200082-BUR" response.status="200 OK" size=230 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd"
time="2023-10-18T13:24:47.156066764-06:00" level=debug msg="fetch response received" digest="sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-length=301261 response.header.content-md5="R+Y0FZj1Wto7rT9gTULVcg==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:24:47 GMT" response.header.etag="\"0x8D8DECD29E1022C\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=229e9453247200b7e7807ee424ce446e35b357d8 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=76d654fa-5faf-44e7-be31-1fab2ada3763 response.header.x-ms-copy-progress=301261/301261 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=be9ac745-801e-00d8-30f8-0161ef000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kcgs7200046-IAD, cache-bur-kbur8200082-BUR" response.status="200 OK" size=301261 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a"
time="2023-10-18T13:24:47.162450331-06:00" level=debug msg="fetch response received" digest="sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-disposition= response.header.content-length=2233 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:24:47 GMT" response.header.etag="\"0x8D950DD3FE78031\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:23 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=f27def3c8181c2a3e66f6bbccc01c29513e2564d response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:23 GMT" response.header.x-ms-copy-id=07945db6-426d-45cd-8734-4c1ace32645f response.header.x-ms-copy-progress=2233/2233 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:23 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=e8044889-201e-0086-07f8-01a0c1000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kiad7000130-IAD, cache-bur-kbur8200082-BUR" response.status="200 OK" size=2233 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa"
time="2023-10-18T13:24:47.193006061-06:00" level=debug msg="fetch response received" digest="sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-length=31506523 response.header.content-md5="Zv0aX6nCRCbDhK8ON7Qm6w==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:24:47 GMT" response.header.etag="\"0x8D8DECD29E65DF1\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=d656506b365e01c17510afd00af2670aa3a84f0b response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=5640d7c0-6252-4446-86dc-177b315dbf08 response.header.x-ms-copy-progress=31506523/31506523 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=973a2f0f-f01e-00af-5ff8-0142e1000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kiad7000025-IAD, cache-bur-kbur8200082-BUR" response.status="200 OK" size=31506523 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e"
time="2023-10-18T13:24:47.194848996-06:00" level=debug msg="fetch response received" digest="sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-disposition= response.header.content-length=637463 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:24:47 GMT" response.header.etag="\"0x8D950DD4137CD4C\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:26 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=2dc7bf1454e6f08051a3bd419f27683d27d3a5ad response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:26 GMT" response.header.x-ms-copy-id=5112efd1-4292-4e39-bc9b-e40a2203877f response.header.x-ms-copy-progress=637463/637463 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:26 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=553c8f16-201e-0086-36f8-01a0c1000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kjyo7100097-IAD, cache-bur-kbur8200082-BUR" response.status="200 OK" size=637463 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d"
time="2023-10-18T13:24:47.202099301-06:00" level=debug msg="fetch response received" digest="sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-length=1887472 response.header.content-md5="i9atZyjofrewlZ3ngsMOnA==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:24:47 GMT" response.header.etag="\"0x8D8DECD29DEACF9\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=55cf21b5e04f798abb060564ff064537ee986ec3 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=13d27f76-4a1b-4ae1-8f7a-582a9ed3e3e3 response.header.x-ms-copy-progress=1887472/1887472 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=0ac68026-f01e-0017-59f8-016cb4000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kcgs7200111-IAD, cache-bur-kbur8200082-BUR" response.status="200 OK" size=1887472 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d"
time="2023-10-18T13:24:47.330927502-06:00" level=debug msg="fetch response received" digest="sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-disposition= response.header.content-length=20365161 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:24:47 GMT" response.header.etag="\"0x8D950DD41BFE9A7\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:27 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=7ab3893f3e47357e62851c0e69aa98ebba68c701 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:27 GMT" response.header.x-ms-copy-id=6c84899f-271a-4bda-89af-0d225eed6986 response.header.x-ms-copy-progress=20365161/20365161 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:27 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=c82ed068-801e-008c-49f8-01aed4000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kjyo7100146-IAD, cache-bur-kbur8200082-BUR" response.status="200 OK" size=20365161 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a"
time="2023-10-18T13:24:52.122530681-06:00" level=debug msg="image unpacked" chainID="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253" config="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734"
time="2023-10-18T13:24:52.194835950-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:24:52.194992476-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:24:53.982826775-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:24:54.095914648-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:24:54.269555740-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:24:54.269877815-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:24:54.283780394-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:24:54.970899604-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=1
time="2023-10-18T13:24:54.974427115-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=1
time="2023-10-18T13:24:54.975015115-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=1
time="2023-10-18T13:24:54.976710263-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=1
time="2023-10-18T13:24:54.976892661-06:00" level=debug msg="Removing instance" fID=1 isSync=true
time="2023-10-18T13:24:54.976977520-06:00" level=debug msg="Orchestrator received StopVM" vmID=1-0
time="2023-10-18T13:25:04.990707307-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 1-0" vmID=1-0
time="2023-10-18T13:25:04.990975585-06:00" level=debug msg="Freeing a VM instance" vmID=1-0
time="2023-10-18T13:25:04.991107637-06:00" level=debug msg="Removing network config for function instance" funcID=1-0
time="2023-10-18T13:25:04.991241397-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=1-0
time="2023-10-18T13:25:04.991416671-06:00" level=debug msg="Stopped VM successfully" vmID=1-0
--- PASS: TestSendToFunctionSerial (20.21s)
=== RUN   TestSendToFunctionParallel
time="2023-10-18T13:25:04.992628235-06:00" level=debug msg="fIDint=2, err=<nil>, pinnedFuncNum=0"
time="2023-10-18T13:25:04.992843335-06:00" level=debug msg="Created function, pinned=true, shut down after 0 requests" fID=2 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:25:04.993089294-06:00" level=info msg="New function added" fID=2 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T13:25:04.993311899-06:00" level=debug msg="Function is inactive, starting the instance..." fID=2
time="2023-10-18T13:25:04.993463638-06:00" level=debug msg="Adding instance" fID=2
time="2023-10-18T13:25:04.993616208-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:25:04.993818632-06:00" level=debug msg="Allocating a VM instance" vmID=2-0
time="2023-10-18T13:25:04.993975299-06:00" level=debug msg="Creating network config for function instance" funcID=2-0
time="2023-10-18T13:25:04.994408450-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=2-0
time="2023-10-18T13:25:04.994724560-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=2-0
time="2023-10-18T13:25:05.000298859-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns70 tap=tap0
time="2023-10-18T13:25:05.057333018-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:25:05.057632125-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:25:06.921232685-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:25:07.023865076-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:25:07.195455858-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:25:07.195822192-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:25:07.208949045-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:25:07.911694473-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912674915-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913153757-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.911913924-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912841452-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914124008-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914332687-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914465712-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913101798-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913261091-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914419631-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913845884-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912052769-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912354392-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.911806566-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913150081-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913788052-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913684336-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914653344-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914758400-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912238958-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912387275-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912357987-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912483631-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912391545-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912552943-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912522722-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912512141-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912589225-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912755691-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912944948-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912864780-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912972312-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913031844-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913495236-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913328499-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913619684-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913579661-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913388974-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.912069758-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913753057-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914275410-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914031348-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914306182-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914308627-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.913969981-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914359804-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914527406-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914509284-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914584510-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914732630-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914879280-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914971697-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.914908806-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915012913-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914994480-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.914991913-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915091571-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915144134-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915199493-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915244064-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915263700-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915318823-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915398292-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915409144-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915426366-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915443087-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915447477-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915940689-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915639276-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915727059-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915775149-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915782660-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915764413-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915679831-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915561217-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.915603974-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.916116283-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916161055-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916261767-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916173073-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916354995-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916343355-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916375256-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916467437-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916521112-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916651613-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916610885-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.916686397-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916566434-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916677985-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916794986-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916806770-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916883446-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916919047-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.916941830-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.917018121-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.917043514-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.917109977-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.917085242-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.917232440-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.917171985-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.917118730-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.917291064-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:25:07.917622480-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.918720902-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.919239977-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.919891700-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.920277318-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.925785409-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.920675158-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.927937703-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.922142718-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.921480844-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.928270448-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.923130808-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.923407215-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.923785140-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.924212965-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.924412870-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.924687292-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.924955656-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.932888535-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.932131881-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.930121914-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.930629098-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.930946048-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.931585425-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.931239791-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.928851516-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.935874841-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.936283236-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.936513277-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.936800103-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.937095124-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.937399433-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.937693611-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.937964754-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.938260393-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.938516263-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.938786217-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.939051730-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.939381539-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.939713357-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.939893736-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.940228479-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.940692482-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.940838099-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.941342559-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.941476030-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.941926572-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.942294264-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.942644752-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.942944660-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.943532219-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.943731221-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.943938332-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.944416688-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.944670753-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.945029302-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.945266474-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.945728181-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.945987230-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.946341911-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.946681497-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.947066202-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.947291560-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.947753941-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.948053190-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.948463820-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.948817749-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.949112601-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.949400494-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.949920302-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.950485674-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.951214748-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.951788302-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.952081718-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.952492275-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.952958745-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.953533663-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.953995357-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.955048057-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.955356586-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.955627436-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.956101959-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.956510661-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.956967468-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.957448303-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.957992482-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.958459637-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.959136240-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.959570378-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.959984580-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.960417779-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.961028424-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.961569966-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.962037141-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.962557600-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.963076122-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:25:07.963393866-06:00" level=debug msg="Removing instance" fID=2 isSync=true
time="2023-10-18T13:25:07.963544308-06:00" level=debug msg="Orchestrator received StopVM" vmID=2-0
time="2023-10-18T13:25:17.977741371-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 2-0" vmID=2-0
time="2023-10-18T13:25:17.978157625-06:00" level=debug msg="Freeing a VM instance" vmID=2-0
time="2023-10-18T13:25:17.978316130-06:00" level=debug msg="Removing network config for function instance" funcID=2-0
time="2023-10-18T13:25:17.978480799-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=2-0
time="2023-10-18T13:25:17.978683811-06:00" level=debug msg="Stopped VM successfully" vmID=2-0
--- PASS: TestSendToFunctionParallel (12.99s)
=== RUN   TestStartSendStopTwice
time="2023-10-18T13:25:17.979648777-06:00" level=debug msg="fIDint=3, err=<nil>, pinnedFuncNum=2"
time="2023-10-18T13:25:17.979883049-06:00" level=debug msg="Created function, pinned=true, shut down after 1 requests" fID=3 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:25:17.980119366-06:00" level=info msg="New function added" fID=3 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T13:25:17.980353789-06:00" level=debug msg="Function is inactive, starting the instance..." fID=3
time="2023-10-18T13:25:17.980491970-06:00" level=debug msg="Adding instance" fID=3
time="2023-10-18T13:25:17.980628873-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:25:17.980833063-06:00" level=debug msg="Allocating a VM instance" vmID=3-0
time="2023-10-18T13:25:17.980985285-06:00" level=debug msg="Creating network config for function instance" funcID=3-0
time="2023-10-18T13:25:17.981205968-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=3-0
time="2023-10-18T13:25:17.981488707-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=3-0
time="2023-10-18T13:25:17.987200483-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns71 tap=tap0
time="2023-10-18T13:25:18.046234046-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:25:18.046536683-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:25:19.811807303-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:25:19.913558628-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:25:20.082266552-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:25:20.082598031-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:25:20.095144559-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:25:20.783547902-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=3
time="2023-10-18T13:25:20.785418425-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=3
time="2023-10-18T13:25:20.785832244-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=3
time="2023-10-18T13:25:20.787295120-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=3
time="2023-10-18T13:25:20.787611447-06:00" level=debug msg="Removing instance" fID=3 isSync=true
time="2023-10-18T13:25:20.787798478-06:00" level=debug msg="Orchestrator received StopVM" vmID=3-0
time="2023-10-18T13:25:30.802558599-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 3-0" vmID=3-0
time="2023-10-18T13:25:30.802876356-06:00" level=debug msg="Freeing a VM instance" vmID=3-0
time="2023-10-18T13:25:30.803064059-06:00" level=debug msg="Removing network config for function instance" funcID=3-0
time="2023-10-18T13:25:30.803234405-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=3-0
time="2023-10-18T13:25:30.803451547-06:00" level=debug msg="Stopped VM successfully" vmID=3-0
time="2023-10-18T13:25:30.803996663-06:00" level=debug msg="Function is inactive, starting the instance..." fID=3
time="2023-10-18T13:25:30.804154867-06:00" level=debug msg="Adding instance" fID=3
time="2023-10-18T13:25:30.804322001-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-1
time="2023-10-18T13:25:30.804478902-06:00" level=debug msg="Allocating a VM instance" vmID=3-1
time="2023-10-18T13:25:30.804658493-06:00" level=debug msg="Creating network config for function instance" funcID=3-1
time="2023-10-18T13:25:30.804911027-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=3-1
time="2023-10-18T13:25:30.805209506-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=3-1
time="2023-10-18T13:25:30.811277718-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns72 tap=tap0
time="2023-10-18T13:25:30.868985640-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:25:30.869114046-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:25:32.631709497-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-1
time="2023-10-18T13:25:32.724513458-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-1
time="2023-10-18T13:25:32.893659916-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-1
time="2023-10-18T13:25:32.893985682-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-1
time="2023-10-18T13:25:32.907319049-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-1
time="2023-10-18T13:25:33.605627719-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=3
time="2023-10-18T13:25:33.607860695-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=3
time="2023-10-18T13:25:33.608206965-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=3
time="2023-10-18T13:25:33.609509248-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=3
time="2023-10-18T13:25:33.609759437-06:00" level=debug msg="Removing instance" fID=3 isSync=true
time="2023-10-18T13:25:33.609908202-06:00" level=debug msg="Orchestrator received StopVM" vmID=3-1
time="2023-10-18T13:25:43.623705378-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 3-1" vmID=3-1
time="2023-10-18T13:25:43.624030897-06:00" level=debug msg="Freeing a VM instance" vmID=3-1
time="2023-10-18T13:25:43.624194343-06:00" level=debug msg="Removing network config for function instance" funcID=3-1
time="2023-10-18T13:25:43.624346970-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=3-1
time="2023-10-18T13:25:43.624546170-06:00" level=debug msg="Stopped VM successfully" vmID=3-1
--- PASS: TestStartSendStopTwice (25.65s)
=== RUN   TestStatsNotNumericFunction
time="2023-10-18T13:25:43.625731315-06:00" level=debug msg="fIDint=0, err=strconv.Atoi: parsing \"not-cld\": invalid syntax, pinnedFuncNum=2"
time="2023-10-18T13:25:43.625964153-06:00" level=debug msg="Created function, pinned=true, shut down after 1 requests" fID=not-cld imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:25:43.626176898-06:00" level=info msg="New function added" fID=not-cld image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T13:25:43.626424284-06:00" level=debug msg="Function is inactive, starting the instance..." fID=not-cld
time="2023-10-18T13:25:43.626567556-06:00" level=debug msg="Adding instance" fID=not-cld
time="2023-10-18T13:25:43.626737080-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:25:43.626944516-06:00" level=debug msg="Allocating a VM instance" vmID=not-cld-0
time="2023-10-18T13:25:43.627110224-06:00" level=debug msg="Creating network config for function instance" funcID=not-cld-0
time="2023-10-18T13:25:43.627380703-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=not-cld-0
time="2023-10-18T13:25:43.627714545-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=not-cld-0
time="2023-10-18T13:25:43.633211052-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns73 tap=tap0
time="2023-10-18T13:25:43.695828180-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:25:43.695999779-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:25:45.559186779-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:25:45.645819478-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:25:45.819514340-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:25:45.819740773-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:25:45.835687782-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:25:46.536019122-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=not-cld
time="2023-10-18T13:25:46.538021624-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=not-cld
time="2023-10-18T13:25:46.538336929-06:00" level=debug msg="Removing instance" fID=not-cld isSync=true
time="2023-10-18T13:25:46.538540158-06:00" level=debug msg="Orchestrator received StopVM" vmID=not-cld-0
time="2023-10-18T13:25:56.552509079-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM not-cld-0" vmID=not-cld-0
time="2023-10-18T13:25:56.552795824-06:00" level=debug msg="Freeing a VM instance" vmID=not-cld-0
time="2023-10-18T13:25:56.553270432-06:00" level=debug msg="Removing network config for function instance" funcID=not-cld-0
time="2023-10-18T13:25:56.553425855-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=not-cld-0
time="2023-10-18T13:25:56.553971827-06:00" level=debug msg="Stopped VM successfully" vmID=not-cld-0
--- PASS: TestStatsNotNumericFunction (12.93s)
=== RUN   TestStatsNotColdFunction
time="2023-10-18T13:25:56.555009173-06:00" level=debug msg="fIDint=4, err=<nil>, pinnedFuncNum=4"
time="2023-10-18T13:25:56.555233976-06:00" level=debug msg="Created function, pinned=true, shut down after 1 requests" fID=4 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:25:56.555569853-06:00" level=info msg="New function added" fID=4 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T13:25:56.555827175-06:00" level=debug msg="Function is inactive, starting the instance..." fID=4
time="2023-10-18T13:25:56.556029188-06:00" level=debug msg="Adding instance" fID=4
time="2023-10-18T13:25:56.556239073-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:25:56.556467478-06:00" level=debug msg="Allocating a VM instance" vmID=4-0
time="2023-10-18T13:25:56.556719889-06:00" level=debug msg="Creating network config for function instance" funcID=4-0
time="2023-10-18T13:25:56.556998354-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=4-0
time="2023-10-18T13:25:56.557330345-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=4-0
time="2023-10-18T13:25:56.562721418-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns74 tap=tap0
time="2023-10-18T13:25:56.623734676-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:25:56.623912379-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:25:58.484801197-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:25:58.589166117-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:25:58.761276152-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:25:58.761598522-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:25:58.775896580-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:25:59.471463735-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=4
time="2023-10-18T13:25:59.473306439-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=4
time="2023-10-18T13:25:59.473597494-06:00" level=debug msg="Removing instance" fID=4 isSync=true
time="2023-10-18T13:25:59.473794034-06:00" level=debug msg="Orchestrator received StopVM" vmID=4-0
time="2023-10-18T13:26:09.487629879-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 4-0" vmID=4-0
time="2023-10-18T13:26:09.488013868-06:00" level=debug msg="Freeing a VM instance" vmID=4-0
time="2023-10-18T13:26:09.488191718-06:00" level=debug msg="Removing network config for function instance" funcID=4-0
time="2023-10-18T13:26:09.488340089-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=4-0
time="2023-10-18T13:26:09.488551671-06:00" level=debug msg="Stopped VM successfully" vmID=4-0
--- PASS: TestStatsNotColdFunction (12.93s)
=== RUN   TestSaveMemorySerial
time="2023-10-18T13:26:09.489616534-06:00" level=debug msg="fIDint=5, err=<nil>, pinnedFuncNum=2"
time="2023-10-18T13:26:09.489863723-06:00" level=debug msg="Created function, pinned=false, shut down after 40 requests" fID=5 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:26:09.490134043-06:00" level=info msg="New function added" fID=5 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=false servedTh=40
time="2023-10-18T13:26:09.490371124-06:00" level=debug msg="Function is inactive, starting the instance..." fID=5
time="2023-10-18T13:26:09.490532185-06:00" level=debug msg="Adding instance" fID=5
time="2023-10-18T13:26:09.490688387-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:26:09.490916825-06:00" level=debug msg="Allocating a VM instance" vmID=5-0
time="2023-10-18T13:26:09.491105003-06:00" level=debug msg="Creating network config for function instance" funcID=5-0
time="2023-10-18T13:26:09.491344386-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=5-0
time="2023-10-18T13:26:09.491731660-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=5-0
time="2023-10-18T13:26:09.497621645-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns75 tap=tap0
time="2023-10-18T13:26:09.555079556-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:26:09.555360782-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:26:11.517224917-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:26:11.619749252-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:26:11.788989380-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:26:11.789320181-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:26:11.801493746-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:26:12.481580717-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.483485494-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.483872597-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.485361637-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.485699041-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.487107883-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.487468855-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.488995108-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.489315743-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.490986331-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.491279100-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.492880988-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.493044703-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.494557296-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.494847435-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.496688005-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.496998445-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.498548105-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.498725203-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.500234228-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.500541775-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.501962418-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.502310383-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.503598663-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.503860755-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.505179949-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.505411769-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.506734034-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.507000027-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.508384312-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.508556490-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.509770614-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.510042414-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.511397297-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.511617883-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.513005790-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.513299826-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.514528030-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.514747702-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.515999598-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.516262962-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.517589909-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.517856660-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.519267520-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.519491945-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.520805507-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.521071782-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.522454890-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.522736405-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.524319068-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.524590539-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.525963286-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.526221058-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.527760792-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.528086852-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.529548294-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.529850111-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.531414678-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.531709121-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.533297974-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.533584442-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.535081520-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.535391837-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.536769990-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.537072345-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.538621268-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.538887011-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.540227468-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.540439163-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.541681853-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.541961398-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.543406845-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.543701430-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.545170827-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.545426812-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.546811953-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.547021070-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.548379160-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.548654812-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:12.549954880-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:12.550150775-06:00" level=debug msg="Function has to shut down its instance, served 40 requests" fID=5
time="2023-10-18T13:26:12.550294011-06:00" level=debug msg="Removing instance" fID=5 isSync=false
time="2023-10-18T13:26:12.550421225-06:00" level=debug msg="Removing instance (async)" fID=5
time="2023-10-18T13:26:12.550748455-06:00" level=debug msg="Function is inactive, starting the instance..." fID=5
time="2023-10-18T13:26:12.550877757-06:00" level=debug msg="Adding instance" fID=5
time="2023-10-18T13:26:12.550752407-06:00" level=debug msg="Orchestrator received StopVM" vmID=5-0
time="2023-10-18T13:26:12.551083867-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-1
time="2023-10-18T13:26:12.551499049-06:00" level=debug msg="Allocating a VM instance" vmID=5-1
time="2023-10-18T13:26:12.551669689-06:00" level=debug msg="Creating network config for function instance" funcID=5-1
time="2023-10-18T13:26:12.551894779-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=5-1
time="2023-10-18T13:26:12.552166024-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.76 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns75 NamespacePath=/var/run/netns/uvmns75 Veth0CIDR=172.17.1.46/30 Veth0Name=veth75-0 Veth1CIDR=172.17.1.45/30 Veth1Name=veth75-1 funcID=5-1
time="2023-10-18T13:26:12.557667599-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns76 tap=tap0
time="2023-10-18T13:26:12.613986120-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:26:12.614240061-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:26:14.374474568-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-1
time="2023-10-18T13:26:14.480927227-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-1
time="2023-10-18T13:26:14.652393451-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-1
time="2023-10-18T13:26:14.652706581-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-1
time="2023-10-18T13:26:14.667254121-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-1
time="2023-10-18T13:26:15.364149470-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.366225698-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.366555871-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.368047340-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.368371529-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.369855831-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.370191751-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.371478795-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.371765450-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.373270814-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.373560787-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.375013040-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.375296765-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.376636827-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.376883602-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.378330274-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.378606137-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.379902780-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.380181232-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.381572964-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.381876458-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.383241073-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.383526922-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.385221991-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.385507009-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.387042755-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.387332848-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.388932273-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.389196274-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.390779725-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.391049684-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.392629102-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.392964130-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.394461322-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.394661468-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.395958427-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.396285807-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.397633503-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.397945497-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.399276781-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.399579700-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.400811204-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.401101767-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.402527659-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.402844334-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.404261319-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.404602962-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.406009963-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.406191706-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.407333954-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.407506718-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.408582155-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.408754002-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.409825808-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.410116372-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.411454229-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.411692024-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.412939308-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.413220507-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.414534293-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.414840898-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.416168025-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.416443614-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.417745856-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.418037733-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.419309137-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.419617409-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.421040010-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.421282106-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.422600789-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.422828031-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.424109276-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.424412814-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.425842918-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.426077515-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.427348160-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.427975466-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.429331399-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.429577781-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:15.430789306-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:15.430913156-06:00" level=debug msg="Function has to shut down its instance, served 40 requests" fID=5
time="2023-10-18T13:26:15.431008748-06:00" level=debug msg="Removing instance" fID=5 isSync=false
time="2023-10-18T13:26:15.431116833-06:00" level=debug msg="Removing instance (async)" fID=5
time="2023-10-18T13:26:15.431308981-06:00" level=debug msg="Function is inactive, starting the instance..." fID=5
time="2023-10-18T13:26:15.431393654-06:00" level=debug msg="Adding instance" fID=5
time="2023-10-18T13:26:15.431487063-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-2
time="2023-10-18T13:26:15.431362116-06:00" level=debug msg="Orchestrator received StopVM" vmID=5-1
time="2023-10-18T13:26:15.431574636-06:00" level=debug msg="Allocating a VM instance" vmID=5-2
time="2023-10-18T13:26:15.431923680-06:00" level=debug msg="Creating network config for function instance" funcID=5-2
time="2023-10-18T13:26:15.432066682-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=5-2
time="2023-10-18T13:26:15.432232246-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.77 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns76 NamespacePath=/var/run/netns/uvmns76 Veth0CIDR=172.17.1.50/30 Veth0Name=veth76-0 Veth1CIDR=172.17.1.49/30 Veth1Name=veth76-1 funcID=5-2
time="2023-10-18T13:26:15.436888514-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns77 tap=tap0
time="2023-10-18T13:26:15.491829685-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:26:15.492091354-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:26:17.379209436-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-2
time="2023-10-18T13:26:17.494455797-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-2
time="2023-10-18T13:26:17.668651043-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-2
time="2023-10-18T13:26:17.668980980-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-2
time="2023-10-18T13:26:17.680498224-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-2
time="2023-10-18T13:26:18.386129518-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.387886362-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.388241139-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.389580629-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.389900521-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.391176439-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.391484119-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.392799493-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.393172182-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.394429476-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.394750130-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.396248528-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.396592130-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.398139259-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.398461717-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.400203539-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.400496291-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.402264044-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.402455724-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.403821449-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.403969552-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.405291302-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.405563728-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.406808159-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.407025098-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.408222790-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.408531038-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.409938702-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.410128408-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.411314796-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.411521667-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.412860613-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.413130405-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.414560702-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.414800375-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.416128142-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.416363919-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.417812819-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.418035608-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:26:18.419450299-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:26:18.419739249-06:00" level=debug msg="Removing instance" fID=5 isSync=true
time="2023-10-18T13:26:18.419935819-06:00" level=debug msg="Orchestrator received StopVM" vmID=5-2
time="2023-10-18T13:26:22.565484128-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-0" vmID=5-0
time="2023-10-18T13:26:22.565797806-06:00" level=debug msg="Freeing a VM instance" vmID=5-0
time="2023-10-18T13:26:22.565951259-06:00" level=debug msg="Removing network config for function instance" funcID=5-0
time="2023-10-18T13:26:22.566097829-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=5-0
time="2023-10-18T13:26:22.566283370-06:00" level=debug msg="Stopped VM successfully" vmID=5-0
time="2023-10-18T13:26:25.446789210-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-1" vmID=5-1
time="2023-10-18T13:26:25.447125297-06:00" level=debug msg="Freeing a VM instance" vmID=5-1
time="2023-10-18T13:26:25.447278212-06:00" level=debug msg="Removing network config for function instance" funcID=5-1
time="2023-10-18T13:26:25.447442667-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=5-1
time="2023-10-18T13:26:25.447620991-06:00" level=debug msg="Stopped VM successfully" vmID=5-1
time="2023-10-18T13:26:28.438058642-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-2" vmID=5-2
time="2023-10-18T13:26:28.438386998-06:00" level=debug msg="Freeing a VM instance" vmID=5-2
time="2023-10-18T13:26:28.438563655-06:00" level=debug msg="Removing network config for function instance" funcID=5-2
time="2023-10-18T13:26:28.438729770-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=5-2
time="2023-10-18T13:26:28.438922585-06:00" level=debug msg="Stopped VM successfully" vmID=5-2
--- PASS: TestSaveMemorySerial (18.95s)
=== RUN   TestSaveMemoryParallel
time="2023-10-18T13:26:28.440302013-06:00" level=debug msg="fIDint=6, err=<nil>, pinnedFuncNum=2"
time="2023-10-18T13:26:28.440506299-06:00" level=debug msg="Created function, pinned=false, shut down after 40 requests" fID=6 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:26:28.440729655-06:00" level=info msg="New function added" fID=6 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=false servedTh=40
time="2023-10-18T13:26:28.441014631-06:00" level=debug msg="Function is inactive, starting the instance..." fID=6
time="2023-10-18T13:26:28.441163517-06:00" level=debug msg="Adding instance" fID=6
time="2023-10-18T13:26:28.441318429-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:26:28.441518675-06:00" level=debug msg="Allocating a VM instance" vmID=6-0
time="2023-10-18T13:26:28.441686323-06:00" level=debug msg="Creating network config for function instance" funcID=6-0
time="2023-10-18T13:26:28.441921807-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=6-0
time="2023-10-18T13:26:28.442179711-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.77 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns76 NamespacePath=/var/run/netns/uvmns76 Veth0CIDR=172.17.1.50/30 Veth0Name=veth76-0 Veth1CIDR=172.17.1.49/30 Veth1Name=veth76-1 funcID=6-0
time="2023-10-18T13:26:28.447812324-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns78 tap=tap0
time="2023-10-18T13:26:28.514157896-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:26:28.514363710-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:26:30.380979747-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:26:30.469230374-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:26:30.646932462-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:26:30.647312529-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:26:30.661054746-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:26:31.356695485-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357003386-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357237537-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357504907-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.356986563-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358072950-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358355114-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358404700-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357091703-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.359081057-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357181109-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357488849-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357636351-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357443597-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358768255-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358730717-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357317845-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357865210-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358090919-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.356788917-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358160138-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357660555-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357613013-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357614203-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357799042-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357813606-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358018399-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357844241-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358282875-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358225036-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358555817-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358451373-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358583308-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358636159-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.356922647-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358947865-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358948647-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357133163-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.358858459-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.357425521-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:31.360544928-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.360107060-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.361279772-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.363008089-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.361582378-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.367101261-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.363466035-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.364406901-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.364831175-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.368087760-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.368338377-06:00" level=debug msg="Function has to shut down its instance, served 40 requests" fID=6
time="2023-10-18T13:26:31.368594493-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.369111064-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.369400372-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.369845997-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.370465040-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.371056953-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.371353574-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.371848028-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.372137693-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.372481217-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.372985461-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.373331679-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.373673932-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.373885880-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.374343124-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.374864551-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.375404778-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.375996278-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.376606751-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.377107904-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.377840357-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.378313946-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.378913186-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.379377846-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.380303511-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.380509728-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.381223073-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.381412729-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.382049259-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.382378552-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:31.382704656-06:00" level=debug msg="Removing instance" fID=6 isSync=false
time="2023-10-18T13:26:31.382915074-06:00" level=debug msg="Removing instance (async)" fID=6
time="2023-10-18T13:26:31.383213885-06:00" level=debug msg="Orchestrator received StopVM" vmID=6-0
time="2023-10-18T13:26:31.383307440-06:00" level=debug msg="Function is inactive, starting the instance..." fID=6
time="2023-10-18T13:26:31.383927429-06:00" level=debug msg="Adding instance" fID=6
time="2023-10-18T13:26:31.384151851-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-1
time="2023-10-18T13:26:31.384402620-06:00" level=debug msg="Allocating a VM instance" vmID=6-1
time="2023-10-18T13:26:31.384514442-06:00" level=debug msg="Creating network config for function instance" funcID=6-1
time="2023-10-18T13:26:31.384644890-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=6-1
time="2023-10-18T13:26:31.384822518-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.79 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns78 NamespacePath=/var/run/netns/uvmns78 Veth0CIDR=172.17.1.58/30 Veth0Name=veth78-0 Veth1CIDR=172.17.1.57/30 Veth1Name=veth78-1 funcID=6-1
time="2023-10-18T13:26:31.390355767-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns79 tap=tap0
time="2023-10-18T13:26:31.450663104-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:26:31.450931463-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:26:33.212024889-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-1
time="2023-10-18T13:26:33.307434325-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-1
time="2023-10-18T13:26:33.481135970-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-1
time="2023-10-18T13:26:33.481354951-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-1
time="2023-10-18T13:26:33.495146864-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-1
time="2023-10-18T13:26:34.185114340-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.185653309-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.185942937-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186148375-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.185288017-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186556116-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186782309-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.187268504-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.187575520-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.187172834-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.187162707-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.185406768-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.185542768-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.185576576-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186813987-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.185455813-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186261436-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186018264-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.189644668-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.187668905-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.187843735-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186022136-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.185786594-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186078779-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.185865529-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186346870-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186427980-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186342022-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186489654-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186613461-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186706953-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186896964-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.186985695-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.187004369-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.187076415-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.187380096-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.187497992-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.187976612-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.187452609-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.189961083-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.187892388-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.191523739-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.187958015-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.188857355-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.189319955-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.187734967-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:34.191123187-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.193071917-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.194886696-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.193441829-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.193776820-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.194124471-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.194626968-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.195225029-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.192187639-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.195594450-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.197374155-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.197686204-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.198053127-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.198510551-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.198797273-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.199146313-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.199613992-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.200154519-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.200588079-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.201232378-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.201537567-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.202198741-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.202436752-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.202993318-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.203482083-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.203850566-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.204430957-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.204999398-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.205594091-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.206108803-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.206493720-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.206830258-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.207200395-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.207757038-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:34.208021979-06:00" level=debug msg="Function has to shut down its instance, served 40 requests" fID=6
time="2023-10-18T13:26:34.208215739-06:00" level=debug msg="Removing instance" fID=6 isSync=false
time="2023-10-18T13:26:34.208400726-06:00" level=debug msg="Removing instance (async)" fID=6
time="2023-10-18T13:26:34.208687228-06:00" level=debug msg="Orchestrator received StopVM" vmID=6-1
time="2023-10-18T13:26:34.208765326-06:00" level=debug msg="Function is inactive, starting the instance..." fID=6
time="2023-10-18T13:26:34.209137946-06:00" level=debug msg="Adding instance" fID=6
time="2023-10-18T13:26:34.209229277-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-2
time="2023-10-18T13:26:34.209352992-06:00" level=debug msg="Allocating a VM instance" vmID=6-2
time="2023-10-18T13:26:34.209452283-06:00" level=debug msg="Creating network config for function instance" funcID=6-2
time="2023-10-18T13:26:34.209580343-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=6-2
time="2023-10-18T13:26:34.209739371-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.80 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns79 NamespacePath=/var/run/netns/uvmns79 Veth0CIDR=172.17.1.62/30 Veth0Name=veth79-0 Veth1CIDR=172.17.1.61/30 Veth1Name=veth79-1 funcID=6-2
time="2023-10-18T13:26:34.213832716-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns80 tap=tap0
time="2023-10-18T13:26:34.264019432-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:26:34.264258582-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:26:36.029199804-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-2
time="2023-10-18T13:26:36.132549948-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-2
time="2023-10-18T13:26:36.303277113-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-2
time="2023-10-18T13:26:36.303580109-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-2
time="2023-10-18T13:26:36.317352190-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-2
time="2023-10-18T13:26:37.029524758-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.029701963-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.029899774-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.029616673-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.030345423-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.030639171-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.030355727-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.030630370-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.029890070-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.029980131-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.029873775-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.030102878-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.030240303-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.032938732-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.029673032-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.033244685-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.033873384-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.030460212-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.030554937-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.034538190-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.030489253-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.030102357-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.030734467-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.030689335-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:26:37.035150300-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.035920781-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.036765472-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.037348401-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.037538355-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.038121449-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.038475710-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.038987068-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.039373681-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.039871656-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.040248902-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.040801329-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.041121793-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.041725907-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.042053307-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.042498941-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:26:37.042961688-06:00" level=debug msg="Removing instance" fID=6 isSync=true
time="2023-10-18T13:26:37.043159920-06:00" level=debug msg="Orchestrator received StopVM" vmID=6-2
time="2023-10-18T13:26:41.397462001-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-0" vmID=6-0
time="2023-10-18T13:26:41.397803167-06:00" level=debug msg="Freeing a VM instance" vmID=6-0
time="2023-10-18T13:26:41.398018357-06:00" level=debug msg="Removing network config for function instance" funcID=6-0
time="2023-10-18T13:26:41.398197026-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=6-0
time="2023-10-18T13:26:41.398401237-06:00" level=debug msg="Stopped VM successfully" vmID=6-0
time="2023-10-18T13:26:44.219014425-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-1" vmID=6-1
time="2023-10-18T13:26:44.219364160-06:00" level=debug msg="Freeing a VM instance" vmID=6-1
time="2023-10-18T13:26:44.219581933-06:00" level=debug msg="Removing network config for function instance" funcID=6-1
time="2023-10-18T13:26:44.219782105-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=6-1
time="2023-10-18T13:26:44.220088907-06:00" level=debug msg="Stopped VM successfully" vmID=6-1
time="2023-10-18T13:26:47.060637921-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-2" vmID=6-2
time="2023-10-18T13:26:47.061045983-06:00" level=debug msg="Freeing a VM instance" vmID=6-2
time="2023-10-18T13:26:47.061259390-06:00" level=debug msg="Removing network config for function instance" funcID=6-2
time="2023-10-18T13:26:47.061401075-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=6-2
time="2023-10-18T13:26:47.061586987-06:00" level=debug msg="Stopped VM successfully" vmID=6-2
--- PASS: TestSaveMemoryParallel (18.62s)
=== RUN   TestDirectStartStopVM
time="2023-10-18T13:26:47.062693488-06:00" level=debug msg="fIDint=7, err=<nil>, pinnedFuncNum=0"
time="2023-10-18T13:26:47.062921769-06:00" level=debug msg="Created function, pinned=true, shut down after 0 requests" fID=7 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:26:47.063163351-06:00" level=info msg="New function added" fID=7 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T13:26:47.063385665-06:00" level=debug msg="Function is inactive, starting the instance..." fID=7
time="2023-10-18T13:26:47.063539515-06:00" level=debug msg="Adding instance" fID=7
time="2023-10-18T13:26:47.063716908-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:26:47.063917227-06:00" level=debug msg="Allocating a VM instance" vmID=7-0
time="2023-10-18T13:26:47.064102206-06:00" level=debug msg="Creating network config for function instance" funcID=7-0
time="2023-10-18T13:26:47.064337380-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=7-0
time="2023-10-18T13:26:47.064635772-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.80 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns79 NamespacePath=/var/run/netns/uvmns79 Veth0CIDR=172.17.1.62/30 Veth0Name=veth79-0 Veth1CIDR=172.17.1.61/30 Veth1Name=veth79-1 funcID=7-0
time="2023-10-18T13:26:47.070432392-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns81 tap=tap0
time="2023-10-18T13:26:47.132877708-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:26:47.133137928-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:26:48.899587070-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:26:49.011724491-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:26:49.179988006-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:26:49.180341584-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:26:49.194298868-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:26:49.884100709-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=7
time="2023-10-18T13:26:49.885990604-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=7
time="2023-10-18T13:26:49.886246406-06:00" level=debug msg="Removing instance" fID=7 isSync=true
time="2023-10-18T13:26:49.886382355-06:00" level=debug msg="Orchestrator received StopVM" vmID=7-0
time="2023-10-18T13:26:59.899591321-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 7-0" vmID=7-0
time="2023-10-18T13:26:59.899898513-06:00" level=debug msg="Freeing a VM instance" vmID=7-0
time="2023-10-18T13:26:59.900091174-06:00" level=debug msg="Removing network config for function instance" funcID=7-0
time="2023-10-18T13:26:59.900248710-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=7-0
time="2023-10-18T13:26:59.900447439-06:00" level=debug msg="Stopped VM successfully" vmID=7-0
--- PASS: TestDirectStartStopVM (12.84s)
=== RUN   TestAllFunctions
    vhive_test.go:290: skipping TestAllFunctions in non-nightly runs.
--- SKIP: TestAllFunctions (0.00s)
PASS
coverage: 46.0% of statements
time="2023-10-18T13:26:59.901664411-06:00" level=info msg="waiting for goroutines"
time="2023-10-18T13:26:59.901815004-06:00" level=info msg="waiting done"
time="2023-10-18T13:26:59.901931818-06:00" level=info msg="Closing fcClient"
time="2023-10-18T13:26:59.902087559-06:00" level=info msg="Closing containerd client"
time="2023-10-18T13:26:59.902411494-06:00" level=info msg="Cleaning up network manager"
time="2023-10-18T13:26:59.964096711-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.008739228-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.008712810-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.008691236-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.008681380-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.031881752-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.031838098-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.031821060-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.031850858-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.031832947-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.031896832-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.031904191-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.031955747-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.031948796-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.076039643-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.075880424-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.075900585-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.075914885-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.075914059-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.076102731-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.075994391-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.075971347-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:27:00.075901129-06:00" level=debug msg="Removing tap" tap=tap0
ok  	command-line-arguments	135.479s	coverage: 46.0% of statements
./scripts/clean_fcctr.sh
Killing firecracker agents and VMs
Resetting nftables
Deleting veth* devices created by CNI
Cannot find device "br0"
Cannot find device "br1"
Cleaning in /var/lib/cni/ non-network
find: ‘/var/lib/cni/’: No such file or directory
Removing devmapper devices
Removed fc-dev-thinpool-snap-13
device-mapper: remove ioctl on fc-dev-thinpool  failed: Device or resource busy
Command failed.
Removed fc-dev-thinpool-snap-12
Removed fc-dev-thinpool-snap-11
Removed fc-dev-thinpool-snap-10
Removed fc-dev-thinpool-snap-19
Removed fc-dev-thinpool-snap-21
Removed fc-dev-thinpool-snap-18
Removed fc-dev-thinpool-snap-20
Removed fc-dev-thinpool-snap-17
Removed fc-dev-thinpool-snap-16
Removed fc-dev-thinpool-snap-15
Removed fc-dev-thinpool-snap-14
Removed fc-dev-thinpool-snap-9
Cleaning /var/lib/firecracker-containerd/containerd /var/lib/firecracker-containerd/runtime /var/lib/firecracker-containerd/shim-base /var/lib/firecracker-containerd/snapshotter
Cleaning /run/firecracker-containerd/*
Cleaning CNI state, e.g., allocated addresses
rm: cannot remove '/var/lib/cni/networks/fcnet*/last_reserved_ip.0': No such file or directory
clean already
rm: cannot remove '/var/lib/cni/networks/fcnet*/19*': No such file or directory
clean already
Cleaning snapshots
Creating a fresh devmapper
0 209715200 thin-pool /dev/loop31 /dev/loop30 128 32768 1 skip_block_zeroing
device-mapper: reload ioctl on fc-dev-thinpool  failed: No such device or address
Command failed.
sudo mkdir -m777 -p /tmp/ctrd-logs && sudo env "PATH=/usr/local/go/bin:/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>/tmp/ctrd-logs/fccd_orch_noupf_log.out 2>/tmp/ctrd-logs/fccd_orch_noupf_log.err &
sudo env "PATH=/usr/local/go/bin:/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin" go test vhive_test.go stats.go vhive.go functions.go -short -v -race -cover -args -snapshotsTest
time="2023-10-18T13:27:05.846712718-06:00" level=info msg="Orchestrator snapshots enabled: true"
time="2023-10-18T13:27:05.846968969-06:00" level=info msg="Orchestrator UPF enabled: false"
time="2023-10-18T13:27:05.847046597-06:00" level=info msg="Orchestrator lazy serving mode enabled: false"
time="2023-10-18T13:27:05.847103167-06:00" level=info msg="Orchestrator UPF metrics enabled: false"
time="2023-10-18T13:27:05.847189997-06:00" level=info msg="Drop cache: true"
time="2023-10-18T13:27:05.847234015-06:00" level=info msg="Bench dir: bench_results"
time="2023-10-18T13:27:05.852102361-06:00" level=debug msg="Initializing network pool" poolSize=10
time="2023-10-18T13:27:05.861746898-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns60 tap=tap0
time="2023-10-18T13:27:05.864182067-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns61 tap=tap0
time="2023-10-18T13:27:05.865809500-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns59 tap=tap0
time="2023-10-18T13:27:05.867638838-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns63 tap=tap0
time="2023-10-18T13:27:05.869872892-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns62 tap=tap0
time="2023-10-18T13:27:05.872589267-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns68 tap=tap0
time="2023-10-18T13:27:05.873876288-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns64 tap=tap0
time="2023-10-18T13:27:05.876175029-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns65 tap=tap0
time="2023-10-18T13:27:05.876993284-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns66 tap=tap0
time="2023-10-18T13:27:05.877985689-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns67 tap=tap0
time="2023-10-18T13:27:05.888198605-06:00" level=info msg="Creating containerd client"
time="2023-10-18T13:27:05.889624058-06:00" level=info msg="Created containerd client"
time="2023-10-18T13:27:05.889739952-06:00" level=info msg="Creating firecracker client"
time="2023-10-18T13:27:05.890170448-06:00" level=info msg="Created firecracker client"
time="2023-10-18T13:27:05.890264897-06:00" level=info msg="Creating image manager"
=== RUN   TestSendToFunctionSerial
time="2023-10-18T13:27:05.890765902-06:00" level=debug msg="fIDint=1, err=<nil>, pinnedFuncNum=0"
time="2023-10-18T13:27:05.890868970-06:00" level=debug msg="Created function, pinned=true, shut down after 0 requests" fID=1 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:27:05.891243852-06:00" level=info msg="New function added" fID=1 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T13:27:05.891451549-06:00" level=debug msg="Function is inactive, starting the instance..." fID=1
time="2023-10-18T13:27:05.891632349-06:00" level=debug msg="Adding instance" fID=1
time="2023-10-18T13:27:05.892762871-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:27:05.893093669-06:00" level=debug msg="Allocating a VM instance" vmID=1-0
time="2023-10-18T13:27:05.893274476-06:00" level=debug msg="Creating network config for function instance" funcID=1-0
time="2023-10-18T13:27:05.893552513-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=1-0
time="2023-10-18T13:27:05.893771713-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=1-0
time="2023-10-18T13:27:05.899268539-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns69 tap=tap0
time="2023-10-18T13:27:06.000272719-06:00" level=debug msg=resolving host=ghcr.io
time="2023-10-18T13:27:06.000481578-06:00" level=debug msg="do request" host=ghcr.io request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=HEAD url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:27:06.405802252-06:00" level=debug msg="fetch response received" host=ghcr.io response.header.content-length=73 response.header.content-type=application/json response.header.date="Wed, 18 Oct 2023 19:27:06 GMT" response.header.www-authenticate="Bearer realm=\"https://ghcr.io/token\",service=\"ghcr.io\",scope=\"repository:ease-lab/helloworld:pull\"" response.header.x-github-request-id="A4FF:70FF:54B6:7FF3:6530318A" response.status="401 Unauthorized" url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:27:06.407247112-06:00" level=debug msg=Unauthorized header="Bearer realm=\"https://ghcr.io/token\",service=\"ghcr.io\",scope=\"repository:ease-lab/helloworld:pull\"" host=ghcr.io
time="2023-10-18T13:27:06.407574405-06:00" level=debug msg="do request" host=ghcr.io request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=HEAD url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:27:06.694520918-06:00" level=debug msg="fetch response received" host=ghcr.io response.header.content-length=1998 response.header.content-type=application/vnd.docker.distribution.manifest.v2+json response.header.date="Wed, 18 Oct 2023 19:27:06 GMT" response.header.docker-content-digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" response.header.docker-distribution-api-version=registry/2.0 response.header.etag="\"sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25\"" response.header.x-github-request-id="A4FF:70FF:54BD:8000:6530318A" response.status="200 OK" url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:27:06.694983535-06:00" level=debug msg=resolved desc.digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" host=ghcr.io
time="2023-10-18T13:27:06.695619223-06:00" level=debug msg=fetch digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" mediatype=application/vnd.docker.distribution.manifest.v2+json size=1998
time="2023-10-18T13:27:06.706230841-06:00" level=debug msg="do request" digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" mediatype=application/vnd.docker.distribution.manifest.v2+json request.header.accept="application/vnd.docker.distribution.manifest.v2+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=1998 url="https://ghcr.io/v2/ease-lab/helloworld/manifests/sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25"
time="2023-10-18T13:27:06.918914458-06:00" level=debug msg="fetch response received" digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" mediatype=application/vnd.docker.distribution.manifest.v2+json response.header.content-length=1998 response.header.content-type=application/vnd.docker.distribution.manifest.v2+json response.header.date="Wed, 18 Oct 2023 19:27:06 GMT" response.header.docker-content-digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" response.header.docker-distribution-api-version=registry/2.0 response.header.etag="\"sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25\"" response.header.x-github-request-id="A4FF:70FF:54C2:800A:6530318A" response.status="200 OK" size=1998 url="https://ghcr.io/v2/ease-lab/helloworld/manifests/sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25"
time="2023-10-18T13:27:06.935681415-06:00" level=debug msg=fetch digest="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734" mediatype=application/vnd.docker.container.image.v1+json size=8312
time="2023-10-18T13:27:06.941148223-06:00" level=debug msg="do request" digest="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734" mediatype=application/vnd.docker.container.image.v1+json request.header.accept="application/vnd.docker.container.image.v1+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=8312 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734"
time="2023-10-18T13:27:07.219677419-06:00" level=debug msg="fetch response received" digest="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734" mediatype=application/vnd.docker.container.image.v1+json response.header.accept-ranges=bytes response.header.age=0 response.header.content-disposition= response.header.content-length=8312 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:27:07 GMT" response.header.etag="\"0x8D950DD42AD3ECF\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:28 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=fb36f6c350d7cd3a0893195996722583ad82d60e response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:28 GMT" response.header.x-ms-copy-id=32f6a8de-4aab-4519-b15c-21acf3d16ee7 response.header.x-ms-copy-progress=8312/8312 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:28 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=e4ffb3ee-c01e-0074-7af9-015390000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kjyo7100158-IAD, cache-bur-kbur8200032-BUR" response.status="200 OK" size=8312 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734"
time="2023-10-18T13:27:08.187257206-06:00" level=debug msg=fetch digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=2802957
time="2023-10-18T13:27:08.187557880-06:00" level=debug msg=fetch digest="sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=1887472
time="2023-10-18T13:27:08.187371200-06:00" level=debug msg=fetch digest="sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=301261
time="2023-10-18T13:27:08.187382500-06:00" level=debug msg=fetch digest="sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=31506523
time="2023-10-18T13:27:08.187442294-06:00" level=debug msg=fetch digest="sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=230
time="2023-10-18T13:27:08.187909703-06:00" level=debug msg=fetch digest="sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=2233
time="2023-10-18T13:27:08.188209495-06:00" level=debug msg=fetch digest="sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=637463
time="2023-10-18T13:27:08.188043392-06:00" level=debug msg=fetch digest="sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=20365161
time="2023-10-18T13:27:08.192814991-06:00" level=debug msg="do request" digest="sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=1887472 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d"
time="2023-10-18T13:27:08.195331858-06:00" level=debug msg="do request" digest="sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=301261 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a"
time="2023-10-18T13:27:08.196679727-06:00" level=debug msg="do request" digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=2802957 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9"
time="2023-10-18T13:27:08.198121727-06:00" level=debug msg="do request" digest="sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=31506523 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e"
time="2023-10-18T13:27:08.199690954-06:00" level=debug msg="do request" digest="sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=2233 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa"
time="2023-10-18T13:27:08.202078640-06:00" level=debug msg="do request" digest="sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=230 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd"
time="2023-10-18T13:27:08.202489549-06:00" level=debug msg="do request" digest="sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=637463 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d"
time="2023-10-18T13:27:08.203969868-06:00" level=debug msg="do request" digest="sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=20365161 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a"
time="2023-10-18T13:27:08.382288955-06:00" level=debug msg="fetch response received" digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=121 response.header.content-length=2802957 response.header.content-md5="Xv6UEe5ym/lkcH1POk+jEQ==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:27:08 GMT" response.header.etag="\"0x8D8DEA430F3018D\"" response.header.last-modified="Thu, 04 Mar 2021 00:26:45 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="HIT, HIT" response.header.x-cache-hits="5, 1" response.header.x-fastly-request-id=64024820c98dd2d6d28c6a0b73fb2c95e2b9f296 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 00:26:45 GMT" response.header.x-ms-copy-id=5eaa9616-c230-429e-a0df-0f62ee445e79 response.header.x-ms-copy-progress=2802957/2802957 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 00:26:45 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=d1198414-a01e-001a-1bf8-01b3af000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kjyo7100129-IAD, cache-bur-kbur8200032-BUR" response.status="200 OK" size=2802957 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9"
time="2023-10-18T13:27:08.444263885-06:00" level=debug msg="fetch response received" digest="sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-disposition= response.header.content-length=2233 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:27:08 GMT" response.header.etag="\"0x8D950DD3FE78031\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:23 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=4e6f3d301f1a56a03cf990cc86a952ca9f812c7c response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:23 GMT" response.header.x-ms-copy-id=07945db6-426d-45cd-8734-4c1ace32645f response.header.x-ms-copy-progress=2233/2233 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:23 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=0a4480ba-601e-0082-2af9-010cc9000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kiad7000099-IAD, cache-bur-kbur8200032-BUR" response.status="200 OK" size=2233 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa"
time="2023-10-18T13:27:08.491057675-06:00" level=debug msg="fetch response received" digest="sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-length=301261 response.header.content-md5="R+Y0FZj1Wto7rT9gTULVcg==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:27:08 GMT" response.header.etag="\"0x8D8DECD29E1022C\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=dfe233a7ef9ba24763c5aa27349c1b138aedc4f5 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=76d654fa-5faf-44e7-be31-1fab2ada3763 response.header.x-ms-copy-progress=301261/301261 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=3c5894b7-d01e-006d-27f9-0190a1000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kcgs7200149-IAD, cache-bur-kbur8200032-BUR" response.status="200 OK" size=301261 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a"
time="2023-10-18T13:27:08.501442615-06:00" level=debug msg="fetch response received" digest="sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-length=1887472 response.header.content-md5="i9atZyjofrewlZ3ngsMOnA==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:27:08 GMT" response.header.etag="\"0x8D8DECD29DEACF9\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=4573c22679303667238b443fe9cd20bcaddc2723 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=13d27f76-4a1b-4ae1-8f7a-582a9ed3e3e3 response.header.x-ms-copy-progress=1887472/1887472 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=72fff6e6-a01e-00a2-54f9-019dfa000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kjyo7100107-IAD, cache-bur-kbur8200032-BUR" response.status="200 OK" size=1887472 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d"
time="2023-10-18T13:27:08.510817555-06:00" level=debug msg="fetch response received" digest="sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-disposition= response.header.content-length=20365161 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:27:08 GMT" response.header.etag="\"0x8D950DD41BFE9A7\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:27 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=eeaefb24726319da593d1729dab897c730036bcc response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:27 GMT" response.header.x-ms-copy-id=6c84899f-271a-4bda-89af-0d225eed6986 response.header.x-ms-copy-progress=20365161/20365161 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:27 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=8ec3d8ac-f01e-00eb-3af9-013df8000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kiad7000081-IAD, cache-bur-kbur8200032-BUR" response.status="200 OK" size=20365161 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a"
time="2023-10-18T13:27:08.512259040-06:00" level=debug msg="fetch response received" digest="sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-length=230 response.header.content-md5="OBaH1satt139FxF1ozF59w==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:27:08 GMT" response.header.etag="\"0x8D8DECD29DC1039\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=1e356c18da830866aaaea13a4e20dae8d9d9b8a2 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=a359c581-3a46-4cf8-a872-9e5325a9976a response.header.x-ms-copy-progress=230/230 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=9d5fd840-e01e-00ce-47f9-012bc1000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kcgs7200058-IAD, cache-bur-kbur8200032-BUR" response.status="200 OK" size=230 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd"
time="2023-10-18T13:27:08.611784211-06:00" level=debug msg="fetch response received" digest="sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-disposition= response.header.content-length=637463 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:27:08 GMT" response.header.etag="\"0x8D950DD4137CD4C\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:26 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=0b26936ae5a89e7b30333112d1c1fa723fb9f2d0 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:26 GMT" response.header.x-ms-copy-id=5112efd1-4292-4e39-bc9b-e40a2203877f response.header.x-ms-copy-progress=637463/637463 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:26 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=664a0704-e01e-0066-08f9-01b5b6000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kcgs7200059-IAD, cache-bur-kbur8200032-BUR" response.status="200 OK" size=637463 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d"
time="2023-10-18T13:27:08.626687094-06:00" level=debug msg="fetch response received" digest="sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=0 response.header.content-length=31506523 response.header.content-md5="Zv0aX6nCRCbDhK8ON7Qm6w==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:27:08 GMT" response.header.etag="\"0x8D8DECD29E65DF1\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, MISS" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=e9772561f8cd104e7330579385c242bb0830b665 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=5640d7c0-6252-4446-86dc-177b315dbf08 response.header.x-ms-copy-progress=31506523/31506523 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=7674c810-e01e-00b6-1af9-0181d0000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kjyo7100121-IAD, cache-bur-kbur8200032-BUR" response.status="200 OK" size=31506523 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e"
time="2023-10-18T13:27:13.442586908-06:00" level=debug msg="image unpacked" chainID="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253" config="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734"
time="2023-10-18T13:27:13.505369194-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:27:13.505598564-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:27:15.389383483-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:27:15.495060935-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:27:15.672592324-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:27:15.672916520-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:27:15.687223998-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:27:16.379180394-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=1
time="2023-10-18T13:27:16.381227702-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=1
time="2023-10-18T13:27:16.381411519-06:00" level=debug msg="First time offloading, need to create a snapshot first" fID=1
time="2023-10-18T13:27:16.382123041-06:00" level=debug msg="Creating instance snapshot" fID=1
time="2023-10-18T13:27:16.384159693-06:00" level=debug msg="Orchestrator received PauseVM" vmID=1-0
time="2023-10-18T13:27:16.386106579-06:00" level=debug msg="Initializing snapshot corresponding to revision and image" image="ghcr.io/ease-lab/helloworld:var_workload" revision=1
time="2023-10-18T13:27:16.386543358-06:00" level=debug msg="Orchestrator received CreateSnapshot" vmID=1-0
time="2023-10-18T13:27:17.263599361-06:00" level=debug msg="Creating patch file with disk state difference" patchFilePath=/fccd/snapshots/1/patch_file vmID=1-0
time="2023-10-18T13:27:17.522215070-06:00" level=debug msg="Serializing snapshot info" vmID=1-0
time="2023-10-18T13:27:17.523088131-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=1-0
time="2023-10-18T13:27:17.524872594-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=1
time="2023-10-18T13:27:17.526396768-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=1
time="2023-10-18T13:27:17.526630007-06:00" level=debug msg="Removing instance" fID=1 isSync=true
time="2023-10-18T13:27:17.526782557-06:00" level=debug msg="Orchestrator received StopVM" vmID=1-0
time="2023-10-18T13:27:22.540728053-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 1-0" vmID=1-0
time="2023-10-18T13:27:22.540998483-06:00" level=debug msg="Freeing a VM instance" vmID=1-0
time="2023-10-18T13:27:22.541117805-06:00" level=debug msg="Removing network config for function instance" funcID=1-0
time="2023-10-18T13:27:22.541244392-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=1-0
time="2023-10-18T13:27:22.541402512-06:00" level=debug msg="Stopped VM successfully" vmID=1-0
--- PASS: TestSendToFunctionSerial (16.65s)
=== RUN   TestSendToFunctionParallel
time="2023-10-18T13:27:22.577287527-06:00" level=debug msg="fIDint=2, err=<nil>, pinnedFuncNum=0"
time="2023-10-18T13:27:22.577619234-06:00" level=debug msg="Created function, pinned=true, shut down after 0 requests" fID=2 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:27:22.577921876-06:00" level=info msg="New function added" fID=2 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T13:27:22.578304559-06:00" level=debug msg="Function is inactive, starting the instance..." fID=2
time="2023-10-18T13:27:22.578551052-06:00" level=debug msg="Adding instance" fID=2
time="2023-10-18T13:27:22.578801843-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:27:22.579166430-06:00" level=debug msg="Allocating a VM instance" vmID=2-0
time="2023-10-18T13:27:22.579454818-06:00" level=debug msg="Creating network config for function instance" funcID=2-0
time="2023-10-18T13:27:22.579920392-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=2-0
time="2023-10-18T13:27:22.580363496-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=2-0
time="2023-10-18T13:27:22.586041859-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns70 tap=tap0
time="2023-10-18T13:27:22.646596946-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:27:22.646864503-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:27:24.408887722-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:27:24.507041956-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:27:24.677936427-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:27:24.678262404-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:27:24.692985589-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:27:25.391855169-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.391989604-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392523431-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392930188-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393266504-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393490392-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.394678567-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392231200-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392434282-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392414690-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392125432-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392558536-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392595105-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392972997-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393515805-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392321525-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392772123-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392757068-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392785588-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.392833594-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393183112-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393113362-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393171551-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393458311-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393102470-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393486419-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393977763-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.394409935-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393595020-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393658342-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393855132-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.394353860-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.393698512-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.394857282-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.394782461-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.394458923-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.394804713-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.394821916-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.394860208-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.394886881-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.394786017-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395169971-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395203944-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395150392-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.395268101-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.394938273-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395026611-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395244052-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395336301-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395491621-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395127203-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395591572-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395719511-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.395571612-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395405389-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395767918-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395727751-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395781794-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395820909-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395638298-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395818650-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395915223-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395939998-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.395991214-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.395941331-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396068141-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396135252-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396166889-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396175967-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396241697-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396304629-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396380943-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396485458-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396392534-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396637127-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396529694-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396540402-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396600578-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.396757181-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396877677-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396874686-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396804047-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397006779-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397210239-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397285505-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396964670-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397550134-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397149112-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397371683-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397556793-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397184545-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397901333-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397676397-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397153075-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397961985-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397760791-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.398077726-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.397885403-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.398240391-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.398324511-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.398301611-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.398445407-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.398337735-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.398405263-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.398218042-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.396932923-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:27:25.398960136-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.400116734-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.400526123-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.400860704-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.401662753-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.403604998-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.402797102-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.403615630-06:00" level=debug msg="First time offloading, need to create a snapshot first" fID=2
time="2023-10-18T13:27:25.404086332-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.404402723-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.404706477-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.404930923-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.411202273-06:00" level=debug msg="Creating instance snapshot" fID=2
time="2023-10-18T13:27:25.411893129-06:00" level=debug msg="Orchestrator received PauseVM" vmID=2-0
time="2023-10-18T13:27:25.411558927-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.412097590-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.412525644-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.412987424-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:25.413311914-06:00" level=debug msg="Initializing snapshot corresponding to revision and image" image="ghcr.io/ease-lab/helloworld:var_workload" revision=2
time="2023-10-18T13:27:25.413664786-06:00" level=debug msg="Orchestrator received CreateSnapshot" vmID=2-0
time="2023-10-18T13:27:26.290150839-06:00" level=debug msg="Creating patch file with disk state difference" patchFilePath=/fccd/snapshots/2/patch_file vmID=2-0
time="2023-10-18T13:27:26.545200923-06:00" level=debug msg="Serializing snapshot info" vmID=2-0
time="2023-10-18T13:27:26.545712434-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=2-0
time="2023-10-18T13:27:26.550635779-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.551222175-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.551485089-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.551939483-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.554744807-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.555211881-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.554938668-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.555644619-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.555133583-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.555476838-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.556456204-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.555530173-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.556556334-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.556146030-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.557035951-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.555991990-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.557333425-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.556257242-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.555762974-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.555840199-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.558002735-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.558653461-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.559793325-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.560536064-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.560945533-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.561291726-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.561910118-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.562246762-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.562782203-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.563120812-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.564194019-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.564779673-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.565458183-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.565844873-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.566512035-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.566700183-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.566777610-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.567576207-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.567926955-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.568310912-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.568973653-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.569715602-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.570447547-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.571153155-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.571969886-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.572536030-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.573272835-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.574085591-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.574719678-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.575531133-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.576042486-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.576768098-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.577525839-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.578180429-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.578817030-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.579525551-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.580226579-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.581145600-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.581756731-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.582356855-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.582864028-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.583581992-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.584027621-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.584663324-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.584942793-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.585582026-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.585972128-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.586386544-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.587125649-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.587716227-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.588376929-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.589078116-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.589689029-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.590270120-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.590932490-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.591753955-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.592326809-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.593151482-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.593708789-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:27:26.594012251-06:00" level=debug msg="Removing instance" fID=2 isSync=true
time="2023-10-18T13:27:26.594125555-06:00" level=debug msg="Orchestrator received StopVM" vmID=2-0
time="2023-10-18T13:27:31.608775370-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 2-0" vmID=2-0
time="2023-10-18T13:27:31.609422876-06:00" level=debug msg="Freeing a VM instance" vmID=2-0
time="2023-10-18T13:27:31.609580689-06:00" level=debug msg="Removing network config for function instance" funcID=2-0
time="2023-10-18T13:27:31.609744720-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=2-0
time="2023-10-18T13:27:31.609938032-06:00" level=debug msg="Stopped VM successfully" vmID=2-0
--- PASS: TestSendToFunctionParallel (9.07s)
=== RUN   TestStartSendStopTwice
time="2023-10-18T13:27:31.643272497-06:00" level=debug msg="fIDint=3, err=<nil>, pinnedFuncNum=2"
time="2023-10-18T13:27:31.643395190-06:00" level=debug msg="Created function, pinned=true, shut down after 1 requests" fID=3 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:27:31.643551900-06:00" level=info msg="New function added" fID=3 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T13:27:31.643693324-06:00" level=debug msg="Function is inactive, starting the instance..." fID=3
time="2023-10-18T13:27:31.643766917-06:00" level=debug msg="Adding instance" fID=3
time="2023-10-18T13:27:31.643866372-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:27:31.643978624-06:00" level=debug msg="Allocating a VM instance" vmID=3-0
time="2023-10-18T13:27:31.644094661-06:00" level=debug msg="Creating network config for function instance" funcID=3-0
time="2023-10-18T13:27:31.644221430-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=3-0
time="2023-10-18T13:27:31.644349335-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=3-0
time="2023-10-18T13:27:31.649904521-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns71 tap=tap0
time="2023-10-18T13:27:31.709605302-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:27:31.709879036-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:27:33.473305612-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:27:33.566839500-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:27:33.752207632-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:27:33.752628103-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:27:33.768156012-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:27:34.486174593-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=3
time="2023-10-18T13:27:34.488266182-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=3
time="2023-10-18T13:27:34.488456058-06:00" level=debug msg="First time offloading, need to create a snapshot first" fID=3
time="2023-10-18T13:27:34.488621990-06:00" level=debug msg="Creating instance snapshot" fID=3
time="2023-10-18T13:27:34.488729269-06:00" level=debug msg="Orchestrator received PauseVM" vmID=3-0
time="2023-10-18T13:27:34.490644136-06:00" level=debug msg="Initializing snapshot corresponding to revision and image" image="ghcr.io/ease-lab/helloworld:var_workload" revision=3
time="2023-10-18T13:27:34.490976822-06:00" level=debug msg="Orchestrator received CreateSnapshot" vmID=3-0
time="2023-10-18T13:27:35.430778737-06:00" level=debug msg="Creating patch file with disk state difference" patchFilePath=/fccd/snapshots/3/patch_file vmID=3-0
time="2023-10-18T13:27:35.691963351-06:00" level=debug msg="Serializing snapshot info" vmID=3-0
time="2023-10-18T13:27:35.692289128-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=3-0
time="2023-10-18T13:27:35.694105046-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=3
time="2023-10-18T13:27:35.696125955-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=3
time="2023-10-18T13:27:35.696414558-06:00" level=debug msg="Removing instance" fID=3 isSync=true
time="2023-10-18T13:27:35.696629538-06:00" level=debug msg="Orchestrator received StopVM" vmID=3-0
time="2023-10-18T13:27:40.710472076-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 3-0" vmID=3-0
time="2023-10-18T13:27:40.710740254-06:00" level=debug msg="Freeing a VM instance" vmID=3-0
time="2023-10-18T13:27:40.710863710-06:00" level=debug msg="Removing network config for function instance" funcID=3-0
time="2023-10-18T13:27:40.711005763-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=3-0
time="2023-10-18T13:27:40.711158594-06:00" level=debug msg="Stopped VM successfully" vmID=3-0
time="2023-10-18T13:27:40.711355679-06:00" level=debug msg="Function is inactive, starting the instance..." fID=3
time="2023-10-18T13:27:40.711490237-06:00" level=debug msg="Adding instance" fID=3
time="2023-10-18T13:27:40.711629146-06:00" level=debug msg="Loading instance" fID=3
time="2023-10-18T13:27:40.711789829-06:00" level=debug msg="Orchestrator received LoadSnapshot" vmID=3-1
time="2023-10-18T13:27:40.711921527-06:00" level=debug msg="Allocating a VM instance" vmID=3-1
time="2023-10-18T13:27:40.712069928-06:00" level=debug msg="Creating network config for function instance" funcID=3-1
time="2023-10-18T13:27:40.712309506-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=3-1
time="2023-10-18T13:27:40.712580477-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=3-1
time="2023-10-18T13:27:40.718588982-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns72 tap=tap0
time="2023-10-18T13:27:40.911328532-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:27:40.911551099-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:27:41.053896430-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=3-1
time="2023-10-18T13:27:41.060659890-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=3
time="2023-10-18T13:27:41.068822022-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=3
time="2023-10-18T13:27:41.069226435-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=3
time="2023-10-18T13:27:41.070961078-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=3
time="2023-10-18T13:27:41.071238802-06:00" level=debug msg="Removing instance" fID=3 isSync=true
time="2023-10-18T13:27:41.071367638-06:00" level=debug msg="Orchestrator received StopVM" vmID=3-1
time="2023-10-18T13:27:51.085716961-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 3-1" vmID=3-1
time="2023-10-18T13:27:51.086012236-06:00" level=debug msg="Freeing a VM instance" vmID=3-1
time="2023-10-18T13:27:51.086159688-06:00" level=debug msg="Removing network config for function instance" funcID=3-1
time="2023-10-18T13:27:51.086285699-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=3-1
time="2023-10-18T13:27:51.089551202-06:00" level=debug msg="Stopped VM successfully" vmID=3-1
--- PASS: TestStartSendStopTwice (19.48s)
=== RUN   TestStatsNotNumericFunction
time="2023-10-18T13:27:51.124889614-06:00" level=debug msg="fIDint=0, err=strconv.Atoi: parsing \"not-cld\": invalid syntax, pinnedFuncNum=2"
time="2023-10-18T13:27:51.125030661-06:00" level=debug msg="Created function, pinned=true, shut down after 1 requests" fID=not-cld imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:27:51.125206327-06:00" level=info msg="New function added" fID=not-cld image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T13:27:51.125348699-06:00" level=debug msg="Function is inactive, starting the instance..." fID=not-cld
time="2023-10-18T13:27:51.125435228-06:00" level=debug msg="Adding instance" fID=not-cld
time="2023-10-18T13:27:51.125510305-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:27:51.125630995-06:00" level=debug msg="Allocating a VM instance" vmID=not-cld-0
time="2023-10-18T13:27:51.125729861-06:00" level=debug msg="Creating network config for function instance" funcID=not-cld-0
time="2023-10-18T13:27:51.125875014-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=not-cld-0
time="2023-10-18T13:27:51.126021713-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=not-cld-0
time="2023-10-18T13:27:51.131827690-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns73 tap=tap0
time="2023-10-18T13:27:51.189012127-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:27:51.189230192-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:27:52.974474353-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:27:53.096165953-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:27:53.269009322-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:27:53.269345563-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:27:53.283496371-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=not-cld-0
time="2023-10-18T13:27:53.975308056-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=not-cld
time="2023-10-18T13:27:53.977273679-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=not-cld
time="2023-10-18T13:27:53.977471392-06:00" level=debug msg="First time offloading, need to create a snapshot first" fID=not-cld
time="2023-10-18T13:27:53.977642823-06:00" level=debug msg="Creating instance snapshot" fID=not-cld
time="2023-10-18T13:27:53.977770118-06:00" level=debug msg="Orchestrator received PauseVM" vmID=not-cld-0
time="2023-10-18T13:27:53.979646752-06:00" level=debug msg="Initializing snapshot corresponding to revision and image" image="ghcr.io/ease-lab/helloworld:var_workload" revision=not-cld
time="2023-10-18T13:27:53.979953123-06:00" level=debug msg="Orchestrator received CreateSnapshot" vmID=not-cld-0
time="2023-10-18T13:27:54.873996056-06:00" level=debug msg="Creating patch file with disk state difference" patchFilePath=/fccd/snapshots/not-cld/patch_file vmID=not-cld-0
time="2023-10-18T13:27:55.116834811-06:00" level=debug msg="Serializing snapshot info" vmID=not-cld-0
time="2023-10-18T13:27:55.117359920-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=not-cld-0
time="2023-10-18T13:27:55.119449894-06:00" level=debug msg="Removing instance" fID=not-cld isSync=true
time="2023-10-18T13:27:55.119609695-06:00" level=debug msg="Orchestrator received StopVM" vmID=not-cld-0
time="2023-10-18T13:28:00.133771445-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM not-cld-0" vmID=not-cld-0
time="2023-10-18T13:28:00.134048842-06:00" level=debug msg="Freeing a VM instance" vmID=not-cld-0
time="2023-10-18T13:28:00.134184523-06:00" level=debug msg="Removing network config for function instance" funcID=not-cld-0
time="2023-10-18T13:28:00.134324758-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=not-cld-0
time="2023-10-18T13:28:00.134548468-06:00" level=debug msg="Stopped VM successfully" vmID=not-cld-0
--- PASS: TestStatsNotNumericFunction (9.04s)
=== RUN   TestStatsNotColdFunction
time="2023-10-18T13:28:00.168209958-06:00" level=debug msg="fIDint=4, err=<nil>, pinnedFuncNum=4"
time="2023-10-18T13:28:00.168337690-06:00" level=debug msg="Created function, pinned=true, shut down after 1 requests" fID=4 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:28:00.168497721-06:00" level=info msg="New function added" fID=4 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T13:28:00.168624743-06:00" level=debug msg="Function is inactive, starting the instance..." fID=4
time="2023-10-18T13:28:00.168700468-06:00" level=debug msg="Adding instance" fID=4
time="2023-10-18T13:28:00.168797591-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:28:00.168904832-06:00" level=debug msg="Allocating a VM instance" vmID=4-0
time="2023-10-18T13:28:00.169007011-06:00" level=debug msg="Creating network config for function instance" funcID=4-0
time="2023-10-18T13:28:00.169150047-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=4-0
time="2023-10-18T13:28:00.169299466-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=4-0
time="2023-10-18T13:28:00.174842815-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns74 tap=tap0
time="2023-10-18T13:28:00.233893653-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:28:00.234186028-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:28:02.003282722-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:28:02.120225502-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:28:02.292157701-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:28:02.292500233-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:28:02.307233658-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=4-0
time="2023-10-18T13:28:02.999204250-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=4
time="2023-10-18T13:28:03.001214059-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=4
time="2023-10-18T13:28:03.001473654-06:00" level=debug msg="First time offloading, need to create a snapshot first" fID=4
time="2023-10-18T13:28:03.001651741-06:00" level=debug msg="Creating instance snapshot" fID=4
time="2023-10-18T13:28:03.001788772-06:00" level=debug msg="Orchestrator received PauseVM" vmID=4-0
time="2023-10-18T13:28:03.003762356-06:00" level=debug msg="Initializing snapshot corresponding to revision and image" image="ghcr.io/ease-lab/helloworld:var_workload" revision=4
time="2023-10-18T13:28:03.004140452-06:00" level=debug msg="Orchestrator received CreateSnapshot" vmID=4-0
time="2023-10-18T13:28:03.943601753-06:00" level=debug msg="Creating patch file with disk state difference" patchFilePath=/fccd/snapshots/4/patch_file vmID=4-0
time="2023-10-18T13:28:04.207155405-06:00" level=debug msg="Serializing snapshot info" vmID=4-0
time="2023-10-18T13:28:04.207613246-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=4-0
time="2023-10-18T13:28:04.209571251-06:00" level=debug msg="Removing instance" fID=4 isSync=true
time="2023-10-18T13:28:04.209789826-06:00" level=debug msg="Orchestrator received StopVM" vmID=4-0
time="2023-10-18T13:28:09.223986638-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 4-0" vmID=4-0
time="2023-10-18T13:28:09.224281572-06:00" level=debug msg="Freeing a VM instance" vmID=4-0
time="2023-10-18T13:28:09.224446244-06:00" level=debug msg="Removing network config for function instance" funcID=4-0
time="2023-10-18T13:28:09.224600756-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=4-0
time="2023-10-18T13:28:09.224778529-06:00" level=debug msg="Stopped VM successfully" vmID=4-0
--- PASS: TestStatsNotColdFunction (9.09s)
=== RUN   TestSaveMemorySerial
time="2023-10-18T13:28:09.261065091-06:00" level=debug msg="fIDint=5, err=<nil>, pinnedFuncNum=2"
time="2023-10-18T13:28:09.261194884-06:00" level=debug msg="Created function, pinned=false, shut down after 40 requests" fID=5 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:28:09.261342637-06:00" level=info msg="New function added" fID=5 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=false servedTh=40
time="2023-10-18T13:28:09.261461610-06:00" level=debug msg="Function is inactive, starting the instance..." fID=5
time="2023-10-18T13:28:09.261547658-06:00" level=debug msg="Adding instance" fID=5
time="2023-10-18T13:28:09.261626117-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:28:09.261735649-06:00" level=debug msg="Allocating a VM instance" vmID=5-0
time="2023-10-18T13:28:09.261824863-06:00" level=debug msg="Creating network config for function instance" funcID=5-0
time="2023-10-18T13:28:09.261949826-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=5-0
time="2023-10-18T13:28:09.262097809-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=5-0
time="2023-10-18T13:28:09.267609599-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns75 tap=tap0
time="2023-10-18T13:28:09.321315131-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:28:09.321521976-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:28:11.087949985-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:28:11.182070188-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:28:11.352848350-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:28:11.353240097-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:28:11.365323096-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=5-0
time="2023-10-18T13:28:12.078459311-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:12.080380873-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:12.080567076-06:00" level=debug msg="First time offloading, need to create a snapshot first" fID=5
time="2023-10-18T13:28:12.080742948-06:00" level=debug msg="Creating instance snapshot" fID=5
time="2023-10-18T13:28:12.080866233-06:00" level=debug msg="Orchestrator received PauseVM" vmID=5-0
time="2023-10-18T13:28:12.082679994-06:00" level=debug msg="Initializing snapshot corresponding to revision and image" image="ghcr.io/ease-lab/helloworld:var_workload" revision=5
time="2023-10-18T13:28:12.082986531-06:00" level=debug msg="Orchestrator received CreateSnapshot" vmID=5-0
time="2023-10-18T13:28:12.959886549-06:00" level=debug msg="Creating patch file with disk state difference" patchFilePath=/fccd/snapshots/5/patch_file vmID=5-0
time="2023-10-18T13:28:13.208885765-06:00" level=debug msg="Serializing snapshot info" vmID=5-0
time="2023-10-18T13:28:13.209354649-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=5-0
time="2023-10-18T13:28:13.211365401-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.213382614-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.213686449-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.215280939-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.215599906-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.217303408-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.217600768-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.219154051-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.219475581-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.221008370-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.221198139-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.222551576-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.222884108-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.224302547-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.224571825-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.225989216-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.226305486-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.227707573-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.228008003-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.229658095-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.229903783-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.231808421-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.232144084-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.233919521-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.234189395-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.235944897-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.236305842-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.238118480-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.238482148-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.240294275-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.240692901-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.242339851-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.242649520-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.244125581-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.244344435-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.245854972-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.246160327-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.247560773-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.247951741-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.249507204-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.249861410-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.251156763-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.251490701-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.252856336-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.253164193-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.254672695-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.254969420-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.256489476-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.256664787-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.257805946-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.258097186-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.259239225-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.259505159-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.260828616-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.261138004-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.262662540-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.262971952-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.264217040-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.264421175-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.265803692-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.265988609-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.267156983-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.267326675-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.268555201-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.268922456-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.270241913-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.270527815-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.271709476-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.271978384-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.273176341-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.273405881-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.274773955-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.275062598-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.276273451-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.276420283-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.277669575-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.277926204-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:13.279089715-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:13.279241306-06:00" level=debug msg="Function has to shut down its instance, served 40 requests" fID=5
time="2023-10-18T13:28:13.280212584-06:00" level=debug msg="Removing instance" fID=5 isSync=false
time="2023-10-18T13:28:13.282135811-06:00" level=debug msg="Removing instance (async)" fID=5
time="2023-10-18T13:28:13.282555360-06:00" level=debug msg="Function is inactive, starting the instance..." fID=5
time="2023-10-18T13:28:13.282587906-06:00" level=debug msg="Orchestrator received StopVM" vmID=5-0
time="2023-10-18T13:28:13.282706354-06:00" level=debug msg="Adding instance" fID=5
time="2023-10-18T13:28:13.283118475-06:00" level=debug msg="Loading instance" fID=5
time="2023-10-18T13:28:13.283256452-06:00" level=debug msg="Orchestrator received LoadSnapshot" vmID=5-1
time="2023-10-18T13:28:13.283389273-06:00" level=debug msg="Allocating a VM instance" vmID=5-1
time="2023-10-18T13:28:13.283556320-06:00" level=debug msg="Creating network config for function instance" funcID=5-1
time="2023-10-18T13:28:13.283803616-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=5-1
time="2023-10-18T13:28:13.284094955-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.76 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns75 NamespacePath=/var/run/netns/uvmns75 Veth0CIDR=172.17.1.46/30 Veth0Name=veth75-0 Veth1CIDR=172.17.1.45/30 Veth1Name=veth75-1 funcID=5-1
time="2023-10-18T13:28:13.287519495-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns76 tap=tap0
time="2023-10-18T13:28:13.547544340-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:28:13.547800062-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:28:18.298498911-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-0" vmID=5-0
time="2023-10-18T13:28:18.298582540-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=5-1
time="2023-10-18T13:28:18.298805091-06:00" level=debug msg="Freeing a VM instance" vmID=5-0
time="2023-10-18T13:28:18.299153193-06:00" level=debug msg="Removing network config for function instance" funcID=5-0
time="2023-10-18T13:28:18.299343198-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=5-0
time="2023-10-18T13:28:18.299542822-06:00" level=debug msg="Stopped VM successfully" vmID=5-0
time="2023-10-18T13:28:18.306911920-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.315769313-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.316141230-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.317937924-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.318261599-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.319937827-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.320284048-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.321973350-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.322326193-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.324109739-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.324446265-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.326114423-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.326496474-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.328111317-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.328403396-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.329929073-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.330247101-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.331783831-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.332043043-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.333542890-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.333836692-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.335560194-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.335833905-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.337308295-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.337578075-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.338908825-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.339242118-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.340923219-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.341185523-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.342787677-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.343088208-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.344625147-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.344843869-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.346142075-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.346356653-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.347568319-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.347793727-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.349148134-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.349379994-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.350746264-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.350950409-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.352315303-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.352611433-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.354117179-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.354432564-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.355848294-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.356121642-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.357472986-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.357799508-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.359477851-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.359802832-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.361459120-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.361773603-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.363310228-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.363619152-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.365097737-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.365377960-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.367005481-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.367378404-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.369150226-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.369547035-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.371180042-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.371527355-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.373318446-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.373612785-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.375579477-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.375911106-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.377583182-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.377900982-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.379407577-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.379748278-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.381303443-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.381642627-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.383133501-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.383487136-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.385070320-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.385439554-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.386938931-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.387236171-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.388645945-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.388815156-06:00" level=debug msg="Function has to shut down its instance, served 40 requests" fID=5
time="2023-10-18T13:28:18.388977704-06:00" level=debug msg="Removing instance" fID=5 isSync=false
time="2023-10-18T13:28:18.389116647-06:00" level=debug msg="Removing instance (async)" fID=5
time="2023-10-18T13:28:18.389375961-06:00" level=debug msg="Orchestrator received StopVM" vmID=5-1
time="2023-10-18T13:28:18.389396798-06:00" level=debug msg="Function is inactive, starting the instance..." fID=5
time="2023-10-18T13:28:18.389808826-06:00" level=debug msg="Adding instance" fID=5
time="2023-10-18T13:28:18.389953953-06:00" level=debug msg="Loading instance" fID=5
time="2023-10-18T13:28:18.390080655-06:00" level=debug msg="Orchestrator received LoadSnapshot" vmID=5-2
time="2023-10-18T13:28:18.390206499-06:00" level=debug msg="Allocating a VM instance" vmID=5-2
time="2023-10-18T13:28:18.390367100-06:00" level=debug msg="Creating network config for function instance" funcID=5-2
time="2023-10-18T13:28:18.390571787-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=5-2
time="2023-10-18T13:28:18.390830448-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=5-2
time="2023-10-18T13:28:18.395595209-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns77 tap=tap0
time="2023-10-18T13:28:18.617370084-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:28:18.617626477-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:28:18.759337359-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=5-2
time="2023-10-18T13:28:18.765366393-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.770526816-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.770964658-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.772697913-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.772922853-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.774362982-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.774731508-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.776436293-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.776804876-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.778632620-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.778947647-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.780693779-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.781027243-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.782597430-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.782903346-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.784717494-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.785037279-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.786710808-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.787038576-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.788787481-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.789097604-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.790676297-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.790996632-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.792543853-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.792875500-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.794497858-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.794784443-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.796381903-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.796695559-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.798400673-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.798751171-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.800949227-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.801248181-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.803061340-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.803402218-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.805035626-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.805324937-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.807000253-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.807309383-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=5
time="2023-10-18T13:28:18.808994472-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=5
time="2023-10-18T13:28:18.809258313-06:00" level=debug msg="Removing instance" fID=5 isSync=true
time="2023-10-18T13:28:18.809411823-06:00" level=debug msg="Orchestrator received StopVM" vmID=5-2
time="2023-10-18T13:28:28.403751197-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-1" vmID=5-1
time="2023-10-18T13:28:28.404075282-06:00" level=debug msg="Freeing a VM instance" vmID=5-1
time="2023-10-18T13:28:28.404257876-06:00" level=debug msg="Removing network config for function instance" funcID=5-1
time="2023-10-18T13:28:28.404462526-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=5-1
time="2023-10-18T13:28:28.407578535-06:00" level=debug msg="Stopped VM successfully" vmID=5-1
time="2023-10-18T13:28:28.825486812-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 5-2" vmID=5-2
time="2023-10-18T13:28:28.825753320-06:00" level=debug msg="Freeing a VM instance" vmID=5-2
time="2023-10-18T13:28:28.825912519-06:00" level=debug msg="Removing network config for function instance" funcID=5-2
time="2023-10-18T13:28:28.826063392-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=5-2
time="2023-10-18T13:28:28.829101489-06:00" level=debug msg="Stopped VM successfully" vmID=5-2
--- PASS: TestSaveMemorySerial (19.60s)
=== RUN   TestSaveMemoryParallel
time="2023-10-18T13:28:28.863133005-06:00" level=debug msg="fIDint=6, err=<nil>, pinnedFuncNum=2"
time="2023-10-18T13:28:28.863475771-06:00" level=debug msg="Created function, pinned=false, shut down after 40 requests" fID=6 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:28:28.863810558-06:00" level=info msg="New function added" fID=6 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=false servedTh=40
time="2023-10-18T13:28:28.864140711-06:00" level=debug msg="Function is inactive, starting the instance..." fID=6
time="2023-10-18T13:28:28.864450640-06:00" level=debug msg="Adding instance" fID=6
time="2023-10-18T13:28:28.864635182-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:28:28.864905255-06:00" level=debug msg="Allocating a VM instance" vmID=6-0
time="2023-10-18T13:28:28.865545830-06:00" level=debug msg="Creating network config for function instance" funcID=6-0
time="2023-10-18T13:28:28.865821724-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=6-0
time="2023-10-18T13:28:28.866098303-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=6-0
time="2023-10-18T13:28:28.871418427-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns78 tap=tap0
time="2023-10-18T13:28:28.931088191-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:28:28.931269582-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:28:30.817714276-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:28:30.903793702-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:28:31.083209418-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:28:31.083541108-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:28:31.097704342-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=6-0
time="2023-10-18T13:28:31.792476932-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.792709149-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.792685467-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793205425-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793346291-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793575310-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793866485-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794062164-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794381187-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794616097-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.792847443-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793887077-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.795113612-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794143760-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793008255-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.792966187-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793117194-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.795030016-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.795047965-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794857399-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.795157846-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793608545-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793308800-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793152680-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793440762-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793676938-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793472125-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794018779-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.793705724-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794069132-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794232496-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794306466-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794536280-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794468764-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794552911-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794751665-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794673953-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794918505-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.795055878-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.794967196-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:31.796520329-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.802649567-06:00" level=debug msg="First time offloading, need to create a snapshot first" fID=6
time="2023-10-18T13:28:31.802766338-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.802360471-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.797843999-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.797489158-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.798138192-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.802791655-06:00" level=debug msg="Creating instance snapshot" fID=6
time="2023-10-18T13:28:31.800244137-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.803408054-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.804356525-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.803902898-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.803965265-06:00" level=debug msg="Orchestrator received PauseVM" vmID=6-0
time="2023-10-18T13:28:31.804693646-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.805418032-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.805978052-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.806184968-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:31.806334320-06:00" level=debug msg="Initializing snapshot corresponding to revision and image" image="ghcr.io/ease-lab/helloworld:var_workload" revision=6
time="2023-10-18T13:28:31.806782132-06:00" level=debug msg="Orchestrator received CreateSnapshot" vmID=6-0
time="2023-10-18T13:28:32.746750625-06:00" level=debug msg="Creating patch file with disk state difference" patchFilePath=/fccd/snapshots/6/patch_file vmID=6-0
time="2023-10-18T13:28:32.989876432-06:00" level=debug msg="Serializing snapshot info" vmID=6-0
time="2023-10-18T13:28:32.990374595-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=6-0
time="2023-10-18T13:28:32.993600014-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:32.994117546-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:32.994735019-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:32.995383705-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:32.996063052-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:32.996754771-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:32.997248182-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:32.997453064-06:00" level=debug msg="Function has to shut down its instance, served 40 requests" fID=6
time="2023-10-18T13:28:32.997952319-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:32.998304306-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:32.998916887-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:32.999423736-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.000205123-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.000734520-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.001526580-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.002386458-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.003328917-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.004130923-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.004725512-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.005370069-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.005878059-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.006425897-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.006958564-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.007583888-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.008191962-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.008715315-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.009116905-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:33.009377593-06:00" level=debug msg="Removing instance" fID=6 isSync=false
time="2023-10-18T13:28:33.009535914-06:00" level=debug msg="Removing instance (async)" fID=6
time="2023-10-18T13:28:33.009828399-06:00" level=debug msg="Function is inactive, starting the instance..." fID=6
time="2023-10-18T13:28:33.010094831-06:00" level=debug msg="Adding instance" fID=6
time="2023-10-18T13:28:33.010346615-06:00" level=debug msg="Loading instance" fID=6
time="2023-10-18T13:28:33.010744750-06:00" level=debug msg="Orchestrator received LoadSnapshot" vmID=6-1
time="2023-10-18T13:28:33.009847095-06:00" level=debug msg="Orchestrator received StopVM" vmID=6-0
time="2023-10-18T13:28:33.011060558-06:00" level=debug msg="Allocating a VM instance" vmID=6-1
time="2023-10-18T13:28:33.011431639-06:00" level=debug msg="Creating network config for function instance" funcID=6-1
time="2023-10-18T13:28:33.011639644-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=6-1
time="2023-10-18T13:28:33.011865412-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.79 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns78 NamespacePath=/var/run/netns/uvmns78 Veth0CIDR=172.17.1.58/30 Veth0Name=veth78-0 Veth1CIDR=172.17.1.57/30 Veth1Name=veth78-1 funcID=6-1
time="2023-10-18T13:28:33.017298204-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns79 tap=tap0
time="2023-10-18T13:28:33.345107529-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:28:33.345331300-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:28:38.027070697-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=6-1
time="2023-10-18T13:28:38.027119819-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-0" vmID=6-0
time="2023-10-18T13:28:38.027582756-06:00" level=debug msg="Freeing a VM instance" vmID=6-0
time="2023-10-18T13:28:38.027776376-06:00" level=debug msg="Removing network config for function instance" funcID=6-0
time="2023-10-18T13:28:38.027951179-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=6-0
time="2023-10-18T13:28:38.028157058-06:00" level=debug msg="Stopped VM successfully" vmID=6-0
time="2023-10-18T13:28:38.033932280-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034202791-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034500357-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034857333-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035099844-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035530082-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035599842-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.036252644-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034039004-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035362362-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034242514-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035836118-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034168139-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034657616-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035153784-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035921371-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.036289029-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034136108-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034402786-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034537882-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034416027-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034741098-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034719086-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034748545-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034986385-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035156911-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.034968564-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035380872-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035452179-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035092714-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035746505-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035410001-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035821745-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.035642522-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.036056862-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.036198039-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.036038265-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.036340703-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.036442786-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.036562312-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.038050320-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.040452880-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.038988630-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.042423038-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.044327815-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.043411837-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.041368857-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.045761101-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.045158793-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.046205059-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.046826192-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.047336250-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.048657767-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.049508937-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.050127577-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.050756308-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.051403552-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.051942810-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.052640274-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.053404206-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.054000085-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.054737464-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.055296473-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.056019551-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.056595612-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.057202258-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.057887926-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.058306772-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.058519730-06:00" level=debug msg="Function has to shut down its instance, served 40 requests" fID=6
time="2023-10-18T13:28:38.058875360-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.059329853-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.059895905-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.060668501-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.061047582-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.061550434-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.061992542-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.062522788-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.063056889-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.063538935-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.064038695-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.064613574-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.064907790-06:00" level=debug msg="Removing instance" fID=6 isSync=false
time="2023-10-18T13:28:38.065119322-06:00" level=debug msg="Removing instance (async)" fID=6
time="2023-10-18T13:28:38.065386802-06:00" level=debug msg="Orchestrator received StopVM" vmID=6-1
time="2023-10-18T13:28:38.065521287-06:00" level=debug msg="Function is inactive, starting the instance..." fID=6
time="2023-10-18T13:28:38.066044557-06:00" level=debug msg="Adding instance" fID=6
time="2023-10-18T13:28:38.066193270-06:00" level=debug msg="Loading instance" fID=6
time="2023-10-18T13:28:38.066382473-06:00" level=debug msg="Orchestrator received LoadSnapshot" vmID=6-2
time="2023-10-18T13:28:38.066543375-06:00" level=debug msg="Allocating a VM instance" vmID=6-2
time="2023-10-18T13:28:38.066729914-06:00" level=debug msg="Creating network config for function instance" funcID=6-2
time="2023-10-18T13:28:38.066952448-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=6-2
time="2023-10-18T13:28:38.067366395-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=6-2
time="2023-10-18T13:28:38.073576315-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns80 tap=tap0
time="2023-10-18T13:28:38.288781989-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:28:38.289026585-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:28:38.430857459-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=6-2
time="2023-10-18T13:28:38.435775782-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.435901596-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436338532-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436044269-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436721336-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436559223-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436254680-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436482590-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436760277-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436753775-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.437029049-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436074276-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436623402-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436762773-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436802572-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436281417-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436628849-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436687778-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436567516-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.436653520-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=6
time="2023-10-18T13:28:38.439963888-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.441172688-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.442030891-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.442772895-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.443820345-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.444543301-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.445216327-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.445589758-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.445979704-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.446430653-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.446795711-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.447300978-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.448010830-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.448459049-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.448926614-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.449443804-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.449903221-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.450304659-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.450828872-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.451290536-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=6
time="2023-10-18T13:28:38.451702154-06:00" level=debug msg="Removing instance" fID=6 isSync=true
time="2023-10-18T13:28:38.451892357-06:00" level=debug msg="Orchestrator received StopVM" vmID=6-2
time="2023-10-18T13:28:48.079615352-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-1" vmID=6-1
time="2023-10-18T13:28:48.080041246-06:00" level=debug msg="Freeing a VM instance" vmID=6-1
time="2023-10-18T13:28:48.080209059-06:00" level=debug msg="Removing network config for function instance" funcID=6-1
time="2023-10-18T13:28:48.080404497-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=6-1
time="2023-10-18T13:28:48.085852450-06:00" level=debug msg="Stopped VM successfully" vmID=6-1
time="2023-10-18T13:28:48.460668870-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 6-2" vmID=6-2
time="2023-10-18T13:28:48.461071479-06:00" level=debug msg="Freeing a VM instance" vmID=6-2
time="2023-10-18T13:28:48.461260312-06:00" level=debug msg="Removing network config for function instance" funcID=6-2
time="2023-10-18T13:28:48.461442959-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=6-2
time="2023-10-18T13:28:48.464509984-06:00" level=debug msg="Stopped VM successfully" vmID=6-2
--- PASS: TestSaveMemoryParallel (19.64s)
=== RUN   TestDirectStartStopVM
time="2023-10-18T13:28:48.496177540-06:00" level=debug msg="fIDint=7, err=<nil>, pinnedFuncNum=0"
time="2023-10-18T13:28:48.496335330-06:00" level=debug msg="Created function, pinned=true, shut down after 0 requests" fID=7 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:28:48.496504675-06:00" level=info msg="New function added" fID=7 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T13:28:48.496606033-06:00" level=debug msg="Function is inactive, starting the instance..." fID=7
time="2023-10-18T13:28:48.496682032-06:00" level=debug msg="Adding instance" fID=7
time="2023-10-18T13:28:48.496768058-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:28:48.496870453-06:00" level=debug msg="Allocating a VM instance" vmID=7-0
time="2023-10-18T13:28:48.496954385-06:00" level=debug msg="Creating network config for function instance" funcID=7-0
time="2023-10-18T13:28:48.497080542-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=7-0
time="2023-10-18T13:28:48.497239389-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.67 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns66 NamespacePath=/var/run/netns/uvmns66 Veth0CIDR=172.17.1.10/30 Veth0Name=veth66-0 Veth1CIDR=172.17.1.9/30 Veth1Name=veth66-1 funcID=7-0
time="2023-10-18T13:28:48.502957353-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns81 tap=tap0
time="2023-10-18T13:28:48.568341922-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:28:48.568616211-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:28:50.551621628-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:28:50.651177180-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:28:50.831379676-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:28:50.831743652-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:28:50.846513391-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=7-0
time="2023-10-18T13:28:51.937548488-06:00" level=debug msg="Reconnecting after an error"
time="2023-10-18T13:28:52.011429294-06:00" level=debug msg="Reconnecting after an error"
time="2023-10-18T13:28:52.851647250-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=7
time="2023-10-18T13:28:52.854658837-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=7
time="2023-10-18T13:28:52.854859387-06:00" level=debug msg="First time offloading, need to create a snapshot first" fID=7
time="2023-10-18T13:28:52.855005566-06:00" level=debug msg="Creating instance snapshot" fID=7
time="2023-10-18T13:28:52.855145998-06:00" level=debug msg="Orchestrator received PauseVM" vmID=7-0
time="2023-10-18T13:28:52.857116020-06:00" level=debug msg="Initializing snapshot corresponding to revision and image" image="ghcr.io/ease-lab/helloworld:var_workload" revision=7
time="2023-10-18T13:28:52.857430015-06:00" level=debug msg="Orchestrator received CreateSnapshot" vmID=7-0
time="2023-10-18T13:28:53.738750010-06:00" level=debug msg="Creating patch file with disk state difference" patchFilePath=/fccd/snapshots/7/patch_file vmID=7-0
time="2023-10-18T13:28:53.985069569-06:00" level=debug msg="Serializing snapshot info" vmID=7-0
time="2023-10-18T13:28:53.985388766-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=7-0
time="2023-10-18T13:28:53.986970041-06:00" level=debug msg="Removing instance" fID=7 isSync=true
time="2023-10-18T13:28:53.987084918-06:00" level=debug msg="Orchestrator received StopVM" vmID=7-0
time="2023-10-18T13:28:57.520110749-06:00" level=debug msg="Reconnecting after an error"
time="2023-10-18T13:28:59.000822229-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 7-0" vmID=7-0
time="2023-10-18T13:28:59.001108352-06:00" level=debug msg="Freeing a VM instance" vmID=7-0
time="2023-10-18T13:28:59.001256639-06:00" level=debug msg="Removing network config for function instance" funcID=7-0
time="2023-10-18T13:28:59.001402113-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=7-0
time="2023-10-18T13:28:59.001574012-06:00" level=debug msg="Stopped VM successfully" vmID=7-0
--- PASS: TestDirectStartStopVM (10.54s)
=== RUN   TestAllFunctions
    vhive_test.go:290: skipping TestAllFunctions in non-nightly runs.
--- SKIP: TestAllFunctions (0.00s)
PASS
coverage: 56.8% of statements
time="2023-10-18T13:28:59.002714198-06:00" level=info msg="waiting for goroutines"
time="2023-10-18T13:28:59.002822448-06:00" level=info msg="waiting done"
time="2023-10-18T13:28:59.002918431-06:00" level=info msg="Closing fcClient"
time="2023-10-18T13:28:59.003035630-06:00" level=info msg="Closing containerd client"
time="2023-10-18T13:28:59.003300006-06:00" level=info msg="Cleaning up network manager"
time="2023-10-18T13:28:59.013877099-06:00" level=error msg="failed to remove network deleting route: no such process:"
time="2023-10-18T13:28:59.014255877-06:00" level=error msg="failed to remove network deleting route: no such process:"
time="2023-10-18T13:28:59.065243305-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.104379667-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.104408805-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.104881154-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.105256365-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.104907348-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.105161705-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.123896406-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.123923063-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.123936855-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.123970435-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.123911642-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.123915085-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.123992954-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.124005904-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.175937771-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.175997672-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.176030031-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.176070455-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.176102981-06:00" level=debug msg="Removing tap" tap=tap0
time="2023-10-18T13:28:59.176128856-06:00" level=debug msg="Removing tap" tap=tap0
ok  	command-line-arguments	113.524s	coverage: 56.8% of statements
./scripts/clean_fcctr.sh
Killing firecracker agents and VMs
Resetting nftables
Deleting veth* devices created by CNI
Cannot find device "br0"
Cannot find device "br1"
Cleaning in /var/lib/cni/ non-network
find: ‘/var/lib/cni/’: No such file or directory
Removing devmapper devices
Removed fc-dev-thinpool-snap-28
Removed fc-dev-thinpool-snap-13
device-mapper: remove ioctl on fc-dev-thinpool  failed: Device or resource busy
Command failed.
Removed fc-dev-thinpool-snap-11
Removed fc-dev-thinpool-snap-24
Removed fc-dev-thinpool-snap-18
Removed fc-dev-thinpool-snap-20
Removed fc-dev-thinpool-snap-16
Removed fc-dev-thinpool-snap-9
Cleaning /var/lib/firecracker-containerd/containerd /var/lib/firecracker-containerd/runtime /var/lib/firecracker-containerd/shim-base /var/lib/firecracker-containerd/snapshotter
Cleaning /run/firecracker-containerd/*
Cleaning CNI state, e.g., allocated addresses
rm: cannot remove '/var/lib/cni/networks/fcnet*/last_reserved_ip.0': No such file or directory
clean already
rm: cannot remove '/var/lib/cni/networks/fcnet*/19*': No such file or directory
clean already
Cleaning snapshots
Creating a fresh devmapper
0 209715200 thin-pool /dev/loop33 /dev/loop32 128 32768 1 skip_block_zeroing
device-mapper: reload ioctl on fc-dev-thinpool  failed: No such device or address
Command failed.
sudo mkdir -m777 -p /tmp/ctrd-logs && sudo env "PATH=/usr/local/go/bin:/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>/tmp/ctrd-logs/fccd_orch_upf_log.out 2>/tmp/ctrd-logs/fccd_orch_upf_log.err &
sudo env "PATH=/usr/local/go/bin:/users/char/.vscode-server/bin/f1b07bd25dfad64b0167beb15359ae573aecd2cc/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin" go test vhive_test.go stats.go vhive.go functions.go -short -v -race -cover -args -snapshotsTest -upfTest
time="2023-10-18T13:29:05.010894968-06:00" level=info msg="Orchestrator snapshots enabled: true"
time="2023-10-18T13:29:05.011138892-06:00" level=info msg="Orchestrator UPF enabled: true"
time="2023-10-18T13:29:05.011194773-06:00" level=info msg="Orchestrator lazy serving mode enabled: false"
time="2023-10-18T13:29:05.011264361-06:00" level=info msg="Orchestrator UPF metrics enabled: false"
time="2023-10-18T13:29:05.011341840-06:00" level=info msg="Drop cache: true"
time="2023-10-18T13:29:05.011396287-06:00" level=info msg="Bench dir: bench_results"
time="2023-10-18T13:29:05.016327452-06:00" level=debug msg="Initializing network pool" poolSize=10
time="2023-10-18T13:29:05.027049401-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns80 tap=tap0
time="2023-10-18T13:29:05.027073618-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns83 tap=tap0
time="2023-10-18T13:29:05.027266011-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns81 tap=tap0
time="2023-10-18T13:29:05.029915155-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns88 tap=tap0
time="2023-10-18T13:29:05.032655654-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns82 tap=tap0
time="2023-10-18T13:29:05.035540770-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns84 tap=tap0
time="2023-10-18T13:29:05.037278350-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns89 tap=tap0
time="2023-10-18T13:29:05.041303115-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns86 tap=tap0
time="2023-10-18T13:29:05.041701511-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns85 tap=tap0
time="2023-10-18T13:29:05.042516363-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns87 tap=tap0
time="2023-10-18T13:29:05.051908486-06:00" level=debug msg="Initializing the memory manager"
time="2023-10-18T13:29:05.052051237-06:00" level=info msg="Creating containerd client"
time="2023-10-18T13:29:05.053333390-06:00" level=info msg="Created containerd client"
time="2023-10-18T13:29:05.053407995-06:00" level=info msg="Creating firecracker client"
time="2023-10-18T13:29:05.053812628-06:00" level=info msg="Created firecracker client"
time="2023-10-18T13:29:05.053949798-06:00" level=info msg="Creating image manager"
=== RUN   TestSendToFunctionSerial
time="2023-10-18T13:29:05.054289489-06:00" level=debug msg="fIDint=1, err=<nil>, pinnedFuncNum=0"
time="2023-10-18T13:29:05.054382838-06:00" level=debug msg="Created function, pinned=true, shut down after 0 requests" fID=1 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:29:05.054490740-06:00" level=info msg="New function added" fID=1 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T13:29:05.054596824-06:00" level=debug msg="Function is inactive, starting the instance..." fID=1
time="2023-10-18T13:29:05.054667099-06:00" level=debug msg="Adding instance" fID=1
time="2023-10-18T13:29:05.054731738-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:29:05.054858054-06:00" level=debug msg="Allocating a VM instance" vmID=1-0
time="2023-10-18T13:29:05.054945850-06:00" level=debug msg="Creating network config for function instance" funcID=1-0
time="2023-10-18T13:29:05.055081240-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=1-0
time="2023-10-18T13:29:05.055225777-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.88 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns87 NamespacePath=/var/run/netns/uvmns87 Veth0CIDR=172.17.1.94/30 Veth0Name=veth87-0 Veth1CIDR=172.17.1.93/30 Veth1Name=veth87-1 funcID=1-0
time="2023-10-18T13:29:05.058828435-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns90 tap=tap0
time="2023-10-18T13:29:05.154013472-06:00" level=debug msg=resolving host=ghcr.io
time="2023-10-18T13:29:05.154202418-06:00" level=debug msg="do request" host=ghcr.io request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=HEAD url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:29:05.570310642-06:00" level=debug msg="fetch response received" host=ghcr.io response.header.content-length=73 response.header.content-type=application/json response.header.date="Wed, 18 Oct 2023 19:29:05 GMT" response.header.www-authenticate="Bearer realm=\"https://ghcr.io/token\",service=\"ghcr.io\",scope=\"repository:ease-lab/helloworld:pull\"" response.header.x-github-request-id="B44F:1221:759C:ACC6:65303201" response.status="401 Unauthorized" url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:29:05.570556227-06:00" level=debug msg=Unauthorized header="Bearer realm=\"https://ghcr.io/token\",service=\"ghcr.io\",scope=\"repository:ease-lab/helloworld:pull\"" host=ghcr.io
time="2023-10-18T13:29:05.571839092-06:00" level=debug msg="do request" host=ghcr.io request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=HEAD url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:29:05.819470023-06:00" level=debug msg="fetch response received" host=ghcr.io response.header.content-length=1998 response.header.content-type=application/vnd.docker.distribution.manifest.v2+json response.header.date="Wed, 18 Oct 2023 19:29:05 GMT" response.header.docker-content-digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" response.header.docker-distribution-api-version=registry/2.0 response.header.etag="\"sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25\"" response.header.x-github-request-id="B44F:1221:75A7:ACDB:65303201" response.status="200 OK" url="https://ghcr.io/v2/ease-lab/helloworld/manifests/var_workload"
time="2023-10-18T13:29:05.819961066-06:00" level=debug msg=resolved desc.digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" host=ghcr.io
time="2023-10-18T13:29:05.820823439-06:00" level=debug msg=fetch digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" mediatype=application/vnd.docker.distribution.manifest.v2+json size=1998
time="2023-10-18T13:29:05.830908924-06:00" level=debug msg="do request" digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" mediatype=application/vnd.docker.distribution.manifest.v2+json request.header.accept="application/vnd.docker.distribution.manifest.v2+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=1998 url="https://ghcr.io/v2/ease-lab/helloworld/manifests/sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25"
time="2023-10-18T13:29:06.000749229-06:00" level=debug msg="fetch response received" digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" mediatype=application/vnd.docker.distribution.manifest.v2+json response.header.content-length=1998 response.header.content-type=application/vnd.docker.distribution.manifest.v2+json response.header.date="Wed, 18 Oct 2023 19:29:05 GMT" response.header.docker-content-digest="sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" response.header.docker-distribution-api-version=registry/2.0 response.header.etag="\"sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25\"" response.header.x-github-request-id="B44F:1221:75B0:ACE6:65303201" response.status="200 OK" size=1998 url="https://ghcr.io/v2/ease-lab/helloworld/manifests/sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25"
time="2023-10-18T13:29:06.014567937-06:00" level=debug msg=fetch digest="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734" mediatype=application/vnd.docker.container.image.v1+json size=8312
time="2023-10-18T13:29:06.019315534-06:00" level=debug msg="do request" digest="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734" mediatype=application/vnd.docker.container.image.v1+json request.header.accept="application/vnd.docker.container.image.v1+json, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=8312 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734"
time="2023-10-18T13:29:06.233511306-06:00" level=debug msg="fetch response received" digest="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734" mediatype=application/vnd.docker.container.image.v1+json response.header.accept-ranges=bytes response.header.age=119 response.header.content-disposition= response.header.content-length=8312 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:29:06 GMT" response.header.etag="\"0x8D950DD42AD3ECF\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:28 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, HIT" response.header.x-cache-hits="0, 1" response.header.x-fastly-request-id=9f93d6d21aa0fd85efd88ad828897ba0688ebc5e response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:28 GMT" response.header.x-ms-copy-id=32f6a8de-4aab-4519-b15c-21acf3d16ee7 response.header.x-ms-copy-progress=8312/8312 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:28 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=e4ffb3ee-c01e-0074-7af9-015390000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kjyo7100158-IAD, cache-bur-kbur8200176-BUR" response.status="200 OK" size=8312 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734"
time="2023-10-18T13:29:07.177374912-06:00" level=debug msg=fetch digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=2802957
time="2023-10-18T13:29:07.177623888-06:00" level=debug msg=fetch digest="sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=31506523
time="2023-10-18T13:29:07.177497102-06:00" level=debug msg=fetch digest="sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=301261
time="2023-10-18T13:29:07.178359962-06:00" level=debug msg=fetch digest="sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=2233
time="2023-10-18T13:29:07.178763901-06:00" level=debug msg=fetch digest="sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=637463
time="2023-10-18T13:29:07.177806192-06:00" level=debug msg=fetch digest="sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=230
time="2023-10-18T13:29:07.177898799-06:00" level=debug msg=fetch digest="sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=1887472
time="2023-10-18T13:29:07.178787967-06:00" level=debug msg=fetch digest="sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=20365161
time="2023-10-18T13:29:07.185586394-06:00" level=debug msg="do request" digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=2802957 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9"
time="2023-10-18T13:29:07.187025543-06:00" level=debug msg="do request" digest="sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=2233 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa"
time="2023-10-18T13:29:07.189871703-06:00" level=debug msg="do request" digest="sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=301261 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a"
time="2023-10-18T13:29:07.190303331-06:00" level=debug msg="do request" digest="sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=31506523 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e"
time="2023-10-18T13:29:07.192192052-06:00" level=debug msg="do request" digest="sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=637463 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d"
time="2023-10-18T13:29:07.194201475-06:00" level=debug msg="do request" digest="sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=230 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd"
time="2023-10-18T13:29:07.194967646-06:00" level=debug msg="do request" digest="sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=1887472 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d"
time="2023-10-18T13:29:07.196007568-06:00" level=debug msg="do request" digest="sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/1.6.20+unknown request.method=GET size=20365161 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a"
time="2023-10-18T13:29:07.346718307-06:00" level=debug msg="fetch response received" digest="sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=119 response.header.content-disposition= response.header.content-length=2233 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:29:07 GMT" response.header.etag="\"0x8D950DD3FE78031\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:23 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, HIT" response.header.x-cache-hits="0, 1" response.header.x-fastly-request-id=91b2a410c4706105a13b199614dac11d835eed79 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:23 GMT" response.header.x-ms-copy-id=07945db6-426d-45cd-8734-4c1ace32645f response.header.x-ms-copy-progress=2233/2233 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:23 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=0a4480ba-601e-0082-2af9-010cc9000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kiad7000099-IAD, cache-bur-kbur8200176-BUR" response.status="200 OK" size=2233 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:92b614cff45fafd028cec952e0cb2584e8d931bf0321e7d14bfafdf7a50ac3fa"
time="2023-10-18T13:29:07.361426286-06:00" level=debug msg="fetch response received" digest="sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=119 response.header.content-disposition= response.header.content-length=20365161 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:29:07 GMT" response.header.etag="\"0x8D950DD41BFE9A7\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:27 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, HIT" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=28c74f3b0c68589338934dcedcde233bf47c58e9 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:27 GMT" response.header.x-ms-copy-id=6c84899f-271a-4bda-89af-0d225eed6986 response.header.x-ms-copy-progress=20365161/20365161 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:27 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=8ec3d8ac-f01e-00eb-3af9-013df8000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kiad7000081-IAD, cache-bur-kbur8200176-BUR" response.status="200 OK" size=20365161 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0d776ee02572ee50935002da7bd7fcda4a60be4a48c4ace5dd3216c327e6767a"
time="2023-10-18T13:29:07.381709380-06:00" level=debug msg="fetch response received" digest="sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=119 response.header.content-length=31506523 response.header.content-md5="Zv0aX6nCRCbDhK8ON7Qm6w==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:29:07 GMT" response.header.etag="\"0x8D8DECD29E65DF1\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, HIT" response.header.x-cache-hits="0, 0" response.header.x-fastly-request-id=81421b775e4d6d115f2dc82de7a5be7b67aad26a response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=5640d7c0-6252-4446-86dc-177b315dbf08 response.header.x-ms-copy-progress=31506523/31506523 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=7674c810-e01e-00b6-1af9-0181d0000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kjyo7100121-IAD, cache-bur-kbur8200176-BUR" response.status="200 OK" size=31506523 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:61614c1a5710c76af6b2a9c7170a81eb0dd76ccf90e921abc0c9dcc1d5ed490e"
time="2023-10-18T13:29:07.382722201-06:00" level=debug msg="fetch response received" digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=240 response.header.content-length=2802957 response.header.content-md5="Xv6UEe5ym/lkcH1POk+jEQ==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:29:07 GMT" response.header.etag="\"0x8D8DEA430F3018D\"" response.header.last-modified="Thu, 04 Mar 2021 00:26:45 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="HIT, HIT" response.header.x-cache-hits="5, 1" response.header.x-fastly-request-id=a6818c9ccd7e8a59a80fbe2b0b04559ae4aafc9b response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 00:26:45 GMT" response.header.x-ms-copy-id=5eaa9616-c230-429e-a0df-0f62ee445e79 response.header.x-ms-copy-progress=2802957/2802957 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 00:26:45 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=d1198414-a01e-001a-1bf8-01b3af000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kjyo7100129-IAD, cache-bur-kbur8200176-BUR" response.status="200 OK" size=2802957 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9"
time="2023-10-18T13:29:07.383452601-06:00" level=debug msg="fetch response received" digest="sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=119 response.header.content-length=230 response.header.content-md5="OBaH1satt139FxF1ozF59w==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:29:07 GMT" response.header.etag="\"0x8D8DECD29DC1039\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, HIT" response.header.x-cache-hits="0, 1" response.header.x-fastly-request-id=4a57d98c45ee6a601d76bed99bf83a893a2e36d8 response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=a359c581-3a46-4cf8-a872-9e5325a9976a response.header.x-ms-copy-progress=230/230 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=9d5fd840-e01e-00ce-47f9-012bc1000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kcgs7200058-IAD, cache-bur-kbur8200176-BUR" response.status="200 OK" size=230 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:0522d30cde10ac29ae2c555b9bde76c2b50aafc7ef7435bbc7e19de706bcadcd"
time="2023-10-18T13:29:07.384378773-06:00" level=debug msg="fetch response received" digest="sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=119 response.header.content-disposition= response.header.content-length=637463 response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:29:07 GMT" response.header.etag="\"0x8D950DD4137CD4C\"" response.header.last-modified="Tue, 27 Jul 2021 09:02:26 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, HIT" response.header.x-cache-hits="0, 1" response.header.x-fastly-request-id=ffed440e934c663d2fd77d6affbc4cd5f5570ddb response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Tue, 27 Jul 2021 09:02:26 GMT" response.header.x-ms-copy-id=5112efd1-4292-4e39-bc9b-e40a2203877f response.header.x-ms-copy-progress=637463/637463 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Tue, 27 Jul 2021 09:02:26 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=664a0704-e01e-0066-08f9-01b5b6000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kcgs7200059-IAD, cache-bur-kbur8200176-BUR" response.status="200 OK" size=637463 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:c09d5cdb7367fff0d581bb8003e3520dc3e8bd78811dfb51c92df81e58c3a50d"
time="2023-10-18T13:29:07.402082287-06:00" level=debug msg="fetch response received" digest="sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=119 response.header.content-length=301261 response.header.content-md5="R+Y0FZj1Wto7rT9gTULVcg==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:29:07 GMT" response.header.etag="\"0x8D8DECD29E1022C\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, HIT" response.header.x-cache-hits="0, 1" response.header.x-fastly-request-id=052fd0f3d862536896c4a4057558e6842ebdbb6c response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=76d654fa-5faf-44e7-be31-1fab2ada3763 response.header.x-ms-copy-progress=301261/301261 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=3c5894b7-d01e-006d-27f9-0190a1000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kcgs7200149-IAD, cache-bur-kbur8200176-BUR" response.status="200 OK" size=301261 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:2cc5ad85d9abaadf23d5ae53c3f32e7ccb2df1956869980bfd2491ff396d348a"
time="2023-10-18T13:29:07.424561086-06:00" level=debug msg="fetch response received" digest="sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip response.header.accept-ranges=bytes response.header.age=119 response.header.content-length=1887472 response.header.content-md5="i9atZyjofrewlZ3ngsMOnA==" response.header.content-type=application/octet-stream response.header.date="Wed, 18 Oct 2023 19:29:07 GMT" response.header.etag="\"0x8D8DECD29DEACF9\"" response.header.last-modified="Thu, 04 Mar 2021 05:20:02 GMT" response.header.server="Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0" response.header.strict-transport-security="max-age=31536000" response.header.via="1.1 varnish, 1.1 varnish" response.header.x-cache="MISS, HIT" response.header.x-cache-hits="0, 1" response.header.x-fastly-request-id=8b6ad8e2b6027c04f638cdccfb9e0cc2854e06cd response.header.x-ms-blob-type=BlockBlob response.header.x-ms-copy-completion-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-copy-id=13d27f76-4a1b-4ae1-8f7a-582a9ed3e3e3 response.header.x-ms-copy-progress=1887472/1887472 response.header.x-ms-copy-status=success response.header.x-ms-creation-time="Thu, 04 Mar 2021 05:20:02 GMT" response.header.x-ms-lease-state=available response.header.x-ms-lease-status=unlocked response.header.x-ms-request-id=72fff6e6-a01e-00a2-54f9-019dfa000000 response.header.x-ms-server-encrypted=true response.header.x-ms-version=2019-12-12 response.header.x-served-by="cache-iad-kjyo7100107-IAD, cache-bur-kbur8200176-BUR" response.status="200 OK" size=1887472 url="https://ghcr.io/v2/ease-lab/helloworld/blobs/sha256:407da27a03363f6b9d368ec6e131f7f2db7c8cb2a149160d913d6f3698905a5d"
time="2023-10-18T13:29:12.438712704-06:00" level=debug msg="image unpacked" chainID="sha256:21d1c32ff456f90efb6f819dc94981414274877bd50caee6b7866a3f65280253" config="sha256:a87533385b75fd1d476d8a1112a0c5db953e3d5b44d4f9db814b1e2e6abb8734"
time="2023-10-18T13:29:12.510710838-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:29:12.510950445-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:29:14.301040945-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:29:14.406598674-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:29:14.584670922-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:29:14.584984382-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:29:14.602251462-06:00" level=debug msg="Registering VM with the memory manager" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:29:14.602546025-06:00" level=debug msg="Registering the VM with the memory manager" vmID=1-0
time="2023-10-18T13:29:14.602698322-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=1-0
time="2023-10-18T13:29:15.289873184-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=1
time="2023-10-18T13:29:15.292016622-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=1
time="2023-10-18T13:29:15.292206069-06:00" level=debug msg="First time offloading, need to create a snapshot first" fID=1
time="2023-10-18T13:29:15.292361961-06:00" level=debug msg="Creating instance snapshot" fID=1
time="2023-10-18T13:29:15.292495523-06:00" level=debug msg="Orchestrator received PauseVM" vmID=1-0
time="2023-10-18T13:29:15.294390383-06:00" level=debug msg="Initializing snapshot corresponding to revision and image" image="ghcr.io/ease-lab/helloworld:var_workload" revision=1
time="2023-10-18T13:29:15.294720676-06:00" level=debug msg="Orchestrator received CreateSnapshot" vmID=1-0
time="2023-10-18T13:29:16.240530370-06:00" level=debug msg="Creating patch file with disk state difference" patchFilePath=/fccd/snapshots/1/patch_file vmID=1-0
time="2023-10-18T13:29:16.487367369-06:00" level=debug msg="Serializing snapshot info" vmID=1-0
time="2023-10-18T13:29:16.488168630-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=1-0
time="2023-10-18T13:29:16.490186394-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=1
time="2023-10-18T13:29:16.491986263-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=1
time="2023-10-18T13:29:16.492211217-06:00" level=debug msg="Removing instance" fID=1 isSync=true
time="2023-10-18T13:29:16.492360080-06:00" level=debug msg="Orchestrator received StopVM" vmID=1-0
time="2023-10-18T13:29:21.506844530-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 1-0" vmID=1-0
time="2023-10-18T13:29:21.507127349-06:00" level=debug msg="Freeing a VM instance" vmID=1-0
time="2023-10-18T13:29:21.507249358-06:00" level=debug msg="Removing network config for function instance" funcID=1-0
time="2023-10-18T13:29:21.507400950-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=1-0
time="2023-10-18T13:29:21.507567499-06:00" level=debug msg="Stopped VM successfully" vmID=1-0
--- PASS: TestSendToFunctionSerial (16.45s)
=== RUN   TestSendToFunctionParallel
time="2023-10-18T13:29:21.541716925-06:00" level=debug msg="fIDint=2, err=<nil>, pinnedFuncNum=0"
time="2023-10-18T13:29:21.541996780-06:00" level=debug msg="Created function, pinned=true, shut down after 0 requests" fID=2 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:29:21.542292155-06:00" level=info msg="New function added" fID=2 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=0
time="2023-10-18T13:29:21.542568946-06:00" level=debug msg="Function is inactive, starting the instance..." fID=2
time="2023-10-18T13:29:21.542756415-06:00" level=debug msg="Adding instance" fID=2
time="2023-10-18T13:29:21.543063538-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:29:21.543831569-06:00" level=debug msg="Allocating a VM instance" vmID=2-0
time="2023-10-18T13:29:21.544220583-06:00" level=debug msg="Creating network config for function instance" funcID=2-0
time="2023-10-18T13:29:21.544872850-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=2-0
time="2023-10-18T13:29:21.546819838-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.88 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns87 NamespacePath=/var/run/netns/uvmns87 Veth0CIDR=172.17.1.94/30 Veth0Name=veth87-0 Veth1CIDR=172.17.1.93/30 Veth1Name=veth87-1 funcID=2-0
time="2023-10-18T13:29:21.551099579-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns91 tap=tap0
time="2023-10-18T13:29:21.609618755-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:29:21.609877695-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:29:23.471858482-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:29:23.569260560-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:29:23.741343342-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:29:23.741683923-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:29:23.757341721-06:00" level=debug msg="Registering VM with the memory manager" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:29:23.757654865-06:00" level=debug msg="Registering the VM with the memory manager" vmID=2-0
time="2023-10-18T13:29:23.757832333-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=2-0
time="2023-10-18T13:29:24.445351224-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.445519847-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.445831393-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446390782-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446721922-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446922233-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447265493-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446297796-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446084676-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447985098-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448498597-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448745590-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448100209-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446900763-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448909366-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.449754095-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449817398-06:00" level=debug msg="First time offloading, need to create a snapshot first" fID=2
time="2023-10-18T13:29:24.450410350-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450675578-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450878932-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450702474-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.450104153-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449954400-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446696605-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446678443-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447808402-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446746348-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446614479-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450255063-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449183330-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446990944-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446959517-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446204326-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446325673-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446465604-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446147463-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446625023-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446542444-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446597506-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447802501-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448059530-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448084344-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449919491-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449018698-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450305705-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450128139-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450358939-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450169860-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450468813-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447036227-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447159592-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447146518-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447148516-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447187444-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447367661-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447426931-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447214954-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447466798-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447586864-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447604435-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447548605-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447707308-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.447745340-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.446790348-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448314887-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448298355-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448212564-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448338960-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448586008-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448594595-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448445032-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448378901-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448636098-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448671137-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448716173-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.448712017-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449441077-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449050372-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449452136-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449099741-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449515144-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.449569396-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449623077-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449513370-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449866757-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449731949-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449679920-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450032542-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.450007488-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450041376-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450021868-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.449984986-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450151186-06:00" level=debug msg="Creating instance snapshot" fID=2
time="2023-10-18T13:29:24.450068742-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450181841-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.450185926-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450368279-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450437536-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450508279-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450576737-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450793079-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450785946-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450901152-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450724312-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.451010525-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450764967-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.450921687-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=2
time="2023-10-18T13:29:24.452693525-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.453073962-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.454158081-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.464609550-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.465524365-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.454856188-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.459411450-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.455942368-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.455199571-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.456394042-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.460191970-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.461803821-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.457310928-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.454613210-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.466503652-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.466923611-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.462053436-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.461964147-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.462286428-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.462514997-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.463720791-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.463066444-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.464171073-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.464366166-06:00" level=debug msg="Orchestrator received PauseVM" vmID=2-0
time="2023-10-18T13:29:24.466132821-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:24.469618415-06:00" level=debug msg="Initializing snapshot corresponding to revision and image" image="ghcr.io/ease-lab/helloworld:var_workload" revision=2
time="2023-10-18T13:29:24.469795313-06:00" level=debug msg="Orchestrator received CreateSnapshot" vmID=2-0
time="2023-10-18T13:29:25.406157631-06:00" level=debug msg="Creating patch file with disk state difference" patchFilePath=/fccd/snapshots/2/patch_file vmID=2-0
time="2023-10-18T13:29:25.643437390-06:00" level=debug msg="Serializing snapshot info" vmID=2-0
time="2023-10-18T13:29:25.643941023-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=2-0
time="2023-10-18T13:29:25.646937161-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.647647617-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.648478556-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.648979994-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.649682081-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.650304665-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.650915331-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.651779141-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.652628269-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.653522425-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.654317299-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.654962062-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.655575487-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.656193208-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.656924437-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.657021290-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.657482117-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.657902287-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.658368269-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.658928753-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.659452503-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.659993506-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.660613387-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.661022894-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.661703329-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.661938028-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.662406582-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.662835994-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.663439071-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.663899106-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.664616988-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.665101916-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.665657506-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.665874852-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.666692403-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.667037708-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.667488533-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.667839636-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.668303342-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.668777377-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.669422584-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.669899756-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.670519016-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.670953616-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.671603361-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.671942027-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.672538413-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.672976740-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.673806817-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.674165467-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.674630362-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.675137533-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.675631838-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.676030729-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.676544926-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.677115592-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.677759119-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.678171711-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.678516019-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.678973774-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.679401677-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.679985439-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.680574659-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.681006720-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.681299587-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.681721185-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.682145925-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.682647934-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.683229316-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.683604309-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.684091005-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=2
time="2023-10-18T13:29:25.684397370-06:00" level=debug msg="Removing instance" fID=2 isSync=true
time="2023-10-18T13:29:25.684544669-06:00" level=debug msg="Orchestrator received StopVM" vmID=2-0
time="2023-10-18T13:29:30.699019756-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 2-0" vmID=2-0
time="2023-10-18T13:29:30.699410060-06:00" level=debug msg="Freeing a VM instance" vmID=2-0
time="2023-10-18T13:29:30.699552390-06:00" level=debug msg="Removing network config for function instance" funcID=2-0
time="2023-10-18T13:29:30.699713199-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=2-0
time="2023-10-18T13:29:30.699891787-06:00" level=debug msg="Stopped VM successfully" vmID=2-0
--- PASS: TestSendToFunctionParallel (9.19s)
=== RUN   TestStartSendStopTwice
time="2023-10-18T13:29:30.734279146-06:00" level=debug msg="fIDint=3, err=<nil>, pinnedFuncNum=2"
time="2023-10-18T13:29:30.734433094-06:00" level=debug msg="Created function, pinned=true, shut down after 1 requests" fID=3 imageName="ghcr.io/ease-lab/helloworld:var_workload"
time="2023-10-18T13:29:30.734613868-06:00" level=info msg="New function added" fID=3 image="ghcr.io/ease-lab/helloworld:var_workload" isPinned=true servedTh=1
time="2023-10-18T13:29:30.734753920-06:00" level=debug msg="Function is inactive, starting the instance..." fID=3
time="2023-10-18T13:29:30.734830607-06:00" level=debug msg="Adding instance" fID=3
time="2023-10-18T13:29:30.734913944-06:00" level=debug msg="StartVM: Received StartVM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:29:30.735044013-06:00" level=debug msg="Allocating a VM instance" vmID=3-0
time="2023-10-18T13:29:30.735127139-06:00" level=debug msg="Creating network config for function instance" funcID=3-0
time="2023-10-18T13:29:30.735274322-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=3-0
time="2023-10-18T13:29:30.735427745-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.88 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns87 NamespacePath=/var/run/netns/uvmns87 Veth0CIDR=172.17.1.94/30 Veth0Name=veth87-0 Veth1CIDR=172.17.1.93/30 Veth1Name=veth87-1 funcID=3-0
time="2023-10-18T13:29:30.741220227-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns92 tap=tap0
time="2023-10-18T13:29:30.796091079-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:29:30.796372070-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:29:32.563431410-06:00" level=debug msg="StartVM: Creating a new container" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:29:32.666737394-06:00" level=debug msg="StartVM: Creating a new task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:29:32.849908430-06:00" level=debug msg="StartVM: Waiting for the task to get ready" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:29:32.850260424-06:00" level=debug msg="StartVM: Starting the task" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:29:32.862906809-06:00" level=debug msg="Registering VM with the memory manager" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:29:32.863264072-06:00" level=debug msg="Registering the VM with the memory manager" vmID=3-0
time="2023-10-18T13:29:32.863421678-06:00" level=debug msg="Successfully started a VM" image="ghcr.io/ease-lab/helloworld:var_workload" vmID=3-0
time="2023-10-18T13:29:33.579026402-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=3
time="2023-10-18T13:29:33.580923089-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=3
time="2023-10-18T13:29:33.581156476-06:00" level=debug msg="First time offloading, need to create a snapshot first" fID=3
time="2023-10-18T13:29:33.581371118-06:00" level=debug msg="Creating instance snapshot" fID=3
time="2023-10-18T13:29:33.581508907-06:00" level=debug msg="Orchestrator received PauseVM" vmID=3-0
time="2023-10-18T13:29:33.583241738-06:00" level=debug msg="Initializing snapshot corresponding to revision and image" image="ghcr.io/ease-lab/helloworld:var_workload" revision=3
time="2023-10-18T13:29:33.583538513-06:00" level=debug msg="Orchestrator received CreateSnapshot" vmID=3-0
time="2023-10-18T13:29:34.519244629-06:00" level=debug msg="Creating patch file with disk state difference" patchFilePath=/fccd/snapshots/3/patch_file vmID=3-0
time="2023-10-18T13:29:34.776168789-06:00" level=debug msg="Serializing snapshot info" vmID=3-0
time="2023-10-18T13:29:34.776663938-06:00" level=debug msg="Orchestrator received ResumeVM" vmID=3-0
time="2023-10-18T13:29:34.778688621-06:00" level=debug msg="FwdRPC: Forwarding RPC to function instance" fID=3
time="2023-10-18T13:29:34.781531988-06:00" level=debug msg="FwdRPC: Received a response from the  function instance" fID=3
time="2023-10-18T13:29:34.781805826-06:00" level=debug msg="Removing instance" fID=3 isSync=true
time="2023-10-18T13:29:34.781944043-06:00" level=debug msg="Orchestrator received StopVM" vmID=3-0
time="2023-10-18T13:29:39.795559324-06:00" level=error msg="failed to stop firecracker-containerd VM" error="rpc error: code = Internal desc = forcefully terminated VM 3-0" vmID=3-0
time="2023-10-18T13:29:39.795860145-06:00" level=debug msg="Freeing a VM instance" vmID=3-0
time="2023-10-18T13:29:39.795998011-06:00" level=debug msg="Removing network config for function instance" funcID=3-0
time="2023-10-18T13:29:39.796137527-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=3-0
time="2023-10-18T13:29:39.796292370-06:00" level=debug msg="Stopped VM successfully" vmID=3-0
time="2023-10-18T13:29:39.796523190-06:00" level=debug msg="Function is inactive, starting the instance..." fID=3
time="2023-10-18T13:29:39.796669038-06:00" level=debug msg="Adding instance" fID=3
time="2023-10-18T13:29:39.796794840-06:00" level=debug msg="Loading instance" fID=3
time="2023-10-18T13:29:39.796921413-06:00" level=debug msg="Orchestrator received LoadSnapshot" vmID=3-1
time="2023-10-18T13:29:39.797052830-06:00" level=debug msg="Allocating a VM instance" vmID=3-1
time="2023-10-18T13:29:39.797208294-06:00" level=debug msg="Creating network config for function instance" funcID=3-1
time="2023-10-18T13:29:39.797454611-06:00" level=debug msg="Allocating a new network config from network pool to function instance" funcID=3-1
time="2023-10-18T13:29:39.797735278-06:00" level=debug msg="Allocated a new network config" CloneIP=172.18.0.88 ContainerCIDR=172.16.0.2/24 ContainerIP=172.16.0.2 GatewayIP=172.16.0.1 HostDevName=tap0 NamespaceName=uvmns87 NamespacePath=/var/run/netns/uvmns87 Veth0CIDR=172.17.1.94/30 Veth0Name=veth87-0 Veth1CIDR=172.17.1.93/30 Veth1Name=veth87-1 funcID=3-1
time="2023-10-18T13:29:39.803968655-06:00" level=debug msg="Creating tap for virtual network" IP gateway=172.16.0.1/24 namespace=uvmns93 tap=tap0
time="2023-10-18T13:29:40.027589270-06:00" level=warning msg="Failed to Fetch k8s dns clusterIP exit status 1\nThe connection to the server localhost:8080 was refused - did you specify the right host or port?\n\n"
time="2023-10-18T13:29:40.027862145-06:00" level=warning msg="Using google dns 8.8.8.8\n"
time="2023-10-18T13:29:40.028124844-06:00" level=debug msg="Activating instance in the memory manager" vmID=3-1
time="2023-10-18T13:29:40.028288801-06:00" level=error msg="VM not registered with the memory manager" vmID=3-1
time="2023-10-18T13:29:40.028454512-06:00" level=debug msg="Freeing a VM instance" vmID=3-1
time="2023-10-18T13:29:40.028644738-06:00" level=debug msg="Removing network config for function instance" funcID=3-1
time="2023-10-18T13:29:40.028796911-06:00" level=debug msg="Releasing network config from function instance and adding it to network pool" funcID=3-1
time="2023-10-18T13:29:40.028982355-06:00" level=panic msg="VM not registered with the memory manager"
--- FAIL: TestStartSendStopTwice (9.33s)
panic: (*logrus.Entry) 0xc0001a8d90 [recovered]
	panic: (*logrus.Entry) 0xc0001a8d90

goroutine 136 [running]:
testing.tRunner.func1.2({0x1af08c0, 0xc0001a8d90})
	/usr/local/go/src/testing/testing.go:1389 +0x366
testing.tRunner.func1()
	/usr/local/go/src/testing/testing.go:1392 +0x5d2
panic({0x1af08c0, 0xc0001a8d90})
	/usr/local/go/src/runtime/panic.go:844 +0x258
github.com/sirupsen/logrus.(*Entry).log(0xc0001a8d20, 0x0, {0xc001362c90, 0x29})
	/users/char/go/pkg/mod/github.com/sirupsen/logrus@v1.9.0/entry.go:260 +0xa07
github.com/sirupsen/logrus.(*Entry).Log(0xc0001a8d20, 0x0, {0xc0005e1218, 0x1, 0x1})
	/users/char/go/pkg/mod/github.com/sirupsen/logrus@v1.9.0/entry.go:304 +0x8c
github.com/sirupsen/logrus.(*Logger).Log(0xc0001dc000, 0x0, {0xc0005e1218, 0x1, 0x1})
	/users/char/go/pkg/mod/github.com/sirupsen/logrus@v1.9.0/logger.go:204 +0x85
github.com/sirupsen/logrus.(*Logger).Panic(...)
	/users/char/go/pkg/mod/github.com/sirupsen/logrus@v1.9.0/logger.go:253
github.com/sirupsen/logrus.Panic(...)
	/users/char/go/pkg/mod/github.com/sirupsen/logrus@v1.9.0/exported.go:129
command-line-arguments.(*Function).LoadInstance(0xc0005ec240, {0xc000598d0d, 0x3})
	/users/char/vHive/functions.go:497 +0x44f
command-line-arguments.(*Function).AddInstance(0xc0005ec240)
	/users/char/vHive/functions.go:365 +0x2d8
command-line-arguments.(*Function).Serve.func1()
	/users/char/vHive/functions.go:263 +0x150
sync.(*Once).doSlow(0xc000ab6e84, 0xc0005e1c10)
	/usr/local/go/src/sync/once.go:68 +0x102
sync.(*Once).Do(0xc000ab6e84, 0x1afba4b?)
	/usr/local/go/src/sync/once.go:59 +0x47
command-line-arguments.(*Function).Serve(0xc0005ec240, {0x1afba4b?, 0x1?}, {0x1b25fa8?, 0x28?}, {0x886fc5?, 0x18c1b0e?}, {0x1afd81d, 0x5})
	/users/char/vHive/functions.go:257 +0x455
command-line-arguments.(*FuncPool).Serve(0x1cd9950?, {0x1cdff80, 0xc0001c2008}, {0x1afba4b, 0x1}, {0x1b25fa8, 0x28}, {0x1afd81d, 0x5})
	/users/char/vHive/functions.go:125 +0xea
command-line-arguments.TestStartSendStopTwice(0x0?)
	/users/char/vHive/vhive_test.go:160 +0x3dc
testing.tRunner(0xc000c9d1e0, 0x1b61f48)
	/usr/local/go/src/testing/testing.go:1439 +0x214
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:1486 +0x725
FAIL	command-line-arguments	35.066s
FAIL
make: *** [Makefile:57: test] Error 1