Initial patch for hacking vmd(8) into using both multiple device threads as well as protect event_base from corruption

vmd-thread-safety-02062020.patch

Index: Makefile
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/Makefile,v
retrieving revision 1.22
diff -u -p -u -p -r1.22 Makefile
--- Makefile	18 Jan 2019 01:24:07 -0000	1.22
+++ Makefile	2 Jun 2020 12:43:26 -0000
@@ -7,6 +7,7 @@ SRCS=		vmd.c control.c log.c priv.c proc
 SRCS+=		vm.c loadfile_elf.c pci.c virtio.c i8259.c mc146818.c
 SRCS+=		ns8250.c i8253.c vmboot.c ufs.c disklabel.c dhcp.c packet.c
 SRCS+=		parse.y atomicio.c vioscsi.c vioraw.c vioqcow2.c fw_cfg.c
+SRCS+=		safe_event.c
 
 CFLAGS+=	-Wall -I${.CURDIR}
 CFLAGS+=	-Wstrict-prototypes -Wmissing-prototypes
Index: control.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/control.c,v
retrieving revision 1.30
diff -u -p -u -p -r1.30 control.c
--- control.c	4 Dec 2018 08:15:09 -0000	1.30
+++ control.c	2 Jun 2020 12:43:26 -0000
@@ -29,6 +29,7 @@
 #include <errno.h>
 #include <event.h>
 #include <fcntl.h>
+#include <pthread.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
@@ -39,6 +40,8 @@
 
 #define	CONTROL_BACKLOG	5
 
+extern struct event_base *global_evbase;
+
 struct ctl_connlist ctl_conns;
 
 void
@@ -198,8 +201,10 @@ control_listen(struct control_sock *cs)
 
 	event_set(&cs->cs_ev, cs->cs_fd, EV_READ,
 	    control_accept, cs);
+	event_base_set(global_evbase, &cs->cs_ev);
 	event_add(&cs->cs_ev, NULL);
 	evtimer_set(&cs->cs_evt, control_accept, cs);
+	event_base_set(global_evbase, &cs->cs_evt);
 
 	return (0);
 }
@@ -256,6 +261,7 @@ control_accept(int listenfd, short event
 	c->iev.data = cs;
 	event_set(&c->iev.ev, c->iev.ibuf.fd, c->iev.events,
 	    c->iev.handler, c->iev.data);
+	event_base_set(global_evbase, &c->iev.ev);
 	event_add(&c->iev.ev, NULL);
 
 	TAILQ_INSERT_TAIL(&ctl_conns, c, entry);
Index: i8253.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/i8253.c,v
retrieving revision 1.31
diff -u -p -u -p -r1.31 i8253.c
--- i8253.c	30 Nov 2019 00:51:29 -0000	1.31
+++ i8253.c	2 Jun 2020 12:43:26 -0000
@@ -23,6 +23,7 @@
 #include <machine/vmmvar.h>
 
 #include <event.h>
+#include <pthread.h>
 #include <string.h>
 #include <stddef.h>
 #include <time.h>
@@ -32,6 +33,7 @@
 #include "proc.h"
 #include "vmm.h"
 #include "atomicio.h"
+#include "safe_event.h"
 
 extern char *__progname;
 
@@ -43,6 +45,18 @@ extern char *__progname;
  */
 struct i8253_channel i8253_channel[3];
 
+static struct event_base *evbase;
+static struct event watchdog;
+static struct timeval watchdog_tv;
+static pthread_t i8253_thread;
+static pthread_mutex_t evmutex;
+
+static void
+i8253_watchdog(int i, short s, void *arg)
+{
+	timer_add(&evmutex, &watchdog, &watchdog_tv);
+}
+
 /*
  * i8253_init
  *
@@ -54,6 +68,8 @@ struct i8253_channel i8253_channel[3];
 void
 i8253_init(uint32_t vm_id)
 {
+	int ret;
+
 	memset(&i8253_channel, 0, sizeof(struct i8253_channel));
 	clock_gettime(CLOCK_MONOTONIC, &i8253_channel[0].ts);
 	i8253_channel[0].start = 0xFFFF;
@@ -74,9 +90,42 @@ i8253_init(uint32_t vm_id)
 	i8253_channel[2].vm_id = vm_id;
 	i8253_channel[2].state = 0;
 
+	evbase = event_base_new();
+
 	evtimer_set(&i8253_channel[0].timer, i8253_fire, &i8253_channel[0]);
+	event_base_set(evbase, &i8253_channel[0].timer);
+
 	evtimer_set(&i8253_channel[1].timer, i8253_fire, &i8253_channel[1]);
+	event_base_set(evbase, &i8253_channel[1].timer);
+
 	evtimer_set(&i8253_channel[2].timer, i8253_fire, &i8253_channel[2]);
+	event_base_set(evbase, &i8253_channel[2].timer);
+
+	/* Configure watchdog timevals and timer */
+	timerclear(&watchdog_tv);
+	watchdog_tv.tv_sec = 1;
+
+	evtimer_set(&watchdog, i8253_watchdog, NULL);
+	event_base_set(evbase, &watchdog);
+	timer_add(&evmutex, &watchdog, &watchdog_tv);
+
+	ret = pthread_mutex_init(&evmutex, NULL);
+	if (ret) {
+		fatal("%s: failed to create pthread_mutex", __func__);
+	}
+
+}
+
+void
+i8253_init_thread(void)
+{
+	int ret;
+
+	log_debug("%s: starting thread with evbase %p", __func__, evbase);
+	ret = pthread_create(&i8253_thread, NULL, event_loop_thread, evbase);
+	if (ret) {
+		fatal("%s: could not create thread", __func__);
+	}
 }
 
 /*
@@ -311,14 +360,15 @@ i8253_reset(uint8_t chn)
 {
 	struct timeval tv;
 
-	evtimer_del(&i8253_channel[chn].timer);
+	timer_del(&evmutex, &i8253_channel[chn].timer);
 	timerclear(&tv);
 
 	i8253_channel[chn].in_use = 1;
 	i8253_channel[chn].state = 0;
 	tv.tv_usec = (i8253_channel[chn].start * NS_PER_TICK) / 1000;
 	clock_gettime(CLOCK_MONOTONIC, &i8253_channel[chn].ts);
-	evtimer_add(&i8253_channel[chn].timer, &tv);
+
+	timer_add(&evmutex, &i8253_channel[chn].timer, &tv);
 }
 
 /*
@@ -344,7 +394,7 @@ i8253_fire(int fd, short type, void *arg
 	if (ctr->mode != TIMER_INTTC) {
 		timerclear(&tv);
 		tv.tv_usec = (ctr->start * NS_PER_TICK) / 1000;
-		evtimer_add(&ctr->timer, &tv);
+		timer_add(&evmutex, &ctr->timer, &tv);
 	} else
 		ctr->state = 1;
 }
@@ -364,7 +414,7 @@ i8253_dump(int fd)
 int
 i8253_restore(int fd, uint32_t vm_id)
 {
-	int i;
+	int i, ret;
 	log_debug("%s: restoring PIT", __func__);
 	if (atomicio(read, fd, &i8253_channel, sizeof(i8253_channel)) !=
 	    sizeof(i8253_channel)) {
@@ -372,11 +422,20 @@ i8253_restore(int fd, uint32_t vm_id)
 		return (-1);
 	}
 
+	evbase = event_base_new();
+
+	ret = pthread_mutex_init(&evmutex, NULL);
+	if (ret) {
+		fatal("%s: failed to create pthread_mutex", __func__);
+	}
+
 	for (i = 0; i < 3; i++) {
 		memset(&i8253_channel[i].timer, 0, sizeof(struct event));
 		i8253_channel[i].vm_id = vm_id;
 		evtimer_set(&i8253_channel[i].timer, i8253_fire,
 		    &i8253_channel[i]);
+		event_base_set(evbase, &i8253_channel[i].timer);
+
 		i8253_reset(i);
 	}
 	return (0);
@@ -387,7 +446,7 @@ i8253_stop()
 {
 	int i;
 	for (i = 0; i < 3; i++)
-		evtimer_del(&i8253_channel[i].timer);
+		timer_del(&evmutex, &i8253_channel[i].timer);
 }
 
 void
Index: i8253.h
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/i8253.h,v
retrieving revision 1.9
diff -u -p -u -p -r1.9 i8253.h
--- i8253.h	26 Apr 2018 17:10:10 -0000	1.9
+++ i8253.h	2 Jun 2020 12:43:26 -0000
@@ -44,6 +44,7 @@ struct i8253_channel {
 };
 
 void i8253_init(uint32_t);
+void i8253_init_thread(void);
 void i8253_reset(uint8_t);
 void i8253_fire(int, short, void *);
 int i8253_dump(int);
Index: mc146818.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/mc146818.c,v
retrieving revision 1.21
diff -u -p -u -p -r1.21 mc146818.c
--- mc146818.c	30 Nov 2019 00:51:29 -0000	1.21
+++ mc146818.c	2 Jun 2020 12:43:26 -0000
@@ -23,6 +23,7 @@
 #include <machine/vmmvar.h>
 
 #include <event.h>
+#include <pthread.h>
 #include <stddef.h>
 #include <string.h>
 #include <time.h>
@@ -34,6 +35,7 @@
 #include "virtio.h"
 #include "vmm.h"
 #include "atomicio.h"
+#include "safe_event.h"
 
 #define MC_DIVIDER_MASK 0xe0
 #define MC_RATE_MASK 0xf
@@ -50,6 +52,12 @@
 
 #define TOBCD(x)	(((x) / 10 * 16) + ((x) % 10))
 
+static struct event_base *evbase;
+static struct event watchdog;
+static struct timeval watchdog_tv;
+static pthread_t mc146818_thread;
+static pthread_mutex_t evmutex;
+
 struct mc146818 {
 	time_t now;
 	uint8_t idx;
@@ -110,7 +118,7 @@ rtc_fire1(int fd, short type, void *arg)
 		    "resync", __func__, (rtc.now - old));
 		vmmci_ctl(VMMCI_SYNCRTC);
 	}
-	evtimer_add(&rtc.sec, &rtc.sec_tv);
+	ev_add(&evmutex, &rtc.sec, &rtc.sec_tv);
 }
 
 /*
@@ -131,7 +139,13 @@ rtc_fireper(int fd, short type, void *ar
 	vcpu_assert_pic_irq((ptrdiff_t)arg, 0, 8);
 	vcpu_deassert_pic_irq((ptrdiff_t)arg, 0, 8);
 
-	evtimer_add(&rtc.per, &rtc.per_tv);
+	ev_add(&evmutex, &rtc.per, &rtc.per_tv);
+}
+
+static void
+mc146818_watchdog(int fd, short type, void *arg)
+{
+	timer_add(&evmutex, &watchdog, &watchdog_tv);
 }
 
 /*
@@ -147,6 +161,8 @@ rtc_fireper(int fd, short type, void *ar
 void
 mc146818_init(uint32_t vm_id, uint64_t memlo, uint64_t memhi)
 {
+	int ret;
+
 	memset(&rtc, 0, sizeof(rtc));
 	time(&rtc.now);
 
@@ -171,10 +187,38 @@ mc146818_init(uint32_t vm_id, uint64_t m
 
 	timerclear(&rtc.per_tv);
 
+	ret = pthread_mutex_init(&evmutex, NULL);
+	if (ret) {
+		fatal("%s: failed to create pthread_mutex", __func__);
+	}
+
+	evbase = event_base_new();
+
 	evtimer_set(&rtc.sec, rtc_fire1, NULL);
-	evtimer_add(&rtc.sec, &rtc.sec_tv);
+	event_base_set(evbase, &rtc.sec);
+	timer_add(&evmutex, &rtc.sec, &rtc.sec_tv);
 
 	evtimer_set(&rtc.per, rtc_fireper, (void *)(intptr_t)rtc.vm_id);
+	event_base_set(evbase, &rtc.per);
+
+	/* Watchdog */
+	timerclear(&watchdog_tv);
+	watchdog_tv.tv_sec = 30;
+	evtimer_set(&watchdog, mc146818_watchdog, NULL);
+	event_base_set(evbase, &watchdog);
+	timer_add(&evmutex, &watchdog, &watchdog_tv);
+}
+
+void
+mc146818_init_thread(void)
+{
+	int ret;
+
+	log_debug("%s: starting thread with evbase %p", __func__, evbase);
+	ret = pthread_create(&mc146818_thread, NULL, event_loop_thread, evbase);
+	if (ret) {
+		fatal("%s: could not create thread", __func__);
+	}
 }
 
 /*
@@ -193,10 +237,13 @@ rtc_reschedule_per(void)
 		rate = 32768 >> ((rtc.regs[MC_REGA] & MC_RATE_MASK) - 1);
 		us = (1.0 / rate) * 1000000;
 		rtc.per_tv.tv_usec = us;
+
+		pthread_mutex_lock(&evmutex);
 		if (evtimer_pending(&rtc.per, NULL))
 			evtimer_del(&rtc.per);
 
 		evtimer_add(&rtc.per, &rtc.per_tv);
+		pthread_mutex_unlock(&evmutex);
 	}
 }
 
@@ -341,20 +388,24 @@ mc146818_restore(int fd, uint32_t vm_id)
 	memset(&rtc.sec, 0, sizeof(struct event));
 	memset(&rtc.per, 0, sizeof(struct event));
 	evtimer_set(&rtc.sec, rtc_fire1, NULL);
+	event_base_set(evbase, &rtc.sec);
+
 	evtimer_set(&rtc.per, rtc_fireper, (void *)(intptr_t)rtc.vm_id);
+	event_base_set(evbase, &rtc.per);
+
 	return (0);
 }
 
 void
 mc146818_stop()
 {
-	evtimer_del(&rtc.per);
-	evtimer_del(&rtc.sec);
+	timer_del(&evmutex, &rtc.per);
+	timer_del(&evmutex, &rtc.sec);
 }
 
 void
 mc146818_start()
 {
-	evtimer_add(&rtc.sec, &rtc.sec_tv);
+	timer_add(&evmutex, &rtc.sec, &rtc.sec_tv);
 	rtc_reschedule_per();
 }
Index: mc146818.h
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/mc146818.h,v
retrieving revision 1.5
diff -u -p -u -p -r1.5 mc146818.h
--- mc146818.h	9 Jul 2017 00:51:40 -0000	1.5
+++ mc146818.h	2 Jun 2020 12:43:26 -0000
@@ -16,6 +16,7 @@
  */
 
 void mc146818_init(uint32_t, uint64_t, uint64_t);
+void mc146818_init_thread(void);
 uint8_t vcpu_exit_mc146818(struct vm_run_params *vrp);
 void dump_mc146818(void);
 int mc146818_dump(int);
Index: ns8250.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/ns8250.c,v
retrieving revision 1.25
diff -u -p -u -p -r1.25 ns8250.c
--- ns8250.c	11 Dec 2019 06:45:16 -0000	1.25
+++ ns8250.c	2 Jun 2020 12:43:26 -0000
@@ -33,13 +33,30 @@
 #include "vmd.h"
 #include "vmm.h"
 #include "atomicio.h"
+#include "safe_event.h"
 
 extern char *__progname;
 struct ns8250_dev com1_dev;
 
+static struct event_base *evbase;
+static struct event watchdog;
+static struct timeval watchdog_tv;
+static pthread_t ns8250_thread;
+static pthread_mutex_t evmutex;
+
+static pthread_t ratelimit_thread;
+static struct event_base *ratelimit_evbase;
+static pthread_mutex_t ratelimit_mutex;
+
 static void com_rcv_event(int, short, void *);
 static void com_rcv(struct ns8250_dev *, uint32_t, uint32_t);
 
+static void
+ns8250_watchdog(int fd, short type, void *arg)
+{
+	timer_add(&evmutex, &watchdog, &watchdog_tv);
+}
+
 /*
  * ratelimit
  *
@@ -54,11 +71,10 @@ static void com_rcv(struct ns8250_dev *,
 static void
 ratelimit(int fd, short type, void *arg)
 {
-	/* Set TXRDY and clear "no pending interrupt" */
-	com1_dev.regs.iir |= IIR_TXRDY;
-	com1_dev.regs.iir &= ~IIR_NOPEND;
 	vcpu_assert_pic_irq(com1_dev.vmid, 0, com1_dev.irq);
 	vcpu_deassert_pic_irq(com1_dev.vmid, 0, com1_dev.irq);
+
+	timer_add(&ratelimit_mutex, &com1_dev.rate, &com1_dev.rate_tv);
 }
 
 void
@@ -66,12 +82,28 @@ ns8250_init(int fd, uint32_t vmid)
 {
 	int ret;
 
+	ret = pthread_mutex_init(&evmutex, NULL);
+	if (ret) {
+		errno = ret;
+		fatal("could not initialize ns8250 event mutex");
+	}
+
+	evbase = event_base_new();
+	ratelimit_evbase = event_base_new();
+
 	memset(&com1_dev, 0, sizeof(com1_dev));
 	ret = pthread_mutex_init(&com1_dev.mutex, NULL);
 	if (ret) {
 		errno = ret;
 		fatal("could not initialize com1 mutex");
 	}
+
+	ret = pthread_mutex_init(&ratelimit_mutex, NULL);
+	if (ret) {
+		errno = ret;
+		fatal("could not initialize ratelimit mutex");
+	}
+
 	com1_dev.fd = fd;
 	com1_dev.irq = 4;
 	com1_dev.portid = NS8250_COM1;
@@ -98,6 +130,7 @@ ns8250_init(int fd, uint32_t vmid)
 
 	event_set(&com1_dev.event, com1_dev.fd, EV_READ | EV_PERSIST,
 	    com_rcv_event, (void *)(intptr_t)vmid);
+	event_base_set(evbase, &com1_dev.event);
 
 	/*
 	 * Whenever fd is writable implies that the pty slave is connected.
@@ -106,12 +139,46 @@ ns8250_init(int fd, uint32_t vmid)
 	 */
 	event_set(&com1_dev.wake, com1_dev.fd, EV_WRITE,
 	    com_rcv_event, (void *)(intptr_t)vmid);
-	event_add(&com1_dev.wake, NULL);
+	event_base_set(evbase, &com1_dev.wake);
+	ev_add(&evmutex, &com1_dev.event, NULL);
 
 	/* Rate limiter for simulating baud rate */
 	timerclear(&com1_dev.rate_tv);
 	com1_dev.rate_tv.tv_usec = 10000;
 	evtimer_set(&com1_dev.rate, ratelimit, NULL);
+	event_base_set(ratelimit_evbase, &com1_dev.rate);
+	evtimer_add(&com1_dev.rate, &com1_dev.rate_tv);
+
+	/* Watchdog for keeping the event pump running */
+	timerclear(&watchdog_tv);
+	watchdog_tv.tv_sec = 5;
+	evtimer_set(&watchdog, ns8250_watchdog, NULL);
+	event_base_set(evbase, &watchdog);
+	evtimer_add(&watchdog, &watchdog_tv);
+}
+
+static void
+init_ratelimit_thread(void)
+{
+	int ret;
+	log_debug("%s: starting ratelimit thread with evbase: %p", __func__, ratelimit_evbase);
+	ret = pthread_create(&ratelimit_thread, NULL, event_loop_thread, ratelimit_evbase);
+	if (ret) {
+		fatal("%s: failed to start ratelimit thread: %d", __func__, ret);
+	}
+}
+
+void
+ns8250_init_thread(void)
+{
+	int ret;
+
+	log_debug("%s: starting thread with evbase %p", __func__, evbase);
+	ret = pthread_create(&ns8250_thread, NULL, event_loop_thread, evbase);
+	if (ret) {
+		fatal("%s: failed to start ns8250 thread: %d", __func__, ret);
+	}
+	init_ratelimit_thread();
 }
 
 static void
@@ -120,7 +187,7 @@ com_rcv_event(int fd, short kind, void *
 	mutex_lock(&com1_dev.mutex);
 
 	if (kind == EV_WRITE) {
-		event_add(&com1_dev.event, NULL);
+		ev_add(&evmutex, &com1_dev.event, NULL);
 		mutex_unlock(&com1_dev.mutex);
 		return;
 	}
@@ -201,8 +268,8 @@ com_rcv(struct ns8250_dev *com, uint32_t
 		if (errno != EAGAIN)
 			log_warn("unexpected read error on com device");
 	} else if (sz == 0) {
-		event_del(&com->event);
-		event_add(&com->wake, NULL);
+		ev_del(&evmutex, &com->event);
+		ev_add(&evmutex, &com->wake, NULL);
 		return;
 	} else if (sz != 1 && sz != 2)
 		log_warnx("unexpected read return value %zd on com device", sz);
@@ -255,16 +322,8 @@ vcpu_process_com_data(struct vm_exit *ve
 		com1_dev.byte_out++;
 
 		if (com1_dev.regs.ier & IER_ETXRDY) {
-			/* Limit output rate if needed */
-			if (com1_dev.pause_ct > 0 &&
-			    com1_dev.byte_out % com1_dev.pause_ct == 0) {
-					evtimer_add(&com1_dev.rate,
-					    &com1_dev.rate_tv);
-			} else {
-				/* Set TXRDY and clear "no pending interrupt" */
-				com1_dev.regs.iir |= IIR_TXRDY;
-				com1_dev.regs.iir &= ~IIR_NOPEND;
-			}
+			com1_dev.regs.iir |= IIR_TXRDY;
+			com1_dev.regs.iir &= ~IIR_NOPEND;
 		}
 	} else {
 		if (com1_dev.regs.lcr & LCR_DLAB) {
@@ -654,15 +713,15 @@ ns8250_restore(int fd, int con_fd, uint3
 	com1_dev.byte_out = 0;
 	com1_dev.regs.divlo = 1;
 	com1_dev.baudrate = 115200;
-	com1_dev.rate_tv.tv_usec = 10000;
 	com1_dev.pause_ct = (com1_dev.baudrate / 8) / 1000 * 10;
-	evtimer_set(&com1_dev.rate, ratelimit, NULL);
 
 	event_set(&com1_dev.event, com1_dev.fd, EV_READ | EV_PERSIST,
 	    com_rcv_event, (void *)(intptr_t)vmid);
+	event_base_set(evbase, &com1_dev.event);
 
 	event_set(&com1_dev.wake, com1_dev.fd, EV_WRITE,
 	    com_rcv_event, (void *)(intptr_t)vmid);
+	event_base_set(evbase, &com1_dev.wake);
 
 	return (0);
 }
@@ -670,15 +729,19 @@ ns8250_restore(int fd, int con_fd, uint3
 void
 ns8250_stop()
 {
-	if(event_del(&com1_dev.event))
+	if (ev_del(&evmutex, &com1_dev.event))
 		log_warn("could not delete ns8250 event handler");
-	evtimer_del(&com1_dev.rate);
+	ev_del(&ratelimit_mutex, &com1_dev.rate);
+
+	// Safely abort the ratelimiter thread by closing the event base
+	event_base_loopexit(ratelimit_evbase, NULL);
 }
 
 void
 ns8250_start()
 {
-	event_add(&com1_dev.event, NULL);
-	event_add(&com1_dev.wake, NULL);
+	ev_add(&evmutex, &com1_dev.event, NULL);
+	ev_add(&evmutex, &com1_dev.wake, NULL);
 	evtimer_add(&com1_dev.rate, &com1_dev.rate_tv);
+	init_ratelimit_thread();
 }
Index: ns8250.h
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/ns8250.h,v
retrieving revision 1.9
diff -u -p -u -p -r1.9 ns8250.h
--- ns8250.h	11 Dec 2019 06:45:16 -0000	1.9
+++ ns8250.h	2 Jun 2020 12:43:26 -0000
@@ -63,8 +63,8 @@ struct ns8250_dev {
 	pthread_mutex_t mutex;
 	struct ns8250_regs regs;
 	struct event event;
-	struct event rate;
 	struct event wake;
+	struct event rate;
 	struct timeval rate_tv;
 	enum ns8250_portid portid;
 	int fd;
@@ -77,6 +77,7 @@ struct ns8250_dev {
 };
 
 void ns8250_init(int, uint32_t);
+void ns8250_init_thread(void);
 uint8_t vcpu_exit_com(struct vm_run_params *);
 uint8_t vcpu_process_com_data(struct vm_exit *, uint32_t, uint32_t);
 void vcpu_process_com_lcr(struct vm_exit *);
Index: priv.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/priv.c,v
retrieving revision 1.15
diff -u -p -u -p -r1.15 priv.c
--- priv.c	28 Jun 2019 13:32:51 -0000	1.15
+++ priv.c	2 Jun 2020 12:43:26 -0000
@@ -46,6 +46,8 @@
 #include "proc.h"
 #include "vmd.h"
 
+extern struct event_base *global_evbase;
+
 int	 priv_dispatch_parent(int, struct privsep_proc *, struct imsg *);
 void	 priv_run(struct privsep *, struct privsep_proc *, void *);
 
Index: proc.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/proc.c,v
retrieving revision 1.18
diff -u -p -u -p -r1.18 proc.c
--- proc.c	10 Sep 2018 10:36:01 -0000	1.18
+++ proc.c	2 Jun 2020 12:43:26 -0000
@@ -30,12 +30,15 @@
 #include <errno.h>
 #include <signal.h>
 #include <paths.h>
+#include <pthread.h>
 #include <pwd.h>
 #include <event.h>
 #include <imsg.h>
 
 #include "proc.h"
 
+extern struct event_base *global_evbase;
+
 void	 proc_exec(struct privsep *, struct privsep_proc *, unsigned int, int,
 	    int, char **);
 void	 proc_setup(struct privsep *, struct privsep_proc *, unsigned int);
@@ -158,6 +161,7 @@ proc_exec(struct privsep *ps, struct pri
 			default:
 				/* Close child end. */
 				close(fd);
+				event_reinit(global_evbase);
 				break;
 			}
 		}
@@ -185,6 +189,7 @@ proc_connect(struct privsep *ps)
 			imsg_init(&iev->ibuf, ps->ps_pp->pp_pipes[dst][inst]);
 			event_set(&iev->ev, iev->ibuf.fd, iev->events,
 			    iev->handler, iev->data);
+			event_base_set(global_evbase, &iev->ev);
 			event_add(&iev->ev, NULL);
 		}
 	}
@@ -290,6 +295,7 @@ proc_accept(struct privsep *ps, int fd, 
 	iev = &ps->ps_ievs[dst][n];
 	imsg_init(&iev->ibuf, fd);
 	event_set(&iev->ev, iev->ibuf.fd, iev->events, iev->handler, iev->data);
+	event_base_set(global_evbase, &iev->ev);
 	event_add(&iev->ev, NULL);
 }
 
@@ -566,14 +572,21 @@ proc_run(struct privsep *ps, struct priv
 	    setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid))
 		fatal("%s: cannot drop privileges", __func__);
 
-	event_init();
+	//event_init();
+	//global_evbase = event_base_new();
 
 	signal_set(&ps->ps_evsigint, SIGINT, proc_sig_handler, p);
+	event_base_set(global_evbase, &ps->ps_evsigint);
 	signal_set(&ps->ps_evsigterm, SIGTERM, proc_sig_handler, p);
+	event_base_set(global_evbase, &ps->ps_evsigterm);
 	signal_set(&ps->ps_evsigchld, SIGCHLD, proc_sig_handler, p);
+	event_base_set(global_evbase, &ps->ps_evsigchld);
 	signal_set(&ps->ps_evsighup, SIGHUP, proc_sig_handler, p);
+	event_base_set(global_evbase, &ps->ps_evsighup);
 	signal_set(&ps->ps_evsigpipe, SIGPIPE, proc_sig_handler, p);
+	event_base_set(global_evbase, &ps->ps_evsigpipe);
 	signal_set(&ps->ps_evsigusr1, SIGUSR1, proc_sig_handler, p);
+	event_base_set(global_evbase, &ps->ps_evsigusr1);
 
 	signal_add(&ps->ps_evsigint, NULL);
 	signal_add(&ps->ps_evsigterm, NULL);
@@ -599,7 +612,7 @@ proc_run(struct privsep *ps, struct priv
 	if (run != NULL)
 		run(ps, p, arg);
 
-	event_dispatch();
+	event_base_dispatch(global_evbase);
 
 	proc_shutdown(p);
 }
@@ -626,7 +639,7 @@ proc_dispatch(int fd, short event, void 
 		if (n == 0) {
 			/* this pipe is dead, so remove the event handler */
 			event_del(&iev->ev);
-			event_loopexit(NULL);
+			event_base_loopexit(global_evbase, NULL);
 			return;
 		}
 	}
@@ -637,7 +650,7 @@ proc_dispatch(int fd, short event, void 
 		if (n == 0) {
 			/* this pipe is dead, so remove the event handler */
 			event_del(&iev->ev);
-			event_loopexit(NULL);
+			event_base_loopexit(global_evbase, NULL);
 			return;
 		}
 	}
@@ -714,6 +727,7 @@ imsg_event_add(struct imsgev *iev)
 
 	event_del(&iev->ev);
 	event_set(&iev->ev, iev->ibuf.fd, iev->events, iev->handler, iev->data);
+	event_base_set(global_evbase, &iev->ev);
 	event_add(&iev->ev, NULL);
 }
 
Index: safe_event.c
===================================================================
RCS file: safe_event.c
diff -N safe_event.c
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ safe_event.c	2 Jun 2020 12:43:26 -0000
@@ -0,0 +1,68 @@
+#include <event.h>
+#include <pthread.h>
+
+#include "proc.h"
+#include "safe_event.h"
+
+int
+ev_add(pthread_mutex_t *mutex, struct event *ev, struct timeval *tv)
+{
+	int ret;
+	pthread_mutex_lock(mutex);
+	ret = event_add(ev, tv);
+	pthread_mutex_unlock(mutex);
+	return ret;
+}
+
+int
+ev_del(pthread_mutex_t *mutex,struct event *ev)
+{
+	int ret;
+	pthread_mutex_lock(mutex);
+	ret = event_del(ev);
+	pthread_mutex_unlock(mutex);
+	return ret;
+}
+
+void
+timer_add(pthread_mutex_t *mutex, struct event *ev, struct timeval *tv)
+{
+	pthread_mutex_lock(mutex);
+	evtimer_add(ev, tv);
+	pthread_mutex_unlock(mutex);
+}
+
+void
+timer_del(pthread_mutex_t *mutex, struct event *ev)
+{
+	pthread_mutex_lock(mutex);
+	evtimer_del(ev);
+	pthread_mutex_unlock(mutex);
+}
+
+int
+timer_pending(pthread_mutex_t *mutex, struct event *ev, struct timeval *tv)
+{
+	int ret;
+	pthread_mutex_lock(mutex);
+	ret = evtimer_pending(ev, tv);
+	pthread_mutex_unlock(mutex);
+	return ret;
+}
+
+void *
+event_loop_thread(void *arg)
+{
+	struct event_base *evbase;
+	intptr_t ret;
+	int r;
+
+	evbase = arg;
+
+	r = event_base_dispatch(evbase);
+	ret = r;
+
+	log_info("%s: event loop for evbase %p exiting (r=%d)",
+	    __func__, evbase, r);
+	return (void *)ret;
+}
Index: safe_event.h
===================================================================
RCS file: safe_event.h
diff -N safe_event.h
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ safe_event.h	2 Jun 2020 12:43:26 -0000
@@ -0,0 +1,6 @@
+int ev_add(pthread_mutex_t *, struct event *, struct timeval *tv);
+int ev_del(pthread_mutex_t *, struct event *);
+void timer_add(pthread_mutex_t *, struct event *, struct timeval *tv);
+void timer_del(pthread_mutex_t *, struct event *);
+int timer_pending(pthread_mutex_t *, struct event *, struct timeval *);
+void *event_loop_thread(void *);
Index: virtio.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/virtio.c,v
retrieving revision 1.82
diff -u -p -u -p -r1.82 virtio.c
--- virtio.c	11 Dec 2019 06:45:16 -0000	1.82
+++ virtio.c	2 Jun 2020 12:43:26 -0000
@@ -34,6 +34,7 @@
 #include <errno.h>
 #include <event.h>
 #include <poll.h>
+#include <pthread.h>
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
@@ -46,6 +47,7 @@
 #include "vioscsi.h"
 #include "loadfile.h"
 #include "atomicio.h"
+#include "safe_event.h"
 
 extern char *__progname;
 struct viornd_dev viornd;
@@ -54,6 +56,12 @@ struct vionet_dev *vionet;
 struct vioscsi_dev *vioscsi;
 struct vmmci_dev vmmci;
 
+static struct event_base *evbase;
+static struct event watchdog;
+static struct timeval watchdog_tv;
+static pthread_t virtio_thread;
+static pthread_mutex_t evmutex;
+
 int nr_vionet;
 int nr_vioblk;
 
@@ -68,6 +76,12 @@ int nr_vioblk;
 #define RXQ	0
 #define TXQ	1
 
+static void
+virtio_watchdog(int fd, short type, void *arg)
+{
+	timer_add(&evmutex, &watchdog, &watchdog_tv);
+}
+
 const char *
 vioblk_cmd_name(uint32_t type)
 {
@@ -1587,7 +1601,7 @@ vmmci_ctl(unsigned int cmd)
 
 		/* Add ACK timeout */
 		tv.tv_sec = VMMCI_TIMEOUT;
-		evtimer_add(&vmmci.timeout, &tv);
+		timer_add(&evmutex, &vmmci.timeout, &tv);
 		break;
 	case VMMCI_SYNCRTC:
 		if (vmmci.cfg.guest_feature & VMMCI_F_SYNCRTC) {
@@ -1627,7 +1641,7 @@ vmmci_ack(unsigned int cmd)
 			log_debug("%s: vm %u requested shutdown", __func__,
 			    vmmci.vm_id);
 			tv.tv_sec = VMMCI_TIMEOUT;
-			evtimer_add(&vmmci.timeout, &tv);
+			timer_add(&evmutex, &vmmci.timeout, &tv);
 			return;
 		}
 		/* FALLTHROUGH */
@@ -1640,11 +1654,11 @@ vmmci_ack(unsigned int cmd)
 		 * killing it forcefully.
 		 */
 		if (cmd == vmmci.cmd &&
-		    evtimer_pending(&vmmci.timeout, NULL)) {
+		    timer_pending(&evmutex, &vmmci.timeout, NULL)) {
 			log_debug("%s: vm %u acknowledged shutdown request",
 			    __func__, vmmci.vm_id);
 			tv.tv_sec = VMMCI_SHUTDOWN_TIMEOUT;
-			evtimer_add(&vmmci.timeout, &tv);
+			timer_add(&evmutex, &vmmci.timeout, &tv);
 		}
 		break;
 	case VMMCI_SYNCRTC:
@@ -1792,6 +1806,13 @@ virtio_init(struct vmd_vm *vm, int child
 	uint8_t i;
 	int ret;
 
+	evbase = event_base_new();
+
+	ret = pthread_mutex_init(&evmutex, NULL);
+	if (ret) {
+		fatal("%s: failed to create pthread_mutex", __func__);
+	}
+
 	/* Virtio entropy device */
 	if (pci_add_device(&id, PCI_VENDOR_QUMRANET,
 	    PCI_PRODUCT_QUMRANET_VIO_RNG, PCI_CLASS_SYSTEM,
@@ -1879,7 +1900,8 @@ virtio_init(struct vmd_vm *vm, int child
 
 			event_set(&vionet[i].event, vionet[i].fd,
 			    EV_READ | EV_PERSIST, vionet_rx_event, &vionet[i]);
-			if (event_add(&vionet[i].event, NULL)) {
+			event_base_set(evbase, &vionet[i].event);
+			if (ev_add(&evmutex, &vionet[i].event, NULL)) {
 				log_warn("could not initialize vionet event "
 				    "handler");
 				return;
@@ -2033,6 +2055,26 @@ virtio_init(struct vmd_vm *vm, int child
 	vmmci.pci_id = id;
 
 	evtimer_set(&vmmci.timeout, vmmci_timeout, NULL);
+	event_base_set(evbase, &vmmci.timeout);
+
+	/* Watchdog setup */
+	timerclear(&watchdog_tv);
+	watchdog_tv.tv_sec = 30;
+	evtimer_set(&watchdog, virtio_watchdog, NULL);
+	event_base_set(evbase, &watchdog);
+	timer_add(&evmutex, &watchdog, &watchdog_tv);
+}
+
+void
+virtio_init_thread(void)
+{
+	int ret;
+
+	log_debug("%s: starting thread with evbase %p", __func__, evbase);
+	ret = pthread_create(&virtio_thread, NULL, event_loop_thread, evbase);
+	if (ret) {
+		fatal("%s: could not create thread", __func__);
+	}
 }
 
 void
@@ -2066,6 +2108,7 @@ vmmci_restore(int fd, uint32_t vm_id)
 	vmmci.irq = pci_get_dev_irq(vmmci.pci_id);
 	memset(&vmmci.timeout, 0, sizeof(struct event));
 	evtimer_set(&vmmci.timeout, vmmci_timeout, NULL);
+	event_base_set(evbase, &vmmci.timeout);
 	return (0);
 }
 
@@ -2096,6 +2139,8 @@ vionet_restore(int fd, struct vmd_vm *vm
 	uint8_t i;
 	int ret;
 
+	evbase = event_base_new();
+
 	nr_vionet = vcp->vcp_nnics;
 	if (vcp->vcp_nnics > 0) {
 		vionet = calloc(vcp->vcp_nnics, sizeof(struct vionet_dev));
@@ -2140,6 +2185,7 @@ vionet_restore(int fd, struct vmd_vm *vm
 			memset(&vionet[i].event, 0, sizeof(struct event));
 			event_set(&vionet[i].event, vionet[i].fd,
 			    EV_READ | EV_PERSIST, vionet_rx_event, &vionet[i]);
+			event_base_set(evbase, &vionet[i].event);
 		}
 	}
 	return (0);
@@ -2339,7 +2385,7 @@ virtio_stop(struct vm_create_params *vcp
 {
 	uint8_t i;
 	for (i = 0; i < vcp->vcp_nnics; i++) {
-		if (event_del(&vionet[i].event)) {
+		if (ev_del(&evmutex, &vionet[i].event)) {
 			log_warn("could not initialize vionet event "
 			    "handler");
 			return;
@@ -2352,7 +2398,7 @@ virtio_start(struct vm_create_params *vc
 {
 	uint8_t i;
 	for (i = 0; i < vcp->vcp_nnics; i++) {
-		if (event_add(&vionet[i].event, NULL)) {
+		if (ev_add(&evmutex, &vionet[i].event, NULL)) {
 			log_warn("could not initialize vionet event "
 			    "handler");
 			return;
Index: virtio.h
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/virtio.h,v
retrieving revision 1.35
diff -u -p -u -p -r1.35 virtio.h
--- virtio.h	11 Dec 2019 06:45:16 -0000	1.35
+++ virtio.h	2 Jun 2020 12:43:26 -0000
@@ -261,6 +261,7 @@ struct ioinfo {
 
 /* virtio.c */
 void virtio_init(struct vmd_vm *, int, int[][VM_MAX_BASE_PER_DISK], int *);
+void virtio_init_thread(void);
 void virtio_shutdown(struct vmd_vm *);
 int virtio_dump(int);
 int virtio_restore(int, struct vmd_vm *, int,
Index: vm.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/vm.c,v
retrieving revision 1.57
diff -u -p -u -p -r1.57 vm.c
--- vm.c	30 Apr 2020 03:50:53 -0000	1.57
+++ vm.c	2 Jun 2020 12:43:26 -0000
@@ -107,6 +107,9 @@ extern struct vmd *env;
 
 extern char *__progname;
 
+extern struct event_base *global_evbase;
+extern pthread_mutex_t global_evmutex;
+
 pthread_mutex_t threadmutex;
 pthread_cond_t threadcond;
 
@@ -364,8 +367,6 @@ start_vm(struct vmd_vm *vm, int fd)
 	for (i = 0; i < VMM_MAX_NICS_PER_VM; i++)
 		nicfds[i] = vm->vm_ifs[i].vif_fd;
 
-	event_init();
-
 	if (vm->vm_state & VM_STATE_RECEIVED) {
 		restore_emulated_hw(vcp, vm->vm_receive_fd, nicfds,
 		    vm->vm_disks, vm->vm_cdrom);
@@ -403,6 +404,8 @@ vm_dispatch_vmm(int fd, short event, voi
 	ssize_t			 n;
 	int			 verbose;
 
+	pthread_mutex_lock(&global_evmutex);
+
 	if (event & EV_READ) {
 		if ((n = imsg_read(ibuf)) == -1 && errno != EAGAIN)
 			fatal("%s: imsg_read", __func__);
@@ -424,7 +427,7 @@ vm_dispatch_vmm(int fd, short event, voi
 			break;
 
 #if DEBUG > 1
-		log_debug("%s: got imsg %d from %s",
+		log_info("%s: got imsg %d from %s",
 		    __func__, imsg.hdr.type,
 		    vm->vm_params.vmc_params.vcp_name);
 #endif
@@ -482,6 +485,8 @@ vm_dispatch_vmm(int fd, short event, voi
 		imsg_free(&imsg);
 	}
 	imsg_event_add(iev);
+
+	pthread_mutex_unlock(&global_evmutex);
 }
 
 /*
@@ -492,6 +497,8 @@ vm_dispatch_vmm(int fd, short event, voi
 __dead void
 vm_shutdown(unsigned int cmd)
 {
+	pthread_mutex_lock(&global_evmutex);
+
 	switch (cmd) {
 	case VMMCI_NONE:
 	case VMMCI_SHUTDOWN:
@@ -507,6 +514,8 @@ vm_shutdown(unsigned int cmd)
 	}
 	imsg_flush(&current_vm->vm_iev.ibuf);
 
+	pthread_mutex_unlock(&global_evmutex);
+
 	_exit(0);
 }
 
@@ -777,9 +786,13 @@ pause_vm(struct vm_create_params *vcp)
 	}
 
 	i8253_stop();
+	log_debug("%s: i8253 stopped", __func__);
 	mc146818_stop();
+	log_debug("%s: mc14618 stopped", __func__);
 	ns8250_stop();
+	log_debug("%s: ns8250 stopped", __func__);
 	virtio_stop(vcp);
+	log_debug("%s: virtio stopped", __func__);
 }
 
 void
@@ -800,10 +813,14 @@ unpause_vm(struct vm_create_params *vcp)
 		}
 	}
 
-	i8253_start();
-	mc146818_start();
-	ns8250_start();
 	virtio_start(vcp);
+	log_debug("%s: virtio restarted", __func__);
+	ns8250_start();
+	log_debug("%s: ns8250 restarted", __func__);
+	mc146818_start();
+	log_debug("%s: mc146818 restarted", __func__);
+	i8253_start();
+	log_debug("%s: i8253 restarted", __func__);
 }
 
 /*
@@ -1300,6 +1317,14 @@ run_vm(int child_cdrom, int child_disks[
 		return (ret);
 	}
 
+	/* XXX: this logic could move elsewhere...need some initial events
+	 * configured before we start the event loops.
+	 */
+	i8253_init_thread();
+	mc146818_init_thread();
+	virtio_init_thread();
+	ns8250_init_thread();
+
 	for (;;) {
 		ret = pthread_cond_wait(&threadcond, &threadmutex);
 		if (ret) {
@@ -1357,7 +1382,7 @@ event_thread(void *arg)
 	uint8_t *donep = arg;
 	intptr_t ret;
 
-	ret = event_dispatch();
+	ret = event_base_dispatch(global_evbase);
 
 	mutex_lock(&threadmutex);
 	*donep = 1;
Index: vmd.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/vmd.c,v
retrieving revision 1.117
diff -u -p -u -p -r1.117 vmd.c
--- vmd.c	12 Dec 2019 03:53:38 -0000	1.117
+++ vmd.c	2 Jun 2020 12:43:27 -0000
@@ -33,6 +33,7 @@
 #include <errno.h>
 #include <event.h>
 #include <fcntl.h>
+#include <pthread.h>
 #include <pwd.h>
 #include <signal.h>
 #include <syslog.h>
@@ -75,6 +76,8 @@ static struct privsep_proc procs[] = {
 	{ "vmm",	PROC_VMM,	vmd_dispatch_vmm, vmm, vmm_shutdown },
 };
 
+pthread_mutex_t global_evmutex;
+struct event_base *global_evbase;
 struct event staggered_start_timer;
 
 /* For the privileged process */
@@ -820,6 +823,12 @@ main(int argc, char **argv)
 	if (title != NULL)
 		ps->ps_title[proc_id] = title;
 
+	global_evbase = event_base_new();
+
+	if (pthread_mutex_init(&global_evmutex, NULL)) {
+		fatal("can't initialize global pthread_mutex");
+	}
+
 	/* only the parent returns */
 	proc_init(ps, procs, nitems(procs), env->vmd_debug, argc0, argv,
 	    proc_id);
@@ -831,13 +840,16 @@ main(int argc, char **argv)
 	if (ps->ps_noaction == 0)
 		log_info("startup");
 
-	event_init();
-
 	signal_set(&ps->ps_evsigint, SIGINT, vmd_sighdlr, ps);
+	event_base_set(global_evbase, &ps->ps_evsigint);
 	signal_set(&ps->ps_evsigterm, SIGTERM, vmd_sighdlr, ps);
+	event_base_set(global_evbase, &ps->ps_evsigterm);
 	signal_set(&ps->ps_evsighup, SIGHUP, vmd_sighdlr, ps);
+	event_base_set(global_evbase, &ps->ps_evsighup);
 	signal_set(&ps->ps_evsigpipe, SIGPIPE, vmd_sighdlr, ps);
+	event_base_set(global_evbase, &ps->ps_evsigpipe);
 	signal_set(&ps->ps_evsigusr1, SIGUSR1, vmd_sighdlr, ps);
+	event_base_set(global_evbase, &ps->ps_evsigusr1);
 
 	signal_add(&ps->ps_evsigint, NULL);
 	signal_add(&ps->ps_evsigterm, NULL);
@@ -851,7 +863,7 @@ main(int argc, char **argv)
 	if (vmd_configure() == -1)
 		fatalx("configuration failed");
 
-	event_dispatch();
+	event_base_dispatch(global_evbase);
 
 	log_debug("parent exiting");
 
@@ -951,6 +963,8 @@ vmd_configure(void)
 
 	log_debug("%s: starting vms in staggered fashion", __func__);
 	evtimer_set(&staggered_start_timer, start_vm_batch, NULL);
+	event_base_set(global_evbase, &staggered_start_timer);
+
 	/* start first batch */
 	start_vm_batch(0, 0, NULL);
 
@@ -1021,6 +1035,8 @@ vmd_reload(unsigned int reset, const cha
 
 		log_debug("%s: starting vms in staggered fashion", __func__);
 		evtimer_set(&staggered_start_timer, start_vm_batch, NULL);
+		event_base_set(global_evbase, &staggered_start_timer);
+
 		/* start first batch */
 		start_vm_batch(0, 0, NULL);
 
Index: vmm.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/vmm.c,v
retrieving revision 1.96
diff -u -p -u -p -r1.96 vmm.c
--- vmm.c	30 Apr 2020 03:50:53 -0000	1.96
+++ vmm.c	2 Jun 2020 12:43:27 -0000
@@ -64,6 +64,9 @@ int opentap(char *);
 
 extern struct vmd *env;
 
+extern pthread_mutex_t global_evmutex;
+extern struct event_base *global_evbase;
+
 static struct privsep_proc procs[] = {
 	{ "parent",	PROC_PARENT,	vmm_dispatch_parent  },
 };
@@ -80,9 +83,12 @@ vmm_run(struct privsep *ps, struct privs
 	if (config_init(ps->ps_env) == -1)
 		fatal("failed to initialize configuration");
 
+	pthread_mutex_lock(&global_evmutex);
 	signal_del(&ps->ps_evsigchld);
 	signal_set(&ps->ps_evsigchld, SIGCHLD, vmm_sighdlr, ps);
+	event_base_set(global_evbase, &ps->ps_evsigchld);
 	signal_add(&ps->ps_evsigchld, NULL);
+	pthread_mutex_unlock(&global_evmutex);
 
 	/*
 	 * pledge in the vmm process:
@@ -198,7 +204,7 @@ vmm_dispatch_parent(int fd, struct privs
 				/*
 				 * Request reboot but mark the VM as shutting
 				 * down. This way we can terminate the VM after
-				 * the triple fault instead of reboot and 
+				 * the triple fault instead of reboot and
 				 * avoid being stuck in the ACPI-less powerdown
 				 * ("press any key to reboot") of the VM.
 				 */
@@ -709,6 +715,8 @@ vmm_start_vm(struct imsg *imsg, uint32_t
 		/* Child */
 		close(fds[0]);
 		close(PROC_PARENT_SOCK_FILENO);
+
+		event_reinit(global_evbase);
 
 		ret = start_vm(vm, fds[1]);