Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Initial patch for hacking vmd(8) into using both multiple device threads as well as protect event_base from corruption
Index: Makefile
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/Makefile,v
retrieving revision 1.22
diff -u -p -u -p -r1.22 Makefile
--- Makefile 18 Jan 2019 01:24:07 -0000 1.22
+++ Makefile 2 Jun 2020 12:43:26 -0000
@@ -7,6 +7,7 @@ SRCS= vmd.c control.c log.c priv.c proc
SRCS+= vm.c loadfile_elf.c pci.c virtio.c i8259.c mc146818.c
SRCS+= ns8250.c i8253.c vmboot.c ufs.c disklabel.c dhcp.c packet.c
SRCS+= parse.y atomicio.c vioscsi.c vioraw.c vioqcow2.c fw_cfg.c
+SRCS+= safe_event.c
CFLAGS+= -Wall -I${.CURDIR}
CFLAGS+= -Wstrict-prototypes -Wmissing-prototypes
Index: control.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/control.c,v
retrieving revision 1.30
diff -u -p -u -p -r1.30 control.c
--- control.c 4 Dec 2018 08:15:09 -0000 1.30
+++ control.c 2 Jun 2020 12:43:26 -0000
@@ -29,6 +29,7 @@
#include <errno.h>
#include <event.h>
#include <fcntl.h>
+#include <pthread.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
@@ -39,6 +40,8 @@
#define CONTROL_BACKLOG 5
+extern struct event_base *global_evbase;
+
struct ctl_connlist ctl_conns;
void
@@ -198,8 +201,10 @@ control_listen(struct control_sock *cs)
event_set(&cs->cs_ev, cs->cs_fd, EV_READ,
control_accept, cs);
+ event_base_set(global_evbase, &cs->cs_ev);
event_add(&cs->cs_ev, NULL);
evtimer_set(&cs->cs_evt, control_accept, cs);
+ event_base_set(global_evbase, &cs->cs_evt);
return (0);
}
@@ -256,6 +261,7 @@ control_accept(int listenfd, short event
c->iev.data = cs;
event_set(&c->iev.ev, c->iev.ibuf.fd, c->iev.events,
c->iev.handler, c->iev.data);
+ event_base_set(global_evbase, &c->iev.ev);
event_add(&c->iev.ev, NULL);
TAILQ_INSERT_TAIL(&ctl_conns, c, entry);
Index: i8253.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/i8253.c,v
retrieving revision 1.31
diff -u -p -u -p -r1.31 i8253.c
--- i8253.c 30 Nov 2019 00:51:29 -0000 1.31
+++ i8253.c 2 Jun 2020 12:43:26 -0000
@@ -23,6 +23,7 @@
#include <machine/vmmvar.h>
#include <event.h>
+#include <pthread.h>
#include <string.h>
#include <stddef.h>
#include <time.h>
@@ -32,6 +33,7 @@
#include "proc.h"
#include "vmm.h"
#include "atomicio.h"
+#include "safe_event.h"
extern char *__progname;
@@ -43,6 +45,18 @@ extern char *__progname;
*/
struct i8253_channel i8253_channel[3];
+static struct event_base *evbase;
+static struct event watchdog;
+static struct timeval watchdog_tv;
+static pthread_t i8253_thread;
+static pthread_mutex_t evmutex;
+
+static void
+i8253_watchdog(int i, short s, void *arg)
+{
+ timer_add(&evmutex, &watchdog, &watchdog_tv);
+}
+
/*
* i8253_init
*
@@ -54,6 +68,8 @@ struct i8253_channel i8253_channel[3];
void
i8253_init(uint32_t vm_id)
{
+ int ret;
+
memset(&i8253_channel, 0, sizeof(struct i8253_channel));
clock_gettime(CLOCK_MONOTONIC, &i8253_channel[0].ts);
i8253_channel[0].start = 0xFFFF;
@@ -74,9 +90,42 @@ i8253_init(uint32_t vm_id)
i8253_channel[2].vm_id = vm_id;
i8253_channel[2].state = 0;
+ evbase = event_base_new();
+
evtimer_set(&i8253_channel[0].timer, i8253_fire, &i8253_channel[0]);
+ event_base_set(evbase, &i8253_channel[0].timer);
+
evtimer_set(&i8253_channel[1].timer, i8253_fire, &i8253_channel[1]);
+ event_base_set(evbase, &i8253_channel[1].timer);
+
evtimer_set(&i8253_channel[2].timer, i8253_fire, &i8253_channel[2]);
+ event_base_set(evbase, &i8253_channel[2].timer);
+
+ /* Configure watchdog timevals and timer */
+ timerclear(&watchdog_tv);
+ watchdog_tv.tv_sec = 1;
+
+ evtimer_set(&watchdog, i8253_watchdog, NULL);
+ event_base_set(evbase, &watchdog);
+ timer_add(&evmutex, &watchdog, &watchdog_tv);
+
+ ret = pthread_mutex_init(&evmutex, NULL);
+ if (ret) {
+ fatal("%s: failed to create pthread_mutex", __func__);
+ }
+
+}
+
+void
+i8253_init_thread(void)
+{
+ int ret;
+
+ log_debug("%s: starting thread with evbase %p", __func__, evbase);
+ ret = pthread_create(&i8253_thread, NULL, event_loop_thread, evbase);
+ if (ret) {
+ fatal("%s: could not create thread", __func__);
+ }
}
/*
@@ -311,14 +360,15 @@ i8253_reset(uint8_t chn)
{
struct timeval tv;
- evtimer_del(&i8253_channel[chn].timer);
+ timer_del(&evmutex, &i8253_channel[chn].timer);
timerclear(&tv);
i8253_channel[chn].in_use = 1;
i8253_channel[chn].state = 0;
tv.tv_usec = (i8253_channel[chn].start * NS_PER_TICK) / 1000;
clock_gettime(CLOCK_MONOTONIC, &i8253_channel[chn].ts);
- evtimer_add(&i8253_channel[chn].timer, &tv);
+
+ timer_add(&evmutex, &i8253_channel[chn].timer, &tv);
}
/*
@@ -344,7 +394,7 @@ i8253_fire(int fd, short type, void *arg
if (ctr->mode != TIMER_INTTC) {
timerclear(&tv);
tv.tv_usec = (ctr->start * NS_PER_TICK) / 1000;
- evtimer_add(&ctr->timer, &tv);
+ timer_add(&evmutex, &ctr->timer, &tv);
} else
ctr->state = 1;
}
@@ -364,7 +414,7 @@ i8253_dump(int fd)
int
i8253_restore(int fd, uint32_t vm_id)
{
- int i;
+ int i, ret;
log_debug("%s: restoring PIT", __func__);
if (atomicio(read, fd, &i8253_channel, sizeof(i8253_channel)) !=
sizeof(i8253_channel)) {
@@ -372,11 +422,20 @@ i8253_restore(int fd, uint32_t vm_id)
return (-1);
}
+ evbase = event_base_new();
+
+ ret = pthread_mutex_init(&evmutex, NULL);
+ if (ret) {
+ fatal("%s: failed to create pthread_mutex", __func__);
+ }
+
for (i = 0; i < 3; i++) {
memset(&i8253_channel[i].timer, 0, sizeof(struct event));
i8253_channel[i].vm_id = vm_id;
evtimer_set(&i8253_channel[i].timer, i8253_fire,
&i8253_channel[i]);
+ event_base_set(evbase, &i8253_channel[i].timer);
+
i8253_reset(i);
}
return (0);
@@ -387,7 +446,7 @@ i8253_stop()
{
int i;
for (i = 0; i < 3; i++)
- evtimer_del(&i8253_channel[i].timer);
+ timer_del(&evmutex, &i8253_channel[i].timer);
}
void
Index: i8253.h
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/i8253.h,v
retrieving revision 1.9
diff -u -p -u -p -r1.9 i8253.h
--- i8253.h 26 Apr 2018 17:10:10 -0000 1.9
+++ i8253.h 2 Jun 2020 12:43:26 -0000
@@ -44,6 +44,7 @@ struct i8253_channel {
};
void i8253_init(uint32_t);
+void i8253_init_thread(void);
void i8253_reset(uint8_t);
void i8253_fire(int, short, void *);
int i8253_dump(int);
Index: mc146818.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/mc146818.c,v
retrieving revision 1.21
diff -u -p -u -p -r1.21 mc146818.c
--- mc146818.c 30 Nov 2019 00:51:29 -0000 1.21
+++ mc146818.c 2 Jun 2020 12:43:26 -0000
@@ -23,6 +23,7 @@
#include <machine/vmmvar.h>
#include <event.h>
+#include <pthread.h>
#include <stddef.h>
#include <string.h>
#include <time.h>
@@ -34,6 +35,7 @@
#include "virtio.h"
#include "vmm.h"
#include "atomicio.h"
+#include "safe_event.h"
#define MC_DIVIDER_MASK 0xe0
#define MC_RATE_MASK 0xf
@@ -50,6 +52,12 @@
#define TOBCD(x) (((x) / 10 * 16) + ((x) % 10))
+static struct event_base *evbase;
+static struct event watchdog;
+static struct timeval watchdog_tv;
+static pthread_t mc146818_thread;
+static pthread_mutex_t evmutex;
+
struct mc146818 {
time_t now;
uint8_t idx;
@@ -110,7 +118,7 @@ rtc_fire1(int fd, short type, void *arg)
"resync", __func__, (rtc.now - old));
vmmci_ctl(VMMCI_SYNCRTC);
}
- evtimer_add(&rtc.sec, &rtc.sec_tv);
+ ev_add(&evmutex, &rtc.sec, &rtc.sec_tv);
}
/*
@@ -131,7 +139,13 @@ rtc_fireper(int fd, short type, void *ar
vcpu_assert_pic_irq((ptrdiff_t)arg, 0, 8);
vcpu_deassert_pic_irq((ptrdiff_t)arg, 0, 8);
- evtimer_add(&rtc.per, &rtc.per_tv);
+ ev_add(&evmutex, &rtc.per, &rtc.per_tv);
+}
+
+static void
+mc146818_watchdog(int fd, short type, void *arg)
+{
+ timer_add(&evmutex, &watchdog, &watchdog_tv);
}
/*
@@ -147,6 +161,8 @@ rtc_fireper(int fd, short type, void *ar
void
mc146818_init(uint32_t vm_id, uint64_t memlo, uint64_t memhi)
{
+ int ret;
+
memset(&rtc, 0, sizeof(rtc));
time(&rtc.now);
@@ -171,10 +187,38 @@ mc146818_init(uint32_t vm_id, uint64_t m
timerclear(&rtc.per_tv);
+ ret = pthread_mutex_init(&evmutex, NULL);
+ if (ret) {
+ fatal("%s: failed to create pthread_mutex", __func__);
+ }
+
+ evbase = event_base_new();
+
evtimer_set(&rtc.sec, rtc_fire1, NULL);
- evtimer_add(&rtc.sec, &rtc.sec_tv);
+ event_base_set(evbase, &rtc.sec);
+ timer_add(&evmutex, &rtc.sec, &rtc.sec_tv);
evtimer_set(&rtc.per, rtc_fireper, (void *)(intptr_t)rtc.vm_id);
+ event_base_set(evbase, &rtc.per);
+
+ /* Watchdog */
+ timerclear(&watchdog_tv);
+ watchdog_tv.tv_sec = 30;
+ evtimer_set(&watchdog, mc146818_watchdog, NULL);
+ event_base_set(evbase, &watchdog);
+ timer_add(&evmutex, &watchdog, &watchdog_tv);
+}
+
+void
+mc146818_init_thread(void)
+{
+ int ret;
+
+ log_debug("%s: starting thread with evbase %p", __func__, evbase);
+ ret = pthread_create(&mc146818_thread, NULL, event_loop_thread, evbase);
+ if (ret) {
+ fatal("%s: could not create thread", __func__);
+ }
}
/*
@@ -193,10 +237,13 @@ rtc_reschedule_per(void)
rate = 32768 >> ((rtc.regs[MC_REGA] & MC_RATE_MASK) - 1);
us = (1.0 / rate) * 1000000;
rtc.per_tv.tv_usec = us;
+
+ pthread_mutex_lock(&evmutex);
if (evtimer_pending(&rtc.per, NULL))
evtimer_del(&rtc.per);
evtimer_add(&rtc.per, &rtc.per_tv);
+ pthread_mutex_unlock(&evmutex);
}
}
@@ -341,20 +388,24 @@ mc146818_restore(int fd, uint32_t vm_id)
memset(&rtc.sec, 0, sizeof(struct event));
memset(&rtc.per, 0, sizeof(struct event));
evtimer_set(&rtc.sec, rtc_fire1, NULL);
+ event_base_set(evbase, &rtc.sec);
+
evtimer_set(&rtc.per, rtc_fireper, (void *)(intptr_t)rtc.vm_id);
+ event_base_set(evbase, &rtc.per);
+
return (0);
}
void
mc146818_stop()
{
- evtimer_del(&rtc.per);
- evtimer_del(&rtc.sec);
+ timer_del(&evmutex, &rtc.per);
+ timer_del(&evmutex, &rtc.sec);
}
void
mc146818_start()
{
- evtimer_add(&rtc.sec, &rtc.sec_tv);
+ timer_add(&evmutex, &rtc.sec, &rtc.sec_tv);
rtc_reschedule_per();
}
Index: mc146818.h
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/mc146818.h,v
retrieving revision 1.5
diff -u -p -u -p -r1.5 mc146818.h
--- mc146818.h 9 Jul 2017 00:51:40 -0000 1.5
+++ mc146818.h 2 Jun 2020 12:43:26 -0000
@@ -16,6 +16,7 @@
*/
void mc146818_init(uint32_t, uint64_t, uint64_t);
+void mc146818_init_thread(void);
uint8_t vcpu_exit_mc146818(struct vm_run_params *vrp);
void dump_mc146818(void);
int mc146818_dump(int);
Index: ns8250.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/ns8250.c,v
retrieving revision 1.25
diff -u -p -u -p -r1.25 ns8250.c
--- ns8250.c 11 Dec 2019 06:45:16 -0000 1.25
+++ ns8250.c 2 Jun 2020 12:43:26 -0000
@@ -33,13 +33,30 @@
#include "vmd.h"
#include "vmm.h"
#include "atomicio.h"
+#include "safe_event.h"
extern char *__progname;
struct ns8250_dev com1_dev;
+static struct event_base *evbase;
+static struct event watchdog;
+static struct timeval watchdog_tv;
+static pthread_t ns8250_thread;
+static pthread_mutex_t evmutex;
+
+static pthread_t ratelimit_thread;
+static struct event_base *ratelimit_evbase;
+static pthread_mutex_t ratelimit_mutex;
+
static void com_rcv_event(int, short, void *);
static void com_rcv(struct ns8250_dev *, uint32_t, uint32_t);
+static void
+ns8250_watchdog(int fd, short type, void *arg)
+{
+ timer_add(&evmutex, &watchdog, &watchdog_tv);
+}
+
/*
* ratelimit
*
@@ -54,11 +71,10 @@ static void com_rcv(struct ns8250_dev *,
static void
ratelimit(int fd, short type, void *arg)
{
- /* Set TXRDY and clear "no pending interrupt" */
- com1_dev.regs.iir |= IIR_TXRDY;
- com1_dev.regs.iir &= ~IIR_NOPEND;
vcpu_assert_pic_irq(com1_dev.vmid, 0, com1_dev.irq);
vcpu_deassert_pic_irq(com1_dev.vmid, 0, com1_dev.irq);
+
+ timer_add(&ratelimit_mutex, &com1_dev.rate, &com1_dev.rate_tv);
}
void
@@ -66,12 +82,28 @@ ns8250_init(int fd, uint32_t vmid)
{
int ret;
+ ret = pthread_mutex_init(&evmutex, NULL);
+ if (ret) {
+ errno = ret;
+ fatal("could not initialize ns8250 event mutex");
+ }
+
+ evbase = event_base_new();
+ ratelimit_evbase = event_base_new();
+
memset(&com1_dev, 0, sizeof(com1_dev));
ret = pthread_mutex_init(&com1_dev.mutex, NULL);
if (ret) {
errno = ret;
fatal("could not initialize com1 mutex");
}
+
+ ret = pthread_mutex_init(&ratelimit_mutex, NULL);
+ if (ret) {
+ errno = ret;
+ fatal("could not initialize ratelimit mutex");
+ }
+
com1_dev.fd = fd;
com1_dev.irq = 4;
com1_dev.portid = NS8250_COM1;
@@ -98,6 +130,7 @@ ns8250_init(int fd, uint32_t vmid)
event_set(&com1_dev.event, com1_dev.fd, EV_READ | EV_PERSIST,
com_rcv_event, (void *)(intptr_t)vmid);
+ event_base_set(evbase, &com1_dev.event);
/*
* Whenever fd is writable implies that the pty slave is connected.
@@ -106,12 +139,46 @@ ns8250_init(int fd, uint32_t vmid)
*/
event_set(&com1_dev.wake, com1_dev.fd, EV_WRITE,
com_rcv_event, (void *)(intptr_t)vmid);
- event_add(&com1_dev.wake, NULL);
+ event_base_set(evbase, &com1_dev.wake);
+ ev_add(&evmutex, &com1_dev.event, NULL);
/* Rate limiter for simulating baud rate */
timerclear(&com1_dev.rate_tv);
com1_dev.rate_tv.tv_usec = 10000;
evtimer_set(&com1_dev.rate, ratelimit, NULL);
+ event_base_set(ratelimit_evbase, &com1_dev.rate);
+ evtimer_add(&com1_dev.rate, &com1_dev.rate_tv);
+
+ /* Watchdog for keeping the event pump running */
+ timerclear(&watchdog_tv);
+ watchdog_tv.tv_sec = 5;
+ evtimer_set(&watchdog, ns8250_watchdog, NULL);
+ event_base_set(evbase, &watchdog);
+ evtimer_add(&watchdog, &watchdog_tv);
+}
+
+static void
+init_ratelimit_thread(void)
+{
+ int ret;
+ log_debug("%s: starting ratelimit thread with evbase: %p", __func__, ratelimit_evbase);
+ ret = pthread_create(&ratelimit_thread, NULL, event_loop_thread, ratelimit_evbase);
+ if (ret) {
+ fatal("%s: failed to start ratelimit thread: %d", __func__, ret);
+ }
+}
+
+void
+ns8250_init_thread(void)
+{
+ int ret;
+
+ log_debug("%s: starting thread with evbase %p", __func__, evbase);
+ ret = pthread_create(&ns8250_thread, NULL, event_loop_thread, evbase);
+ if (ret) {
+ fatal("%s: failed to start ns8250 thread: %d", __func__, ret);
+ }
+ init_ratelimit_thread();
}
static void
@@ -120,7 +187,7 @@ com_rcv_event(int fd, short kind, void *
mutex_lock(&com1_dev.mutex);
if (kind == EV_WRITE) {
- event_add(&com1_dev.event, NULL);
+ ev_add(&evmutex, &com1_dev.event, NULL);
mutex_unlock(&com1_dev.mutex);
return;
}
@@ -201,8 +268,8 @@ com_rcv(struct ns8250_dev *com, uint32_t
if (errno != EAGAIN)
log_warn("unexpected read error on com device");
} else if (sz == 0) {
- event_del(&com->event);
- event_add(&com->wake, NULL);
+ ev_del(&evmutex, &com->event);
+ ev_add(&evmutex, &com->wake, NULL);
return;
} else if (sz != 1 && sz != 2)
log_warnx("unexpected read return value %zd on com device", sz);
@@ -255,16 +322,8 @@ vcpu_process_com_data(struct vm_exit *ve
com1_dev.byte_out++;
if (com1_dev.regs.ier & IER_ETXRDY) {
- /* Limit output rate if needed */
- if (com1_dev.pause_ct > 0 &&
- com1_dev.byte_out % com1_dev.pause_ct == 0) {
- evtimer_add(&com1_dev.rate,
- &com1_dev.rate_tv);
- } else {
- /* Set TXRDY and clear "no pending interrupt" */
- com1_dev.regs.iir |= IIR_TXRDY;
- com1_dev.regs.iir &= ~IIR_NOPEND;
- }
+ com1_dev.regs.iir |= IIR_TXRDY;
+ com1_dev.regs.iir &= ~IIR_NOPEND;
}
} else {
if (com1_dev.regs.lcr & LCR_DLAB) {
@@ -654,15 +713,15 @@ ns8250_restore(int fd, int con_fd, uint3
com1_dev.byte_out = 0;
com1_dev.regs.divlo = 1;
com1_dev.baudrate = 115200;
- com1_dev.rate_tv.tv_usec = 10000;
com1_dev.pause_ct = (com1_dev.baudrate / 8) / 1000 * 10;
- evtimer_set(&com1_dev.rate, ratelimit, NULL);
event_set(&com1_dev.event, com1_dev.fd, EV_READ | EV_PERSIST,
com_rcv_event, (void *)(intptr_t)vmid);
+ event_base_set(evbase, &com1_dev.event);
event_set(&com1_dev.wake, com1_dev.fd, EV_WRITE,
com_rcv_event, (void *)(intptr_t)vmid);
+ event_base_set(evbase, &com1_dev.wake);
return (0);
}
@@ -670,15 +729,19 @@ ns8250_restore(int fd, int con_fd, uint3
void
ns8250_stop()
{
- if(event_del(&com1_dev.event))
+ if (ev_del(&evmutex, &com1_dev.event))
log_warn("could not delete ns8250 event handler");
- evtimer_del(&com1_dev.rate);
+ ev_del(&ratelimit_mutex, &com1_dev.rate);
+
+ // Safely abort the ratelimiter thread by closing the event base
+ event_base_loopexit(ratelimit_evbase, NULL);
}
void
ns8250_start()
{
- event_add(&com1_dev.event, NULL);
- event_add(&com1_dev.wake, NULL);
+ ev_add(&evmutex, &com1_dev.event, NULL);
+ ev_add(&evmutex, &com1_dev.wake, NULL);
evtimer_add(&com1_dev.rate, &com1_dev.rate_tv);
+ init_ratelimit_thread();
}
Index: ns8250.h
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/ns8250.h,v
retrieving revision 1.9
diff -u -p -u -p -r1.9 ns8250.h
--- ns8250.h 11 Dec 2019 06:45:16 -0000 1.9
+++ ns8250.h 2 Jun 2020 12:43:26 -0000
@@ -63,8 +63,8 @@ struct ns8250_dev {
pthread_mutex_t mutex;
struct ns8250_regs regs;
struct event event;
- struct event rate;
struct event wake;
+ struct event rate;
struct timeval rate_tv;
enum ns8250_portid portid;
int fd;
@@ -77,6 +77,7 @@ struct ns8250_dev {
};
void ns8250_init(int, uint32_t);
+void ns8250_init_thread(void);
uint8_t vcpu_exit_com(struct vm_run_params *);
uint8_t vcpu_process_com_data(struct vm_exit *, uint32_t, uint32_t);
void vcpu_process_com_lcr(struct vm_exit *);
Index: priv.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/priv.c,v
retrieving revision 1.15
diff -u -p -u -p -r1.15 priv.c
--- priv.c 28 Jun 2019 13:32:51 -0000 1.15
+++ priv.c 2 Jun 2020 12:43:26 -0000
@@ -46,6 +46,8 @@
#include "proc.h"
#include "vmd.h"
+extern struct event_base *global_evbase;
+
int priv_dispatch_parent(int, struct privsep_proc *, struct imsg *);
void priv_run(struct privsep *, struct privsep_proc *, void *);
Index: proc.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/proc.c,v
retrieving revision 1.18
diff -u -p -u -p -r1.18 proc.c
--- proc.c 10 Sep 2018 10:36:01 -0000 1.18
+++ proc.c 2 Jun 2020 12:43:26 -0000
@@ -30,12 +30,15 @@
#include <errno.h>
#include <signal.h>
#include <paths.h>
+#include <pthread.h>
#include <pwd.h>
#include <event.h>
#include <imsg.h>
#include "proc.h"
+extern struct event_base *global_evbase;
+
void proc_exec(struct privsep *, struct privsep_proc *, unsigned int, int,
int, char **);
void proc_setup(struct privsep *, struct privsep_proc *, unsigned int);
@@ -158,6 +161,7 @@ proc_exec(struct privsep *ps, struct pri
default:
/* Close child end. */
close(fd);
+ event_reinit(global_evbase);
break;
}
}
@@ -185,6 +189,7 @@ proc_connect(struct privsep *ps)
imsg_init(&iev->ibuf, ps->ps_pp->pp_pipes[dst][inst]);
event_set(&iev->ev, iev->ibuf.fd, iev->events,
iev->handler, iev->data);
+ event_base_set(global_evbase, &iev->ev);
event_add(&iev->ev, NULL);
}
}
@@ -290,6 +295,7 @@ proc_accept(struct privsep *ps, int fd,
iev = &ps->ps_ievs[dst][n];
imsg_init(&iev->ibuf, fd);
event_set(&iev->ev, iev->ibuf.fd, iev->events, iev->handler, iev->data);
+ event_base_set(global_evbase, &iev->ev);
event_add(&iev->ev, NULL);
}
@@ -566,14 +572,21 @@ proc_run(struct privsep *ps, struct priv
setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid))
fatal("%s: cannot drop privileges", __func__);
- event_init();
+ //event_init();
+ //global_evbase = event_base_new();
signal_set(&ps->ps_evsigint, SIGINT, proc_sig_handler, p);
+ event_base_set(global_evbase, &ps->ps_evsigint);
signal_set(&ps->ps_evsigterm, SIGTERM, proc_sig_handler, p);
+ event_base_set(global_evbase, &ps->ps_evsigterm);
signal_set(&ps->ps_evsigchld, SIGCHLD, proc_sig_handler, p);
+ event_base_set(global_evbase, &ps->ps_evsigchld);
signal_set(&ps->ps_evsighup, SIGHUP, proc_sig_handler, p);
+ event_base_set(global_evbase, &ps->ps_evsighup);
signal_set(&ps->ps_evsigpipe, SIGPIPE, proc_sig_handler, p);
+ event_base_set(global_evbase, &ps->ps_evsigpipe);
signal_set(&ps->ps_evsigusr1, SIGUSR1, proc_sig_handler, p);
+ event_base_set(global_evbase, &ps->ps_evsigusr1);
signal_add(&ps->ps_evsigint, NULL);
signal_add(&ps->ps_evsigterm, NULL);
@@ -599,7 +612,7 @@ proc_run(struct privsep *ps, struct priv
if (run != NULL)
run(ps, p, arg);
- event_dispatch();
+ event_base_dispatch(global_evbase);
proc_shutdown(p);
}
@@ -626,7 +639,7 @@ proc_dispatch(int fd, short event, void
if (n == 0) {
/* this pipe is dead, so remove the event handler */
event_del(&iev->ev);
- event_loopexit(NULL);
+ event_base_loopexit(global_evbase, NULL);
return;
}
}
@@ -637,7 +650,7 @@ proc_dispatch(int fd, short event, void
if (n == 0) {
/* this pipe is dead, so remove the event handler */
event_del(&iev->ev);
- event_loopexit(NULL);
+ event_base_loopexit(global_evbase, NULL);
return;
}
}
@@ -714,6 +727,7 @@ imsg_event_add(struct imsgev *iev)
event_del(&iev->ev);
event_set(&iev->ev, iev->ibuf.fd, iev->events, iev->handler, iev->data);
+ event_base_set(global_evbase, &iev->ev);
event_add(&iev->ev, NULL);
}
Index: safe_event.c
===================================================================
RCS file: safe_event.c
diff -N safe_event.c
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ safe_event.c 2 Jun 2020 12:43:26 -0000
@@ -0,0 +1,68 @@
+#include <event.h>
+#include <pthread.h>
+
+#include "proc.h"
+#include "safe_event.h"
+
+int
+ev_add(pthread_mutex_t *mutex, struct event *ev, struct timeval *tv)
+{
+ int ret;
+ pthread_mutex_lock(mutex);
+ ret = event_add(ev, tv);
+ pthread_mutex_unlock(mutex);
+ return ret;
+}
+
+int
+ev_del(pthread_mutex_t *mutex,struct event *ev)
+{
+ int ret;
+ pthread_mutex_lock(mutex);
+ ret = event_del(ev);
+ pthread_mutex_unlock(mutex);
+ return ret;
+}
+
+void
+timer_add(pthread_mutex_t *mutex, struct event *ev, struct timeval *tv)
+{
+ pthread_mutex_lock(mutex);
+ evtimer_add(ev, tv);
+ pthread_mutex_unlock(mutex);
+}
+
+void
+timer_del(pthread_mutex_t *mutex, struct event *ev)
+{
+ pthread_mutex_lock(mutex);
+ evtimer_del(ev);
+ pthread_mutex_unlock(mutex);
+}
+
+int
+timer_pending(pthread_mutex_t *mutex, struct event *ev, struct timeval *tv)
+{
+ int ret;
+ pthread_mutex_lock(mutex);
+ ret = evtimer_pending(ev, tv);
+ pthread_mutex_unlock(mutex);
+ return ret;
+}
+
+void *
+event_loop_thread(void *arg)
+{
+ struct event_base *evbase;
+ intptr_t ret;
+ int r;
+
+ evbase = arg;
+
+ r = event_base_dispatch(evbase);
+ ret = r;
+
+ log_info("%s: event loop for evbase %p exiting (r=%d)",
+ __func__, evbase, r);
+ return (void *)ret;
+}
Index: safe_event.h
===================================================================
RCS file: safe_event.h
diff -N safe_event.h
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ safe_event.h 2 Jun 2020 12:43:26 -0000
@@ -0,0 +1,6 @@
+int ev_add(pthread_mutex_t *, struct event *, struct timeval *tv);
+int ev_del(pthread_mutex_t *, struct event *);
+void timer_add(pthread_mutex_t *, struct event *, struct timeval *tv);
+void timer_del(pthread_mutex_t *, struct event *);
+int timer_pending(pthread_mutex_t *, struct event *, struct timeval *);
+void *event_loop_thread(void *);
Index: virtio.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/virtio.c,v
retrieving revision 1.82
diff -u -p -u -p -r1.82 virtio.c
--- virtio.c 11 Dec 2019 06:45:16 -0000 1.82
+++ virtio.c 2 Jun 2020 12:43:26 -0000
@@ -34,6 +34,7 @@
#include <errno.h>
#include <event.h>
#include <poll.h>
+#include <pthread.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
@@ -46,6 +47,7 @@
#include "vioscsi.h"
#include "loadfile.h"
#include "atomicio.h"
+#include "safe_event.h"
extern char *__progname;
struct viornd_dev viornd;
@@ -54,6 +56,12 @@ struct vionet_dev *vionet;
struct vioscsi_dev *vioscsi;
struct vmmci_dev vmmci;
+static struct event_base *evbase;
+static struct event watchdog;
+static struct timeval watchdog_tv;
+static pthread_t virtio_thread;
+static pthread_mutex_t evmutex;
+
int nr_vionet;
int nr_vioblk;
@@ -68,6 +76,12 @@ int nr_vioblk;
#define RXQ 0
#define TXQ 1
+static void
+virtio_watchdog(int fd, short type, void *arg)
+{
+ timer_add(&evmutex, &watchdog, &watchdog_tv);
+}
+
const char *
vioblk_cmd_name(uint32_t type)
{
@@ -1587,7 +1601,7 @@ vmmci_ctl(unsigned int cmd)
/* Add ACK timeout */
tv.tv_sec = VMMCI_TIMEOUT;
- evtimer_add(&vmmci.timeout, &tv);
+ timer_add(&evmutex, &vmmci.timeout, &tv);
break;
case VMMCI_SYNCRTC:
if (vmmci.cfg.guest_feature & VMMCI_F_SYNCRTC) {
@@ -1627,7 +1641,7 @@ vmmci_ack(unsigned int cmd)
log_debug("%s: vm %u requested shutdown", __func__,
vmmci.vm_id);
tv.tv_sec = VMMCI_TIMEOUT;
- evtimer_add(&vmmci.timeout, &tv);
+ timer_add(&evmutex, &vmmci.timeout, &tv);
return;
}
/* FALLTHROUGH */
@@ -1640,11 +1654,11 @@ vmmci_ack(unsigned int cmd)
* killing it forcefully.
*/
if (cmd == vmmci.cmd &&
- evtimer_pending(&vmmci.timeout, NULL)) {
+ timer_pending(&evmutex, &vmmci.timeout, NULL)) {
log_debug("%s: vm %u acknowledged shutdown request",
__func__, vmmci.vm_id);
tv.tv_sec = VMMCI_SHUTDOWN_TIMEOUT;
- evtimer_add(&vmmci.timeout, &tv);
+ timer_add(&evmutex, &vmmci.timeout, &tv);
}
break;
case VMMCI_SYNCRTC:
@@ -1792,6 +1806,13 @@ virtio_init(struct vmd_vm *vm, int child
uint8_t i;
int ret;
+ evbase = event_base_new();
+
+ ret = pthread_mutex_init(&evmutex, NULL);
+ if (ret) {
+ fatal("%s: failed to create pthread_mutex", __func__);
+ }
+
/* Virtio entropy device */
if (pci_add_device(&id, PCI_VENDOR_QUMRANET,
PCI_PRODUCT_QUMRANET_VIO_RNG, PCI_CLASS_SYSTEM,
@@ -1879,7 +1900,8 @@ virtio_init(struct vmd_vm *vm, int child
event_set(&vionet[i].event, vionet[i].fd,
EV_READ | EV_PERSIST, vionet_rx_event, &vionet[i]);
- if (event_add(&vionet[i].event, NULL)) {
+ event_base_set(evbase, &vionet[i].event);
+ if (ev_add(&evmutex, &vionet[i].event, NULL)) {
log_warn("could not initialize vionet event "
"handler");
return;
@@ -2033,6 +2055,26 @@ virtio_init(struct vmd_vm *vm, int child
vmmci.pci_id = id;
evtimer_set(&vmmci.timeout, vmmci_timeout, NULL);
+ event_base_set(evbase, &vmmci.timeout);
+
+ /* Watchdog setup */
+ timerclear(&watchdog_tv);
+ watchdog_tv.tv_sec = 30;
+ evtimer_set(&watchdog, virtio_watchdog, NULL);
+ event_base_set(evbase, &watchdog);
+ timer_add(&evmutex, &watchdog, &watchdog_tv);
+}
+
+void
+virtio_init_thread(void)
+{
+ int ret;
+
+ log_debug("%s: starting thread with evbase %p", __func__, evbase);
+ ret = pthread_create(&virtio_thread, NULL, event_loop_thread, evbase);
+ if (ret) {
+ fatal("%s: could not create thread", __func__);
+ }
}
void
@@ -2066,6 +2108,7 @@ vmmci_restore(int fd, uint32_t vm_id)
vmmci.irq = pci_get_dev_irq(vmmci.pci_id);
memset(&vmmci.timeout, 0, sizeof(struct event));
evtimer_set(&vmmci.timeout, vmmci_timeout, NULL);
+ event_base_set(evbase, &vmmci.timeout);
return (0);
}
@@ -2096,6 +2139,8 @@ vionet_restore(int fd, struct vmd_vm *vm
uint8_t i;
int ret;
+ evbase = event_base_new();
+
nr_vionet = vcp->vcp_nnics;
if (vcp->vcp_nnics > 0) {
vionet = calloc(vcp->vcp_nnics, sizeof(struct vionet_dev));
@@ -2140,6 +2185,7 @@ vionet_restore(int fd, struct vmd_vm *vm
memset(&vionet[i].event, 0, sizeof(struct event));
event_set(&vionet[i].event, vionet[i].fd,
EV_READ | EV_PERSIST, vionet_rx_event, &vionet[i]);
+ event_base_set(evbase, &vionet[i].event);
}
}
return (0);
@@ -2339,7 +2385,7 @@ virtio_stop(struct vm_create_params *vcp
{
uint8_t i;
for (i = 0; i < vcp->vcp_nnics; i++) {
- if (event_del(&vionet[i].event)) {
+ if (ev_del(&evmutex, &vionet[i].event)) {
log_warn("could not initialize vionet event "
"handler");
return;
@@ -2352,7 +2398,7 @@ virtio_start(struct vm_create_params *vc
{
uint8_t i;
for (i = 0; i < vcp->vcp_nnics; i++) {
- if (event_add(&vionet[i].event, NULL)) {
+ if (ev_add(&evmutex, &vionet[i].event, NULL)) {
log_warn("could not initialize vionet event "
"handler");
return;
Index: virtio.h
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/virtio.h,v
retrieving revision 1.35
diff -u -p -u -p -r1.35 virtio.h
--- virtio.h 11 Dec 2019 06:45:16 -0000 1.35
+++ virtio.h 2 Jun 2020 12:43:26 -0000
@@ -261,6 +261,7 @@ struct ioinfo {
/* virtio.c */
void virtio_init(struct vmd_vm *, int, int[][VM_MAX_BASE_PER_DISK], int *);
+void virtio_init_thread(void);
void virtio_shutdown(struct vmd_vm *);
int virtio_dump(int);
int virtio_restore(int, struct vmd_vm *, int,
Index: vm.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/vm.c,v
retrieving revision 1.57
diff -u -p -u -p -r1.57 vm.c
--- vm.c 30 Apr 2020 03:50:53 -0000 1.57
+++ vm.c 2 Jun 2020 12:43:26 -0000
@@ -107,6 +107,9 @@ extern struct vmd *env;
extern char *__progname;
+extern struct event_base *global_evbase;
+extern pthread_mutex_t global_evmutex;
+
pthread_mutex_t threadmutex;
pthread_cond_t threadcond;
@@ -364,8 +367,6 @@ start_vm(struct vmd_vm *vm, int fd)
for (i = 0; i < VMM_MAX_NICS_PER_VM; i++)
nicfds[i] = vm->vm_ifs[i].vif_fd;
- event_init();
-
if (vm->vm_state & VM_STATE_RECEIVED) {
restore_emulated_hw(vcp, vm->vm_receive_fd, nicfds,
vm->vm_disks, vm->vm_cdrom);
@@ -403,6 +404,8 @@ vm_dispatch_vmm(int fd, short event, voi
ssize_t n;
int verbose;
+ pthread_mutex_lock(&global_evmutex);
+
if (event & EV_READ) {
if ((n = imsg_read(ibuf)) == -1 && errno != EAGAIN)
fatal("%s: imsg_read", __func__);
@@ -424,7 +427,7 @@ vm_dispatch_vmm(int fd, short event, voi
break;
#if DEBUG > 1
- log_debug("%s: got imsg %d from %s",
+ log_info("%s: got imsg %d from %s",
__func__, imsg.hdr.type,
vm->vm_params.vmc_params.vcp_name);
#endif
@@ -482,6 +485,8 @@ vm_dispatch_vmm(int fd, short event, voi
imsg_free(&imsg);
}
imsg_event_add(iev);
+
+ pthread_mutex_unlock(&global_evmutex);
}
/*
@@ -492,6 +497,8 @@ vm_dispatch_vmm(int fd, short event, voi
__dead void
vm_shutdown(unsigned int cmd)
{
+ pthread_mutex_lock(&global_evmutex);
+
switch (cmd) {
case VMMCI_NONE:
case VMMCI_SHUTDOWN:
@@ -507,6 +514,8 @@ vm_shutdown(unsigned int cmd)
}
imsg_flush(&current_vm->vm_iev.ibuf);
+ pthread_mutex_unlock(&global_evmutex);
+
_exit(0);
}
@@ -777,9 +786,13 @@ pause_vm(struct vm_create_params *vcp)
}
i8253_stop();
+ log_debug("%s: i8253 stopped", __func__);
mc146818_stop();
+ log_debug("%s: mc14618 stopped", __func__);
ns8250_stop();
+ log_debug("%s: ns8250 stopped", __func__);
virtio_stop(vcp);
+ log_debug("%s: virtio stopped", __func__);
}
void
@@ -800,10 +813,14 @@ unpause_vm(struct vm_create_params *vcp)
}
}
- i8253_start();
- mc146818_start();
- ns8250_start();
virtio_start(vcp);
+ log_debug("%s: virtio restarted", __func__);
+ ns8250_start();
+ log_debug("%s: ns8250 restarted", __func__);
+ mc146818_start();
+ log_debug("%s: mc146818 restarted", __func__);
+ i8253_start();
+ log_debug("%s: i8253 restarted", __func__);
}
/*
@@ -1300,6 +1317,14 @@ run_vm(int child_cdrom, int child_disks[
return (ret);
}
+ /* XXX: this logic could move elsewhere...need some initial events
+ * configured before we start the event loops.
+ */
+ i8253_init_thread();
+ mc146818_init_thread();
+ virtio_init_thread();
+ ns8250_init_thread();
+
for (;;) {
ret = pthread_cond_wait(&threadcond, &threadmutex);
if (ret) {
@@ -1357,7 +1382,7 @@ event_thread(void *arg)
uint8_t *donep = arg;
intptr_t ret;
- ret = event_dispatch();
+ ret = event_base_dispatch(global_evbase);
mutex_lock(&threadmutex);
*donep = 1;
Index: vmd.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/vmd.c,v
retrieving revision 1.117
diff -u -p -u -p -r1.117 vmd.c
--- vmd.c 12 Dec 2019 03:53:38 -0000 1.117
+++ vmd.c 2 Jun 2020 12:43:27 -0000
@@ -33,6 +33,7 @@
#include <errno.h>
#include <event.h>
#include <fcntl.h>
+#include <pthread.h>
#include <pwd.h>
#include <signal.h>
#include <syslog.h>
@@ -75,6 +76,8 @@ static struct privsep_proc procs[] = {
{ "vmm", PROC_VMM, vmd_dispatch_vmm, vmm, vmm_shutdown },
};
+pthread_mutex_t global_evmutex;
+struct event_base *global_evbase;
struct event staggered_start_timer;
/* For the privileged process */
@@ -820,6 +823,12 @@ main(int argc, char **argv)
if (title != NULL)
ps->ps_title[proc_id] = title;
+ global_evbase = event_base_new();
+
+ if (pthread_mutex_init(&global_evmutex, NULL)) {
+ fatal("can't initialize global pthread_mutex");
+ }
+
/* only the parent returns */
proc_init(ps, procs, nitems(procs), env->vmd_debug, argc0, argv,
proc_id);
@@ -831,13 +840,16 @@ main(int argc, char **argv)
if (ps->ps_noaction == 0)
log_info("startup");
- event_init();
-
signal_set(&ps->ps_evsigint, SIGINT, vmd_sighdlr, ps);
+ event_base_set(global_evbase, &ps->ps_evsigint);
signal_set(&ps->ps_evsigterm, SIGTERM, vmd_sighdlr, ps);
+ event_base_set(global_evbase, &ps->ps_evsigterm);
signal_set(&ps->ps_evsighup, SIGHUP, vmd_sighdlr, ps);
+ event_base_set(global_evbase, &ps->ps_evsighup);
signal_set(&ps->ps_evsigpipe, SIGPIPE, vmd_sighdlr, ps);
+ event_base_set(global_evbase, &ps->ps_evsigpipe);
signal_set(&ps->ps_evsigusr1, SIGUSR1, vmd_sighdlr, ps);
+ event_base_set(global_evbase, &ps->ps_evsigusr1);
signal_add(&ps->ps_evsigint, NULL);
signal_add(&ps->ps_evsigterm, NULL);
@@ -851,7 +863,7 @@ main(int argc, char **argv)
if (vmd_configure() == -1)
fatalx("configuration failed");
- event_dispatch();
+ event_base_dispatch(global_evbase);
log_debug("parent exiting");
@@ -951,6 +963,8 @@ vmd_configure(void)
log_debug("%s: starting vms in staggered fashion", __func__);
evtimer_set(&staggered_start_timer, start_vm_batch, NULL);
+ event_base_set(global_evbase, &staggered_start_timer);
+
/* start first batch */
start_vm_batch(0, 0, NULL);
@@ -1021,6 +1035,8 @@ vmd_reload(unsigned int reset, const cha
log_debug("%s: starting vms in staggered fashion", __func__);
evtimer_set(&staggered_start_timer, start_vm_batch, NULL);
+ event_base_set(global_evbase, &staggered_start_timer);
+
/* start first batch */
start_vm_batch(0, 0, NULL);
Index: vmm.c
===================================================================
RCS file: /cvs/src/usr.sbin/vmd/vmm.c,v
retrieving revision 1.96
diff -u -p -u -p -r1.96 vmm.c
--- vmm.c 30 Apr 2020 03:50:53 -0000 1.96
+++ vmm.c 2 Jun 2020 12:43:27 -0000
@@ -64,6 +64,9 @@ int opentap(char *);
extern struct vmd *env;
+extern pthread_mutex_t global_evmutex;
+extern struct event_base *global_evbase;
+
static struct privsep_proc procs[] = {
{ "parent", PROC_PARENT, vmm_dispatch_parent },
};
@@ -80,9 +83,12 @@ vmm_run(struct privsep *ps, struct privs
if (config_init(ps->ps_env) == -1)
fatal("failed to initialize configuration");
+ pthread_mutex_lock(&global_evmutex);
signal_del(&ps->ps_evsigchld);
signal_set(&ps->ps_evsigchld, SIGCHLD, vmm_sighdlr, ps);
+ event_base_set(global_evbase, &ps->ps_evsigchld);
signal_add(&ps->ps_evsigchld, NULL);
+ pthread_mutex_unlock(&global_evmutex);
/*
* pledge in the vmm process:
@@ -198,7 +204,7 @@ vmm_dispatch_parent(int fd, struct privs
/*
* Request reboot but mark the VM as shutting
* down. This way we can terminate the VM after
- * the triple fault instead of reboot and
+ * the triple fault instead of reboot and
* avoid being stuck in the ACPI-less powerdown
* ("press any key to reboot") of the VM.
*/
@@ -709,6 +715,8 @@ vmm_start_vm(struct imsg *imsg, uint32_t
/* Child */
close(fds[0]);
close(PROC_PARENT_SOCK_FILENO);
+
+ event_reinit(global_evbase);
ret = start_vm(vm, fds[1]);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment