Dynamic-tick-support-v2.patch

Signed-off-by: OZAWA Tsuyoshi <ozawa _at_ t-oza.net>
---
 sys/i386/i386/local_apic.c |  132 ++++++++++++++++++++++++++++++++++++++++++++
 sys/kern/kern_clock.c      |   27 +++++++++
 sys/kern/kern_timeout.c    |   30 ++++++++++
 sys/kern/sched_4bsd.c      |    3 +
 sys/kern/sched_ule.c       |    8 ++-
 sys/sys/dynticks.h         |   34 +++++++++++
 sys/sys/systm.h            |    1 +
 7 files changed, 233 insertions(+), 2 deletions(-)
 create mode 100644 sys/sys/dynticks.h

diff --git a/sys/i386/i386/local_apic.c b/sys/i386/i386/local_apic.c
index 10a5854..2c216f4 100644
--- a/sys/i386/i386/local_apic.c
+++ b/sys/i386/i386/local_apic.c
@@ -68,6 +68,8 @@ __FBSDID("$FreeBSD: src/sys/i386/i386/local_apic.c,v 1.61.2.4.2.1 2009/10/25 01:
 #include <ddb/ddb.h>
 #endif
 
+#include <sys/dynticks.h>
+
 #ifdef KDTRACE_HOOKS
 #include <sys/dtrace_bsd.h>
 cyclic_clock_func_t	lapic_cyclic_clock_func[MAXCPU];
@@ -111,6 +113,9 @@ struct lapic {
 	u_long la_hard_ticks;
 	u_long la_stat_ticks;
 	u_long la_prof_ticks;
+	u_long la_last_tick;
+	u_long la_cur_skip;
+	u_long la_skip;
 	/* Include IDT_SYSCALL to make indexing easier. */
 	int la_ioint_irqs[APIC_NUM_IOINTS + 1];
 } static lapics[MAX_APIC_ID + 1];
@@ -144,6 +149,9 @@ static u_int32_t lapic_timer_divisors[] = {
 	APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
 };
 
+
+static void (*timer_handler)(struct trapframe *frame);
+
 extern inthand_t IDTVEC(rsvd);
 
 volatile lapic_t *lapic;
@@ -158,6 +166,9 @@ static void	lapic_timer_periodic(u_int count);
 static void	lapic_timer_set_divisor(u_int divisor);
 static uint32_t	lvt_mode(struct lapic *la, u_int pin, uint32_t value);
 
+static void	lapic_handle_timer_dynamically(struct trapframe *frame);
+static void	__lapic_handle_timer(struct trapframe *frame);
+
 struct pic lapic_pic = { .pic_resume = lapic_resume };
 
 static uint32_t
@@ -216,6 +227,8 @@ lapic_init(vm_paddr_t addr)
 	setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_SYS386IGT, SEL_KPL,
 	    GSEL(GCODE_SEL, SEL_KPL));
 
+	timer_handler = __lapic_handle_timer;
+
 	/* Perform basic initialization of the BSP's local APIC. */
 	lapic_enable();
 
@@ -740,6 +753,12 @@ lapic_handle_intr(int vector, struct trapframe *frame)
 void
 lapic_handle_timer(struct trapframe *frame)
 {
+	timer_handler(frame);
+}
+
+static void
+__lapic_handle_timer(struct trapframe *frame)
+{
 	struct lapic *la;
 
 	/* Send EOI first thing. */
@@ -801,6 +820,10 @@ lapic_handle_timer(struct trapframe *frame)
 		if (profprocs != 0)
 			profclock(TRAPF_USERMODE(frame), TRAPF_PC(frame));
 	}
+
+	la->la_cur_skip = 0;
+	la->la_skip = 1;
+
 	critical_exit();
 }
 
@@ -1398,3 +1421,112 @@ lapic_ipi_vectored(u_int vector, int dest)
 #endif /* DETECT_DEADLOCK */
 }
 #endif /* SMP */
+
+static void set_next_timer_interrupt(void)
+{
+	struct lapic *la;
+	int skip;
+	u_long cnt_to_skip;
+
+	la = &lapics[PCPU_GET(apic_id)];
+	(*la->la_timer_count)++;
+	
+	skip = callout_get_next_event();
+	cnt_to_skip = lapic_timer_period * skip ;
+	lapic_timer_oneshot(cnt_to_skip);
+	la->la_skip = skip;
+	la->la_cur_skip = 0;
+
+	return;
+}
+
+static void
+lapic_handle_timer_dynamically(struct trapframe *frame)
+{
+	struct lapic *la;
+	int skip;
+	int i;
+
+	/* Send EOI first thing. */
+	lapic_eoi();
+
+#if defined(SMP) && !defined(SCHED_ULE)
+	/*
+	 * Don't do any accounting for the disabled HTT cores, since it
+	 * will provide misleading numbers for the userland.
+	 *
+	 * No locking is necessary here, since even if we loose the race
+	 * when hlt_cpus_mask changes it is not a big deal, really.
+	 *
+	 * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask
+	 * and unlike other schedulers it actually schedules threads to
+	 * those CPUs.
+	 */
+	if ((hlt_cpus_mask & (1 << PCPU_GET(cpuid))) != 0)
+		return;
+#endif
+
+	/* Look up our local APIC structure for the tick counters. */
+	la = &lapics[PCPU_GET(apic_id)];
+	(*la->la_timer_count)++;
+	critical_enter();
+
+#ifdef KDTRACE_HOOKS
+	/*
+	 * If the DTrace hooks are configured and a callback function
+	 * has been registered, then call it to process the high speed
+	 * timers.
+	 */
+	int cpu = PCPU_GET(cpuid);
+	/* i dont know this works well? */
+	if (lapic_cyclic_clock_func[cpu] != NULL)
+		(*lapic_cyclic_clock_func[cpu])(frame);
+#endif
+	/* Fire hardclock at hz. */
+	skip = la->la_skip;
+	for(i = 0; i < skip; i++){
+		la->la_hard_ticks += hz;
+		if (la->la_hard_ticks >= lapic_timer_hz) {
+			la->la_hard_ticks -= lapic_timer_hz;
+			if (PCPU_GET(cpuid) == 0)
+				hardclock(TRAPF_USERMODE(frame), TRAPF_PC(frame));
+			else
+				hardclock_cpu(TRAPF_USERMODE(frame));
+		}
+
+		/* Fire statclock at stathz. */
+		la->la_stat_ticks += stathz;
+		if (la->la_stat_ticks >= lapic_timer_hz) {
+			la->la_stat_ticks -= lapic_timer_hz;
+			statclock(TRAPF_USERMODE(frame));
+		}
+
+		/* Fire profclock at profhz, but only when needed. */
+		la->la_prof_ticks += profhz;
+		if (la->la_prof_ticks >= lapic_timer_hz) {
+			if (profprocs != 0){
+				la->la_prof_ticks -= lapic_timer_hz;
+				profclock(TRAPF_USERMODE(frame), TRAPF_PC(frame));
+			}
+		}
+	}
+
+	set_next_timer_interrupt();
+	critical_exit();
+}
+
+void switch_to_dynticks(void)
+{
+	critical_enter();
+	timer_handler = lapic_handle_timer_dynamically;
+	set_next_timer_interrupt();
+	critical_exit();
+}
+
+void switch_to_perticks(void)
+{
+	critical_enter();
+	timer_handler = __lapic_handle_timer;
+	lapic_timer_periodic(lapic_timer_period);
+	critical_exit();
+}
diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index 391df1a..7110117 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -323,6 +323,33 @@ hardclock(int usermode, uintfptr_t pc)
 #endif /* SW_WATCHDOG */
 }
 
+void
+hardclock_dynticks(int usermode, uintfptr_t pc,int skip)
+{
+	//int i;
+
+	//atomic_add_int((volatile int *)&ticks, skip); // FIXME
+	atomic_add_int((volatile int *)&ticks, 1);
+	hardclock_cpu(usermode);
+	tc_ticktock();
+	/*
+	 * If no separate statistics clock is available, run it from here.
+	 *
+	 * XXX: this only works for UP
+	 */
+	if (stathz == 0) {
+		profclock(usermode, pc);
+		statclock(usermode);
+	}
+#ifdef DEVICE_POLLING
+	hardclock_device_poll();	/* this is very short and quick */
+#endif /* DEVICE_POLLING */
+#ifdef SW_WATCHDOG
+	if (watchdog_enabled > 0 && (watchdog_ticks -= skip ) <= 0)
+		watchdog_fire();
+#endif /* SW_WATCHDOG */
+}
+
 /*
  * Compute number of ticks in the specified amount of time.
  */
diff --git a/sys/kern/kern_timeout.c b/sys/kern/kern_timeout.c
index e853ae9..51919ea 100644
--- a/sys/kern/kern_timeout.c
+++ b/sys/kern/kern_timeout.c
@@ -55,6 +55,7 @@ __FBSDID("$FreeBSD: src/sys/kern/kern_timeout.c,v 1.118.2.1.2.1 2009/10/25 01:10
 #include <sys/sleepqueue.h>
 #include <sys/sysctl.h>
 #include <sys/smp.h>
+#include <sys/dynticks.h>
 
 SDT_PROVIDER_DEFINE(callout_execute);
 SDT_PROBE_DEFINE(callout_execute, kernel, , callout_start);
@@ -883,3 +884,32 @@ adjust_timeout_calltodo(time_change)
 	return;
 }
 #endif /* APM_FIXUP_CALLTODO */
+
+int
+callout_get_next_event(void)
+{
+	struct callout_cpu *cc;
+	struct callout *c;
+	struct callout_tailq *sc;
+	int curticks;
+	int skip = 1;
+
+	cc = CC_SELF();
+	curticks = cc->cc_softticks;
+
+	while( skip < ncallout ) {
+		sc = &cc->cc_callwheel[ (curticks+skip) & callwheelmask ];
+		/* search scanning ticks */
+		TAILQ_FOREACH( c, sc, c_links.tqe ){
+			if( c && c->c_time <= curticks + ncallout ){
+				if( c->c_time > 0 ){
+					goto out;
+				}
+			}
+		}
+		skip++;
+	}
+
+out:
+	return skip;
+}
diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c
index 4061c9c..e25c26d 100644
--- a/sys/kern/sched_4bsd.c
+++ b/sys/kern/sched_4bsd.c
@@ -55,6 +55,7 @@ __FBSDID("$FreeBSD: src/sys/kern/sched_4bsd.c,v 1.131.2.1.2.1 2009/10/25 01:10:2
 #include <sys/sx.h>
 #include <sys/turnstile.h>
 #include <sys/umtx.h>
+#include <sys/dynticks.h>
 #include <machine/pcb.h>
 #include <machine/smp.h>
 
@@ -1537,8 +1538,10 @@ sched_idletd(void *dummy)
 	for (;;) {
 		mtx_assert(&Giant, MA_NOTOWNED);
 
+		switch_to_dynticks();
 		while (sched_runnable() == 0)
 			cpu_idle(0);
+		switch_to_perticks();
 
 		mtx_lock_spin(&sched_lock);
 		mi_switch(SW_VOL | SWT_IDLE, NULL);
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index 6b5bafa..1cdf289 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -62,6 +62,7 @@ __FBSDID("$FreeBSD: src/sys/kern/sched_ule.c,v 1.257.2.3.2.1 2009/10/25 01:10:29
 #include <sys/vmmeter.h>
 #include <sys/cpuset.h>
 #include <sys/sbuf.h>
+#include <sys/dynticks.h>
 #ifdef KTRACE
 #include <sys/uio.h>
 #include <sys/ktrace.h>
@@ -2558,9 +2559,12 @@ sched_idletd(void *dummy)
 			}
 		}
 		switchcnt = tdq->tdq_switchcnt + tdq->tdq_oldswitchcnt;
-		if (tdq->tdq_load == 0)
+		if (tdq->tdq_load == 0){
+			switch_to_dynticks();
 			cpu_idle(switchcnt > 1);
-		if (tdq->tdq_load) {
+			/* wake up here */
+			switch_to_perticks();
+		}if (tdq->tdq_load) {
 			thread_lock(td);
 			mi_switch(SW_VOL | SWT_IDLE, NULL);
 			thread_unlock(td);
diff --git a/sys/sys/dynticks.h b/sys/sys/dynticks.h
new file mode 100644
index 0000000..13220b3
--- /dev/null
+++ b/sys/sys/dynticks.h
@@ -0,0 +1,34 @@
+#ifndef _SYS_DYNTICKS_H_
+#define _SYS_DYNTICKS_H_
+/*-
+ * Copyright (c) 2010 Tsuyoshi OZAWA 
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+/* FIXME :  */
+int callout_get_next_event(void);
+void switch_to_dynticks(void);
+void switch_to_perticks(void);
+
+#endif /* _SYS_DYNTICKS_H_ */
diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index 543fe60..3d74cae 100644
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -227,6 +227,7 @@ void	realitexpire(void *);
 int	sysbeep(int hertz, int period);
 
 void	hardclock(int usermode, uintfptr_t pc);
+void	hardclock_dynticks(int usermode, uintfptr_t pc,int skip);
 void	hardclock_cpu(int usermode);
 void	softclock(void *);
 void	statclock(int usermode);
-- 
1.6.4.3