From ab63a633cf072c719f885e46fa4814624312f672 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 25 Oct 2007 14:02:45 +0200 Subject: sched: fix unconditional irq lock Lockdep noticed that this lock can also be taken from hardirq context, and can thus not unconditionally disable/enable irqs. WARNING: at kernel/lockdep.c:2033 trace_hardirqs_on() [show_trace_log_lvl+26/48] show_trace_log_lvl+0x1a/0x30 [show_trace+18/32] show_trace+0x12/0x20 [dump_stack+22/32] dump_stack+0x16/0x20 [trace_hardirqs_on+405/416] trace_hardirqs_on+0x195/0x1a0 [_read_unlock_irq+34/48] _read_unlock_irq+0x22/0x30 [sched_debug_show+2615/4224] sched_debug_show+0xa37/0x1080 [show_state_filter+326/368] show_state_filter+0x146/0x170 [sysrq_handle_showstate+10/16] sysrq_handle_showstate+0xa/0x10 [__handle_sysrq+123/288] __handle_sysrq+0x7b/0x120 [handle_sysrq+40/64] handle_sysrq+0x28/0x40 [kbd_event+1045/1680] kbd_event+0x415/0x690 [input_pass_event+206/208] input_pass_event+0xce/0xd0 [input_handle_event+170/928] input_handle_event+0xaa/0x3a0 [input_event+95/112] input_event+0x5f/0x70 [atkbd_interrupt+434/1456] atkbd_interrupt+0x1b2/0x5b0 [serio_interrupt+59/128] serio_interrupt+0x3b/0x80 [i8042_interrupt+263/576] i8042_interrupt+0x107/0x240 [handle_IRQ_event+40/96] handle_IRQ_event+0x28/0x60 [handle_edge_irq+175/320] handle_edge_irq+0xaf/0x140 [do_IRQ+64/128] do_IRQ+0x40/0x80 [common_interrupt+46/52] common_interrupt+0x2e/0x34 Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/sched_debug.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index e6fb392e516..415e5c38554 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c @@ -80,6 +80,7 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) { struct task_struct *g, *p; + unsigned long flags; SEQ_printf(m, "\nrunnable tasks:\n" @@ -88,7 +89,7 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) "------------------------------------------------------" "----------------------------------------------------\n"); - read_lock_irq(&tasklist_lock); + read_lock_irqsave(&tasklist_lock, flags); do_each_thread(g, p) { if (!p->se.on_rq || task_cpu(p) != rq_cpu) @@ -97,7 +98,7 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) print_task(m, rq, p); } while_each_thread(g, p); - read_unlock_irq(&tasklist_lock); + read_unlock_irqrestore(&tasklist_lock, flags); } void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) -- cgit v1.2.3 From 17aacfb9cdf9a8329a6ece9c437551a29c93e47b Mon Sep 17 00:00:00 2001 From: Gautham R Shenoy Date: Sun, 28 Oct 2007 20:47:01 +0100 Subject: lockdep: fix a typo in the __lock_acquire comment Fix a typo in the __lock_acquire comment. Signed-off-by: Gautham R Shenoy Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/lockdep.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 55fe0c7cd95..ed38bbfc48a 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -2424,7 +2424,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, return 0; /* - * Calculate the chain hash: it's the combined has of all the + * Calculate the chain hash: it's the combined hash of all the * lock keys along the dependency chain. We save the hash value * at every step so that we can get the current hash easily * after unlock. The chain hash is then used to cache dependency -- cgit v1.2.3 From 64e38eb082bd845d6758079f65b191203986336d Mon Sep 17 00:00:00 2001 From: Adrian Bunk Date: Wed, 24 Oct 2007 18:24:22 +0200 Subject: clockevents: unexport tick_nohz_get_sleep_length This patch removes the unused EXPORT_SYMBOL_GPL(tick_nohz_get_sleep_length). Signed-off-by: Adrian Bunk Signed-off-by: Thomas Gleixner --- kernel/time/tick-sched.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel') diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 10a1347597f..5997456ebbc 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -320,8 +320,6 @@ ktime_t tick_nohz_get_sleep_length(void) return ts->sleep_length; } -EXPORT_SYMBOL_GPL(tick_nohz_get_sleep_length); - /** * nohz_restart_sched_tick - restart the idle tick from the idle task * -- cgit v1.2.3 From 129f1d2c5352eea3f7c8af9f8c1006dc0da7be52 Mon Sep 17 00:00:00 2001 From: Vegard Nossum Date: Thu, 11 Oct 2007 08:23:34 +0200 Subject: timer_list: Fix printk format strings This makes sure printk format strings contain no more than a single line. Signed-off-by: Vegard Nossum Signed-off-by: Thomas Gleixner --- kernel/time/timer_list.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index fdb2e03d4fe..12c5f4cb6b8 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c @@ -129,7 +129,8 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now) struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); int i; - SEQ_printf(m, "\ncpu: %d\n", cpu); + SEQ_printf(m, "\n"); + SEQ_printf(m, "cpu: %d\n", cpu); for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { SEQ_printf(m, " clock %d:\n", i); print_base(m, cpu_base->clock_base + i, now); @@ -184,7 +185,8 @@ print_tickdevice(struct seq_file *m, struct tick_device *td) { struct clock_event_device *dev = td->evtdev; - SEQ_printf(m, "\nTick Device: mode: %d\n", td->mode); + SEQ_printf(m, "\n"); + SEQ_printf(m, "Tick Device: mode: %d\n", td->mode); SEQ_printf(m, "Clock Event Device: "); if (!dev) { -- cgit v1.2.3 From edfed66e17854c312e81a2218f9b0592a555c9a3 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Mon, 29 Oct 2007 16:35:29 +1100 Subject: Quieten hrtimer printk: "Switched to high resolution mode .." Change the hrtimer printk "Switched to high resolution mode .." to be KERN_DEBUG, rather than KERN_INFO. If users need to see this they can pass "loglevel" or "debug" on the command line, or check dmesg. Signed-off-by: Michael Ellerman Signed-off-by: Thomas Gleixner kernel/hrtimer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) --- kernel/hrtimer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index b6d2ff7e37e..22a25142e4c 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -602,7 +602,7 @@ static int hrtimer_switch_to_hres(void) /* "Retrigger" the interrupt to get things going */ retrigger_next_event(NULL); local_irq_restore(flags); - printk(KERN_INFO "Switched to high resolution mode on CPU %d\n", + printk(KERN_DEBUG "Switched to high resolution mode on CPU %d\n", smp_processor_id()); return 1; } -- cgit v1.2.3 From ca5cd877ae699e758e6f26efc11b01bf6631d427 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 29 Oct 2007 04:31:16 +0000 Subject: x86 merge fallout: uml Don't undef __i386__/__x86_64__ in uml anymore, make sure that (few) places that required adjusting the ifdefs got those. Signed-off-by: Al Viro Signed-off-by: Linus Torvalds --- kernel/signal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/signal.c b/kernel/signal.c index 12006308c7e..4537bdda1eb 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -732,7 +732,7 @@ static void print_fatal_signal(struct pt_regs *regs, int signr) printk("%s/%d: potentially unexpected fatal signal %d.\n", current->comm, task_pid_nr(current), signr); -#ifdef __i386__ +#if defined(__i386__) && !defined(__arch_um__) printk("code at %08lx: ", regs->eip); { int i; -- cgit v1.2.3 From f7402e0361d4472535e07cfca648f2fa81d85cd2 Mon Sep 17 00:00:00 2001 From: Adrian Bunk Date: Mon, 29 Oct 2007 21:18:10 +0100 Subject: sched: make kernel/sched.c:account_guest_time() static account_guest_time() can become static. Signed-off-by: Adrian Bunk Signed-off-by: Ingo Molnar --- kernel/sched.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index b4fbbc44045..74dbb4020cf 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -3355,7 +3355,7 @@ void account_user_time(struct task_struct *p, cputime_t cputime) * @p: the process that the cpu time gets accounted to * @cputime: the cpu time spent in virtual machine since the last update */ -void account_guest_time(struct task_struct *p, cputime_t cputime) +static void account_guest_time(struct task_struct *p, cputime_t cputime) { cputime64_t tmp; struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; -- cgit v1.2.3 From 73a2bcb0edb9ffb0b007b3546b430e2c6e415eee Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 29 Oct 2007 21:18:11 +0100 Subject: sched: keep utime/stime monotonic keep utime/stime monotonic. cpustats use utime/stime as a ratio against sum_exec_runtime, as a consequence it can happen - when the ratio changes faster than time accumulates - that either can be appear to go backwards. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/fork.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index ddafdfac945..a65bfc47177 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1056,6 +1056,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, p->gtime = cputime_zero; p->utimescaled = cputime_zero; p->stimescaled = cputime_zero; + p->prev_utime = cputime_zero; #ifdef CONFIG_TASK_XACCT p->rchar = 0; /* I/O counter: bytes read */ -- cgit v1.2.3 From 7bae49d498de87f0da0c20c67adaa278eac84566 Mon Sep 17 00:00:00 2001 From: James Bottomley Date: Mon, 29 Oct 2007 21:18:11 +0100 Subject: sched: fix incorrect assumption that cpu 0 exists This patch: commit 9b5b77512dce239fa168183fa71896712232e95a Author: Srivatsa Vaddagiri Date: Mon Oct 15 17:00:09 2007 +0200 sched: clean up code under CONFIG_FAIR_GROUP_SCHED Introduced an assumption of the existence of CPU0 via this line cfs_rq = tg->cfs_rq[0]; If you have no CPU0, that will be NULL. The fix seems to be just to take whatever cfs_rq queue comes out of the for_each_possible_cpu() loop, since they're all equally good for the destruction operation. Signed-off-by: James Bottomley Signed-off-by: Ingo Molnar --- kernel/sched.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 74dbb4020cf..235952b100e 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -7041,7 +7041,7 @@ static void free_sched_group(struct rcu_head *rhp) /* Destroy runqueue etc associated with a task group */ void sched_destroy_group(struct task_group *tg) { - struct cfs_rq *cfs_rq; + struct cfs_rq *cfs_rq = NULL; int i; for_each_possible_cpu(i) { @@ -7049,7 +7049,7 @@ void sched_destroy_group(struct task_group *tg) list_del_rcu(&cfs_rq->leaf_cfs_rq_list); } - cfs_rq = tg->cfs_rq[0]; + BUG_ON(!cfs_rq); /* wait for possible concurrent references to cfs_rqs complete */ call_rcu(&cfs_rq->rcu, free_sched_group); -- cgit v1.2.3 From ae8393e508e5f17add66964688c49bf0bfe4fcf9 Mon Sep 17 00:00:00 2001 From: Srivatsa Vaddagiri Date: Mon, 29 Oct 2007 21:18:11 +0100 Subject: sched: move rcu_head to task_group struct Peter Zijlstra noticed that the rcu_head object need not be present in every cfs_rq of a group. Move it to the task_group structure instead. Signed-off-by: Srivatsa Vaddagiri Signed-off-by: Ingo Molnar --- kernel/sched.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 235952b100e..470480790f3 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -172,6 +172,7 @@ struct task_group { unsigned long shares; /* spinlock to serialize modification to shares */ spinlock_t lock; + struct rcu_head rcu; }; /* Default task group's sched entity on each cpu */ @@ -258,7 +259,6 @@ struct cfs_rq { */ struct list_head leaf_cfs_rq_list; /* Better name : task_cfs_rq_list? */ struct task_group *tg; /* group that "owns" this runqueue */ - struct rcu_head rcu; #endif }; @@ -7019,8 +7019,8 @@ err: /* rcu callback to free various structures associated with a task group */ static void free_sched_group(struct rcu_head *rhp) { - struct cfs_rq *cfs_rq = container_of(rhp, struct cfs_rq, rcu); - struct task_group *tg = cfs_rq->tg; + struct task_group *tg = container_of(rhp, struct task_group, rcu); + struct cfs_rq *cfs_rq; struct sched_entity *se; int i; @@ -7052,7 +7052,7 @@ void sched_destroy_group(struct task_group *tg) BUG_ON(!cfs_rq); /* wait for possible concurrent references to cfs_rqs complete */ - call_rcu(&cfs_rq->rcu, free_sched_group); + call_rcu(&tg->rcu, free_sched_group); } /* change task's runqueue when it moves between groups. -- cgit v1.2.3 From fe5c7cc22897b809a2fbe05bea71963853df7f17 Mon Sep 17 00:00:00 2001 From: Paul Menage Date: Mon, 29 Oct 2007 21:18:11 +0100 Subject: sched: report CPU usage in CFS cgroup directories Adds a cpu.usage file to the CFS cgroup that reports CPU usage in milliseconds for that cgroup's tasks [ mingo@elte.hu: style cleanups. ] Signed-off-by: Paul Menage Signed-off-by: Ingo Molnar --- kernel/sched.c | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 470480790f3..6c4abfbd68e 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -7211,15 +7211,43 @@ static u64 cpu_shares_read_uint(struct cgroup *cgrp, struct cftype *cft) return (u64) tg->shares; } -static struct cftype cpu_shares = { - .name = "shares", - .read_uint = cpu_shares_read_uint, - .write_uint = cpu_shares_write_uint, +static u64 cpu_usage_read(struct cgroup *cgrp, struct cftype *cft) +{ + struct task_group *tg = cgroup_tg(cgrp); + unsigned long flags; + u64 res = 0; + int i; + + for_each_possible_cpu(i) { + /* + * Lock to prevent races with updating 64-bit counters + * on 32-bit arches. + */ + spin_lock_irqsave(&cpu_rq(i)->lock, flags); + res += tg->se[i]->sum_exec_runtime; + spin_unlock_irqrestore(&cpu_rq(i)->lock, flags); + } + /* Convert from ns to ms */ + do_div(res, 1000000); + + return res; +} + +static struct cftype cpu_files[] = { + { + .name = "shares", + .read_uint = cpu_shares_read_uint, + .write_uint = cpu_shares_write_uint, + }, + { + .name = "usage", + .read_uint = cpu_usage_read, + }, }; static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont) { - return cgroup_add_file(cont, ss, &cpu_shares); + return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files)); } struct cgroup_subsys cpu_cgroup_subsys = { -- cgit v1.2.3 From 8eb172d9418c9387234a2c9a344131c46b5eea5b Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 29 Oct 2007 21:18:11 +0100 Subject: sched: fix style of swap() macro in kernel/sched_fair.c fix style of swap() macro in kernel/sched_fair.c. ( this macro should eventually move to a general header, as ext3 uses a similar construct too. ) Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 9971831b560..01859f662ab 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -1025,7 +1025,7 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr) } } -#define swap(a,b) do { typeof(a) tmp = (a); (a) = (b); (b) = tmp; } while (0) +#define swap(a, b) do { typeof(a) tmp = (a); (a) = (b); (b) = tmp; } while (0) /* * Share the fairness runtime between parent and child, thus the -- cgit v1.2.3 From 38605cae99d386332df6822a22dba7bfdc8fae1c Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 29 Oct 2007 21:18:11 +0100 Subject: sched: fix style in kernel/sched.c fallout of recent commits: small coding style fixes. Signed-off-by: Ingo Molnar --- kernel/sched.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 6c4abfbd68e..3f6bd111290 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -5365,7 +5365,7 @@ static struct ctl_table sd_ctl_dir[] = { .procname = "sched_domain", .mode = 0555, }, - {0,}, + {0, }, }; static struct ctl_table sd_ctl_root[] = { @@ -5375,7 +5375,7 @@ static struct ctl_table sd_ctl_root[] = { .mode = 0555, .child = sd_ctl_dir, }, - {0,}, + {0, }, }; static struct ctl_table *sd_alloc_ctl_entry(int n) @@ -7251,13 +7251,13 @@ static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont) } struct cgroup_subsys cpu_cgroup_subsys = { - .name = "cpu", - .create = cpu_cgroup_create, - .destroy = cpu_cgroup_destroy, - .can_attach = cpu_cgroup_can_attach, - .attach = cpu_cgroup_attach, - .populate = cpu_cgroup_populate, - .subsys_id = cpu_cgroup_subsys_id, + .name = "cpu", + .create = cpu_cgroup_create, + .destroy = cpu_cgroup_destroy, + .can_attach = cpu_cgroup_can_attach, + .attach = cpu_cgroup_attach, + .populate = cpu_cgroup_populate, + .subsys_id = cpu_cgroup_subsys_id, .early_init = 1, }; -- cgit v1.2.3 From 9301899be75b464ef097f0b5af7af6d9bd8f68a7 Mon Sep 17 00:00:00 2001 From: Balbir Singh Date: Tue, 30 Oct 2007 00:26:32 +0100 Subject: sched: fix /proc//stat stime/utime monotonicity, part 2 Extend Peter's patch to fix accounting issues, by keeping stime monotonic too. Signed-off-by: Balbir Singh Signed-off-by: Ingo Molnar Tested-by: Frans Pop --- kernel/fork.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index a65bfc47177..28a74015198 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1057,6 +1057,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, p->utimescaled = cputime_zero; p->stimescaled = cputime_zero; p->prev_utime = cputime_zero; + p->prev_stime = cputime_zero; #ifdef CONFIG_TASK_XACCT p->rchar = 0; /* I/O counter: bytes read */ -- cgit v1.2.3 From cc5f916e90a811dd8f809b4d17409f98e74b237c Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Oct 2007 14:37:12 -0700 Subject: Freezer: do not allow freezing processes to clear TIF_SIGPENDING Do not allow processes to clear their TIF_SIGPENDING if TIF_FREEZE is set, so that they will not race with the freezer (like mysqld does, for example). Signed-off-by: Rafael J. Wysocki Acked-by: Nigel Cunningham Acked-by: Pavel Machek Cc: Oleg Nesterov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/signal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/signal.c b/kernel/signal.c index 4537bdda1eb..909a0cc6bc7 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -124,7 +124,7 @@ void recalc_sigpending_and_wake(struct task_struct *t) void recalc_sigpending(void) { - if (!recalc_sigpending_tsk(current)) + if (!recalc_sigpending_tsk(current) && !freezing(current)) clear_thread_flag(TIF_SIGPENDING); } -- cgit v1.2.3 From f3baa4827a4b13905dbbdddf15463541bd671dfd Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 29 Oct 2007 00:54:39 -0700 Subject: [COMPAT]: Fix build on COMPAT platforms when CONFIG_NET is disabled. Add some missing cond_syscall() entries for this case. Signed-off-by: David S. Miller --- kernel/sys_ni.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index 52c7a151e29..56cb009a4b3 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -40,10 +40,14 @@ cond_syscall(sys_recvfrom); cond_syscall(sys_recv); cond_syscall(sys_socket); cond_syscall(sys_setsockopt); +cond_syscall(compat_sys_setsockopt); cond_syscall(sys_getsockopt); +cond_syscall(compat_sys_getsockopt); cond_syscall(sys_shutdown); cond_syscall(sys_sendmsg); +cond_syscall(compat_sys_sendmsg); cond_syscall(sys_recvmsg); +cond_syscall(compat_sys_recvmsg); cond_syscall(sys_socketcall); cond_syscall(sys_futex); cond_syscall(compat_sys_futex); -- cgit v1.2.3 From fad23fc78b959dae89768e523c3a6f5edb83bbe9 Mon Sep 17 00:00:00 2001 From: Adrian Bunk Date: Fri, 2 Nov 2007 16:43:22 +0100 Subject: kernel/futex.c: make 3 functions static The following functions can now become static again: - get_futex_key() - get_futex_key_refs() - drop_futex_key_refs() Signed-off-by: Adrian Bunk Signed-off-by: Rusty Russell --- kernel/futex.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/futex.c b/kernel/futex.c index 32710451dc2..9dc591ab681 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -181,8 +181,8 @@ static inline int match_futex(union futex_key *key1, union futex_key *key2) * For other futexes, it points to ¤t->mm->mmap_sem and * caller must have taken the reader lock. but NOT any spinlocks. */ -int get_futex_key(u32 __user *uaddr, struct rw_semaphore *fshared, - union futex_key *key) +static int get_futex_key(u32 __user *uaddr, struct rw_semaphore *fshared, + union futex_key *key) { unsigned long address = (unsigned long)uaddr; struct mm_struct *mm = current->mm; @@ -268,14 +268,13 @@ int get_futex_key(u32 __user *uaddr, struct rw_semaphore *fshared, } return err; } -EXPORT_SYMBOL_GPL(get_futex_key); /* * Take a reference to the resource addressed by a key. * Can be called while holding spinlocks. * */ -inline void get_futex_key_refs(union futex_key *key) +static void get_futex_key_refs(union futex_key *key) { if (key->both.ptr == 0) return; @@ -288,13 +287,12 @@ inline void get_futex_key_refs(union futex_key *key) break; } } -EXPORT_SYMBOL_GPL(get_futex_key_refs); /* * Drop a reference to the resource addressed by a key. * The hash bucket spinlock must not be held. */ -void drop_futex_key_refs(union futex_key *key) +static void drop_futex_key_refs(union futex_key *key) { if (!key->both.ptr) return; @@ -307,7 +305,6 @@ void drop_futex_key_refs(union futex_key *key) break; } } -EXPORT_SYMBOL_GPL(drop_futex_key_refs); static u32 cmpxchg_futex_value_locked(u32 __user *uaddr, u32 uval, u32 newval) { -- cgit v1.2.3 From 5db6a4dac1c7f459af2f895f6889346cedd467a1 Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Mon, 5 Nov 2007 14:50:52 -0800 Subject: Dump stack during sysctl registration failure Let's make immediately obvious from where sysctl comes from and messages itself more noticeable. Signed-off-by: Alexey Dobriyan Acked-by: "Eric W. Biederman" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/sysctl_check.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index ed6fe51df77..5a2f2b2bf88 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c @@ -1432,6 +1432,7 @@ static void set_fail(const char **fail, struct ctl_table *table, const char *str printk(KERN_ERR "sysctl table check failed: "); sysctl_print_path(table); printk(" %s\n", *fail); + dump_stack(); } *fail = str; } -- cgit v1.2.3 From 8dce39c231af554932f8ab0d671e077ab6db9e46 Mon Sep 17 00:00:00 2001 From: Li Zefan Date: Mon, 5 Nov 2007 14:51:10 -0800 Subject: time: fix inconsistent function names in comments Signed-off-by: Li Zefan Cc: Thomas Gleixner Cc: john stultz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/time/tick-broadcast.c | 2 +- kernel/time/tick-sched.c | 2 +- kernel/timer.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 8cfb8b2ce77..aa82d7bf478 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -508,7 +508,7 @@ static void tick_broadcast_clear_oneshot(int cpu) } /** - * tick_broadcast_setup_highres - setup the broadcast device for highres + * tick_broadcast_setup_oneshot - setup the broadcast device */ void tick_broadcast_setup_oneshot(struct clock_event_device *bc) { diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 5997456ebbc..27a2338deb4 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -321,7 +321,7 @@ ktime_t tick_nohz_get_sleep_length(void) } /** - * nohz_restart_sched_tick - restart the idle tick from the idle task + * tick_nohz_restart_sched_tick - restart the idle tick from the idle task * * Restart the idle tick when the CPU is woken up from idle */ diff --git a/kernel/timer.c b/kernel/timer.c index fb4e67d5dd6..00e44e2afd6 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -790,7 +790,7 @@ static unsigned long cmp_next_hrtimer_event(unsigned long now, } /** - * next_timer_interrupt - return the jiffy of the next pending timer + * get_next_timer_interrupt - return the jiffy of the next pending timer * @now: current time (in jiffies) */ unsigned long get_next_timer_interrupt(unsigned long now) -- cgit v1.2.3 From 10b777246c6953100099af1870d35c8b24d49b12 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 9 Nov 2007 22:39:37 +0100 Subject: sched: fix vslice vslice was missing a factor NICE_0_LOAD, as weight is in weight*NICE_0_LOAD units. the effect of this bug was larger initial slices and thus latency-noisier forks. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 01859f662ab..62b057603f0 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -259,6 +259,7 @@ static u64 __sched_vslice(unsigned long rq_weight, unsigned long nr_running) { u64 vslice = __sched_period(nr_running); + vslice *= NICE_0_LOAD; do_div(vslice, rq_weight); return vslice; -- cgit v1.2.3 From 2cb8600e6be4281e381d39e44de4359e46333e23 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 9 Nov 2007 22:39:37 +0100 Subject: sched: documentation: place_entity() comments Add a few comments to place_entity(). No code changed. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 62b057603f0..8763bee6b66 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -473,19 +473,26 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) } else if (sched_feat(APPROX_AVG) && cfs_rq->nr_running) vruntime += sched_vslice(cfs_rq)/2; + /* + * The 'current' period is already promised to the current tasks, + * however the extra weight of the new task will slow them down a + * little, place the new task so that it fits in the slot that + * stays open at the end. + */ if (initial && sched_feat(START_DEBIT)) vruntime += sched_vslice_add(cfs_rq, se); if (!initial) { + /* sleeps upto a single latency don't count. */ if (sched_feat(NEW_FAIR_SLEEPERS) && entity_is_task(se) && task_of(se)->policy != SCHED_BATCH) vruntime -= sysctl_sched_latency; - vruntime = max_t(s64, vruntime, se->vruntime); + /* ensure we never gain time by being placed backwards. */ + vruntime = max_vruntime(se->vruntime, vruntime); } se->vruntime = vruntime; - } static void -- cgit v1.2.3 From b2be5e96dc0b5a179cf4cb98e65cfb605752ca26 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 9 Nov 2007 22:39:37 +0100 Subject: sched: reintroduce the sched_min_granularity tunable we lost the sched_min_granularity tunable to a clever optimization that uses the sched_latency/min_granularity ratio - but the ratio is quite unintuitive to users and can also crash the kernel if the ratio is set to 0. So reintroduce the min_granularity tunable, while keeping the ratio maintained internally. no functionality changed. [ mingo@elte.hu: some fixlets. ] Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/sched_debug.c | 2 +- kernel/sched_fair.c | 35 ++++++++++++++++++++++++++++------- kernel/sysctl.c | 11 +++++++---- 3 files changed, 36 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index 415e5c38554..ca198a797bf 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c @@ -211,7 +211,7 @@ static int sched_debug_show(struct seq_file *m, void *v) #define PN(x) \ SEQ_printf(m, " .%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x)) PN(sysctl_sched_latency); - PN(sysctl_sched_nr_latency); + PN(sysctl_sched_min_granularity); PN(sysctl_sched_wakeup_granularity); PN(sysctl_sched_batch_wakeup_granularity); PN(sysctl_sched_child_runs_first); diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 8763bee6b66..c495dcf7031 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -35,16 +35,21 @@ const_debug unsigned int sysctl_sched_latency = 20000000ULL; /* - * After fork, child runs first. (default) If set to 0 then - * parent will (try to) run first. + * Minimal preemption granularity for CPU-bound tasks: + * (default: 1 msec, units: nanoseconds) */ -const_debug unsigned int sysctl_sched_child_runs_first = 1; +const_debug unsigned int sysctl_sched_min_granularity = 1000000ULL; /* - * Minimal preemption granularity for CPU-bound tasks: - * (default: 2 msec, units: nanoseconds) + * is kept at sysctl_sched_latency / sysctl_sched_min_granularity */ -const_debug unsigned int sysctl_sched_nr_latency = 20; +const_debug unsigned int sched_nr_latency = 20; + +/* + * After fork, child runs first. (default) If set to 0 then + * parent will (try to) run first. + */ +const_debug unsigned int sysctl_sched_child_runs_first = 1; /* * sys_sched_yield() compat mode @@ -212,6 +217,22 @@ static inline struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) * Scheduling class statistics methods: */ +#ifdef CONFIG_SCHED_DEBUG +int sched_nr_latency_handler(struct ctl_table *table, int write, + struct file *filp, void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret = proc_dointvec_minmax(table, write, filp, buffer, lenp, ppos); + + if (ret || !write) + return ret; + + sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency, + sysctl_sched_min_granularity); + + return 0; +} +#endif /* * The idea is to set a period in which each task runs once. @@ -224,7 +245,7 @@ static inline struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) static u64 __sched_period(unsigned long nr_running) { u64 period = sysctl_sched_latency; - unsigned long nr_latency = sysctl_sched_nr_latency; + unsigned long nr_latency = sched_nr_latency; if (unlikely(nr_running > nr_latency)) { period *= nr_running; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 3b4efbe2644..6e3b63c0685 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -235,11 +235,14 @@ static struct ctl_table kern_table[] = { #ifdef CONFIG_SCHED_DEBUG { .ctl_name = CTL_UNNUMBERED, - .procname = "sched_nr_latency", - .data = &sysctl_sched_nr_latency, + .procname = "sched_min_granularity_ns", + .data = &sysctl_sched_min_granularity, .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = &proc_dointvec, + .proc_handler = &sched_nr_latency_handler, + .strategy = &sysctl_intvec, + .extra1 = &min_sched_granularity_ns, + .extra2 = &max_sched_granularity_ns, }, { .ctl_name = CTL_UNNUMBERED, @@ -247,7 +250,7 @@ static struct ctl_table kern_table[] = { .data = &sysctl_sched_latency, .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = &proc_dointvec_minmax, + .proc_handler = &sched_nr_latency_handler, .strategy = &sysctl_intvec, .extra1 = &min_sched_granularity_ns, .extra2 = &max_sched_granularity_ns, -- cgit v1.2.3 From 9a41785cc43d88397f787a651ed7286a33f8462f Mon Sep 17 00:00:00 2001 From: Balbir Singh Date: Fri, 9 Nov 2007 22:39:37 +0100 Subject: sched: fix delay accounting regression Fix the delay accounting regression introduced by commit 75d4ef16a6aa84f708188bada182315f80aab6fa. rq no longer has sched_info data associated with it. task_struct sched_info structure is used by delay accounting to provide back statistics to user space. also remove direct use of sched_clock() (which is not a valid thing to do anymore) and use rq->clock instead. Signed-off-by: Balbir Singh Signed-off-by: Ingo Molnar --- kernel/sched_stats.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_stats.h b/kernel/sched_stats.h index ef1a7df80ea..630178e53bb 100644 --- a/kernel/sched_stats.h +++ b/kernel/sched_stats.h @@ -127,7 +127,7 @@ rq_sched_info_depart(struct rq *rq, unsigned long long delta) # define schedstat_set(var, val) do { } while (0) #endif -#ifdef CONFIG_SCHEDSTATS +#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) /* * Called when a process is dequeued from the active array and given * the cpu. We should note that with the exception of interactive @@ -155,7 +155,7 @@ static inline void sched_info_dequeued(struct task_struct *t) */ static void sched_info_arrive(struct task_struct *t) { - unsigned long long now = sched_clock(), delta = 0; + unsigned long long now = task_rq(t)->clock, delta = 0; if (t->sched_info.last_queued) delta = now - t->sched_info.last_queued; @@ -186,7 +186,7 @@ static inline void sched_info_queued(struct task_struct *t) { if (unlikely(sched_info_on())) if (!t->sched_info.last_queued) - t->sched_info.last_queued = sched_clock(); + t->sched_info.last_queued = task_rq(t)->clock; } /* @@ -195,7 +195,8 @@ static inline void sched_info_queued(struct task_struct *t) */ static inline void sched_info_depart(struct task_struct *t) { - unsigned long long delta = sched_clock() - t->sched_info.last_arrival; + unsigned long long delta = task_rq(t)->clock - + t->sched_info.last_arrival; t->sched_info.cpu_time += delta; rq_sched_info_depart(task_rq(t), delta); @@ -231,5 +232,5 @@ sched_info_switch(struct task_struct *prev, struct task_struct *next) #else #define sched_info_queued(t) do { } while (0) #define sched_info_switch(t, next) do { } while (0) -#endif /* CONFIG_SCHEDSTATS */ +#endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */ -- cgit v1.2.3 From fa13a5a1f25f671d084d8884be96fc48d9b68275 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Fri, 9 Nov 2007 22:39:38 +0100 Subject: sched: restore deterministic CPU accounting on powerpc Since powerpc started using CONFIG_GENERIC_CLOCKEVENTS, the deterministic CPU accounting (CONFIG_VIRT_CPU_ACCOUNTING) has been broken on powerpc, because we end up counting user time twice: once in timer_interrupt() and once in update_process_times(). This fixes the problem by pulling the code in update_process_times that updates utime and stime into a separate function called account_process_tick. If CONFIG_VIRT_CPU_ACCOUNTING is not defined, there is a version of account_process_tick in kernel/timer.c that simply accounts a whole tick to either utime or stime as before. If CONFIG_VIRT_CPU_ACCOUNTING is defined, then arch code gets to implement account_process_tick. This also lets us simplify the s390 code a bit; it means that the s390 timer interrupt can now call update_process_times even when CONFIG_VIRT_CPU_ACCOUNTING is turned on, and can just implement a suitable account_process_tick(). account_process_tick() now takes the task_struct * as an argument. Tested both with and without CONFIG_VIRT_CPU_ACCOUNTING. Signed-off-by: Paul Mackerras Signed-off-by: Ingo Molnar --- kernel/timer.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/timer.c b/kernel/timer.c index 00e44e2afd6..a05817c021d 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -817,6 +817,19 @@ unsigned long next_timer_interrupt(void) #endif +#ifndef CONFIG_VIRT_CPU_ACCOUNTING +void account_process_tick(struct task_struct *p, int user_tick) +{ + if (user_tick) { + account_user_time(p, jiffies_to_cputime(1)); + account_user_time_scaled(p, jiffies_to_cputime(1)); + } else { + account_system_time(p, HARDIRQ_OFFSET, jiffies_to_cputime(1)); + account_system_time_scaled(p, jiffies_to_cputime(1)); + } +} +#endif + /* * Called from the timer interrupt handler to charge one tick to the current * process. user_tick is 1 if the tick is user time, 0 for system. @@ -827,13 +840,7 @@ void update_process_times(int user_tick) int cpu = smp_processor_id(); /* Note: this timer irq context must be accounted for as well. */ - if (user_tick) { - account_user_time(p, jiffies_to_cputime(1)); - account_user_time_scaled(p, jiffies_to_cputime(1)); - } else { - account_system_time(p, HARDIRQ_OFFSET, jiffies_to_cputime(1)); - account_system_time_scaled(p, jiffies_to_cputime(1)); - } + account_process_tick(p, user_tick); run_local_timers(); if (rcu_pending(cpu)) rcu_check_callbacks(cpu, user_tick); -- cgit v1.2.3 From 19978ca610946ed57c071bad63f8f6642ca1298b Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 9 Nov 2007 22:39:38 +0100 Subject: sched: reintroduce SMP tunings again Yanmin Zhang reported an aim7 regression and bisected it down to: | commit 38ad464d410dadceda1563f36bdb0be7fe4c8938 | Author: Ingo Molnar | Date: Mon Oct 15 17:00:02 2007 +0200 | | sched: uniform tunings | | use the same defaults on both UP and SMP. fix this by reintroducing similar SMP tunings again. This resolves the regression. (also update the comments to match the ilog2(nr_cpus) tuning effect) Signed-off-by: Ingo Molnar --- kernel/sched.c | 28 ++++++++++++++++++++++++++++ kernel/sched_fair.c | 18 +++++++++--------- 2 files changed, 37 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 3f6bd111290..69cae271c63 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -4992,6 +4992,32 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) */ cpumask_t nohz_cpu_mask = CPU_MASK_NONE; +/* + * Increase the granularity value when there are more CPUs, + * because with more CPUs the 'effective latency' as visible + * to users decreases. But the relationship is not linear, + * so pick a second-best guess by going with the log2 of the + * number of CPUs. + * + * This idea comes from the SD scheduler of Con Kolivas: + */ +static inline void sched_init_granularity(void) +{ + unsigned int factor = 1 + ilog2(num_online_cpus()); + const unsigned long limit = 200000000; + + sysctl_sched_min_granularity *= factor; + if (sysctl_sched_min_granularity > limit) + sysctl_sched_min_granularity = limit; + + sysctl_sched_latency *= factor; + if (sysctl_sched_latency > limit) + sysctl_sched_latency = limit; + + sysctl_sched_wakeup_granularity *= factor; + sysctl_sched_batch_wakeup_granularity *= factor; +} + #ifdef CONFIG_SMP /* * This is how migration works: @@ -6688,10 +6714,12 @@ void __init sched_init_smp(void) /* Move init over to a non-isolated CPU */ if (set_cpus_allowed(current, non_isolated_cpus) < 0) BUG(); + sched_init_granularity(); } #else void __init sched_init_smp(void) { + sched_init_granularity(); } #endif /* CONFIG_SMP */ diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index c495dcf7031..7264814ba62 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -22,7 +22,7 @@ /* * Targeted preemption latency for CPU-bound tasks: - * (default: 20ms, units: nanoseconds) + * (default: 20ms * ilog(ncpus), units: nanoseconds) * * NOTE: this latency value is not the same as the concept of * 'timeslice length' - timeslices in CFS are of variable length @@ -32,18 +32,18 @@ * (to see the precise effective timeslice length of your workload, * run vmstat and monitor the context-switches (cs) field) */ -const_debug unsigned int sysctl_sched_latency = 20000000ULL; +unsigned int sysctl_sched_latency = 20000000ULL; /* * Minimal preemption granularity for CPU-bound tasks: - * (default: 1 msec, units: nanoseconds) + * (default: 1 msec * ilog(ncpus), units: nanoseconds) */ -const_debug unsigned int sysctl_sched_min_granularity = 1000000ULL; +unsigned int sysctl_sched_min_granularity = 1000000ULL; /* * is kept at sysctl_sched_latency / sysctl_sched_min_granularity */ -const_debug unsigned int sched_nr_latency = 20; +unsigned int sched_nr_latency = 20; /* * After fork, child runs first. (default) If set to 0 then @@ -61,23 +61,23 @@ unsigned int __read_mostly sysctl_sched_compat_yield; /* * SCHED_BATCH wake-up granularity. - * (default: 10 msec, units: nanoseconds) + * (default: 10 msec * ilog(ncpus), units: nanoseconds) * * This option delays the preemption effects of decoupled workloads * and reduces their over-scheduling. Synchronous workloads will still * have immediate wakeup/sleep latencies. */ -const_debug unsigned int sysctl_sched_batch_wakeup_granularity = 10000000UL; +unsigned int sysctl_sched_batch_wakeup_granularity = 10000000UL; /* * SCHED_OTHER wake-up granularity. - * (default: 10 msec, units: nanoseconds) + * (default: 10 msec * ilog(ncpus), units: nanoseconds) * * This option delays the preemption effects of decoupled workloads * and reduces their over-scheduling. Synchronous workloads will still * have immediate wakeup/sleep latencies. */ -const_debug unsigned int sysctl_sched_wakeup_granularity = 10000000UL; +unsigned int sysctl_sched_wakeup_granularity = 10000000UL; const_debug unsigned int sysctl_sched_migration_cost = 500000UL; -- cgit v1.2.3 From d6322faf296ab5bbb597f8b0abcb50153754cd08 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 9 Nov 2007 22:39:38 +0100 Subject: sched: cleanup, use NSEC_PER_MSEC and NSEC_PER_SEC 1) hardcoded 1000000000 value is used five times in places where NSEC_PER_SEC might be more readable. 2) A conversion from nsec to msec uses the hardcoded 1000000 value, which is a candidate for NSEC_PER_MSEC. no code changed: text data bss dec hex filename 44359 3326 36 47721 ba69 sched.o.before 44359 3326 36 47721 ba69 sched.o.after Signed-off-by: Eric Dumazet Signed-off-by: Ingo Molnar --- kernel/sched.c | 8 ++++---- kernel/sysctl.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 69cae271c63..387258cdd0b 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -75,7 +75,7 @@ */ unsigned long long __attribute__((weak)) sched_clock(void) { - return (unsigned long long)jiffies * (1000000000 / HZ); + return (unsigned long long)jiffies * (NSEC_PER_SEC / HZ); } /* @@ -99,8 +99,8 @@ unsigned long long __attribute__((weak)) sched_clock(void) /* * Some helpers for converting nanosecond timing to jiffy resolution */ -#define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (1000000000 / HZ)) -#define JIFFIES_TO_NS(TIME) ((TIME) * (1000000000 / HZ)) +#define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ)) +#define JIFFIES_TO_NS(TIME) ((TIME) * (NSEC_PER_SEC / HZ)) #define NICE_0_LOAD SCHED_LOAD_SCALE #define NICE_0_SHIFT SCHED_LOAD_SHIFT @@ -7256,7 +7256,7 @@ static u64 cpu_usage_read(struct cgroup *cgrp, struct cftype *cft) spin_unlock_irqrestore(&cpu_rq(i)->lock, flags); } /* Convert from ns to ms */ - do_div(res, 1000000); + do_div(res, NSEC_PER_MSEC); return res; } diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 6e3b63c0685..adddf682d4c 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -226,9 +226,9 @@ static struct ctl_table root_table[] = { #ifdef CONFIG_SCHED_DEBUG static unsigned long min_sched_granularity_ns = 100000; /* 100 usecs */ -static unsigned long max_sched_granularity_ns = 1000000000; /* 1 second */ +static unsigned long max_sched_granularity_ns = NSEC_PER_SEC; /* 1 second */ static unsigned long min_wakeup_granularity_ns; /* 0 usecs */ -static unsigned long max_wakeup_granularity_ns = 1000000000; /* 1 second */ +static unsigned long max_wakeup_granularity_ns = NSEC_PER_SEC; /* 1 second */ #endif static struct ctl_table kern_table[] = { -- cgit v1.2.3 From 52d3da1ad4f442cec877fbeb83902707b56da0cf Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 9 Nov 2007 22:39:39 +0100 Subject: sched: turn off PREEMPT_RESTRICT PREEMPT_RESTRICT was a method aimed at reducing the amount of wakeup related preemption. It has a disadvantage though, it can prevent legitimate wakeups if a task is 'unlucky' to be hit too early by a tick that clears peer_preempt. Now that the wakeup preemption has been cleaned up we dont seem to have excessive preemptions anymore, so this feature can be turned off. (and removed in the next patch) Signed-off-by: Ingo Molnar --- kernel/sched.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 387258cdd0b..4b23dfb4c80 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -469,7 +469,7 @@ const_debug unsigned int sysctl_sched_features = SCHED_FEAT_TREE_AVG * 0 | SCHED_FEAT_APPROX_AVG * 0 | SCHED_FEAT_WAKEUP_PREEMPT * 1 | - SCHED_FEAT_PREEMPT_RESTRICT * 1; + SCHED_FEAT_PREEMPT_RESTRICT * 0; #define sched_feat(x) (sysctl_sched_features & SCHED_FEAT_##x) -- cgit v1.2.3 From 3e3e13f399ac8060a20d14d210a28dc02dda372e Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 9 Nov 2007 22:39:39 +0100 Subject: sched: remove PREEMPT_RESTRICT remove PREEMPT_RESTRICT. (this is a separate commit so that any regression related to the removal itself is bisectable) Signed-off-by: Ingo Molnar --- kernel/sched.c | 4 +--- kernel/sched_fair.c | 10 ++-------- 2 files changed, 3 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 4b23dfb4c80..2a107e4ad5e 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -460,7 +460,6 @@ enum { SCHED_FEAT_TREE_AVG = 4, SCHED_FEAT_APPROX_AVG = 8, SCHED_FEAT_WAKEUP_PREEMPT = 16, - SCHED_FEAT_PREEMPT_RESTRICT = 32, }; const_debug unsigned int sysctl_sched_features = @@ -468,8 +467,7 @@ const_debug unsigned int sysctl_sched_features = SCHED_FEAT_START_DEBIT * 1 | SCHED_FEAT_TREE_AVG * 0 | SCHED_FEAT_APPROX_AVG * 0 | - SCHED_FEAT_WAKEUP_PREEMPT * 1 | - SCHED_FEAT_PREEMPT_RESTRICT * 0; + SCHED_FEAT_WAKEUP_PREEMPT * 1; #define sched_feat(x) (sysctl_sched_features & SCHED_FEAT_##x) diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 7264814ba62..fbcb426029d 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -546,7 +546,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) update_stats_dequeue(cfs_rq, se); if (sleep) { - se->peer_preempt = 0; #ifdef CONFIG_SCHEDSTATS if (entity_is_task(se)) { struct task_struct *tsk = task_of(se); @@ -574,10 +573,8 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) ideal_runtime = sched_slice(cfs_rq, curr); delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; - if (delta_exec > ideal_runtime || - (sched_feat(PREEMPT_RESTRICT) && curr->peer_preempt)) + if (delta_exec > ideal_runtime) resched_task(rq_of(cfs_rq)->curr); - curr->peer_preempt = 0; } static void @@ -867,9 +864,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) gran = calc_delta_fair(gran, &se->load); if (delta > gran) { - int now = !sched_feat(PREEMPT_RESTRICT); - - if (now || p->prio < curr->prio || !se->peer_preempt++) + if (p->prio < curr->prio) resched_task(curr); } } @@ -1083,7 +1078,6 @@ static void task_new_fair(struct rq *rq, struct task_struct *p) swap(curr->vruntime, se->vruntime); } - se->peer_preempt = 0; enqueue_task_fair(rq, p, 0); resched_task(rq->curr); } -- cgit v1.2.3 From 8bc6767acb3236e0345e99cf198168e60e7ae456 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 9 Nov 2007 22:39:39 +0100 Subject: sched: wakeup preemption fix wakeup preemption fix: do not make it dependent on p->prio. Preemption purely depends on ->vruntime. This improves preemption in mixed-nice-level workloads. Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index fbcb426029d..a3badf52bba 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -863,10 +863,8 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) if (unlikely(se->load.weight != NICE_0_LOAD)) gran = calc_delta_fair(gran, &se->load); - if (delta > gran) { - if (p->prio < curr->prio) - resched_task(curr); - } + if (delta > gran) + resched_task(curr); } } -- cgit v1.2.3 From 77d9cc44b543fa831169e54c495ad06ef3a0c726 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 9 Nov 2007 22:39:39 +0100 Subject: sched: clean up the wakeup preempt check clean up the wakeup preemption check. No code changed: text data bss dec hex filename 44227 3326 36 47589 b9e5 sched.o.before 44227 3326 36 47589 b9e5 sched.o.after Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index a3badf52bba..d558716a9ad 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -852,20 +852,21 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) if (unlikely(p->policy == SCHED_BATCH)) return; - if (sched_feat(WAKEUP_PREEMPT)) { - while (!is_same_group(se, pse)) { - se = parent_entity(se); - pse = parent_entity(pse); - } - - delta = se->vruntime - pse->vruntime; - gran = sysctl_sched_wakeup_granularity; - if (unlikely(se->load.weight != NICE_0_LOAD)) - gran = calc_delta_fair(gran, &se->load); + if (!sched_feat(WAKEUP_PREEMPT)) + return; - if (delta > gran) - resched_task(curr); + while (!is_same_group(se, pse)) { + se = parent_entity(se); + pse = parent_entity(pse); } + + delta = se->vruntime - pse->vruntime; + gran = sysctl_sched_wakeup_granularity; + if (unlikely(se->load.weight != NICE_0_LOAD)) + gran = calc_delta_fair(gran, &se->load); + + if (delta > gran) + resched_task(curr); } static struct task_struct *pick_next_task_fair(struct rq *rq) -- cgit v1.2.3 From 502d26b524d8980f3ed80d9aec398e85671a8160 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 9 Nov 2007 22:39:39 +0100 Subject: sched: clean up the wakeup preempt check, #2 clean up the preemption check to not use unnecessary 64-bit variables. This improves code size: text data bss dec hex filename 44227 3326 36 47589 b9e5 sched.o.before 44201 3326 36 47563 b9cb sched.o.after Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index d558716a9ad..6c361472cc7 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -837,7 +837,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) struct task_struct *curr = rq->curr; struct cfs_rq *cfs_rq = task_cfs_rq(curr); struct sched_entity *se = &curr->se, *pse = &p->se; - s64 delta, gran; + unsigned long gran; if (unlikely(rt_prio(p->prio))) { update_rq_clock(rq); @@ -860,12 +860,11 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) pse = parent_entity(pse); } - delta = se->vruntime - pse->vruntime; gran = sysctl_sched_wakeup_granularity; if (unlikely(se->load.weight != NICE_0_LOAD)) gran = calc_delta_fair(gran, &se->load); - if (delta > gran) + if (pse->vruntime + gran < se->vruntime) resched_task(curr); } -- cgit v1.2.3 From 3c90e6e99b08f01d5684a3a07cceae6a543e4fa8 Mon Sep 17 00:00:00 2001 From: Srivatsa Vaddagiri Date: Fri, 9 Nov 2007 22:39:39 +0100 Subject: sched: fix copy_namespace() <-> sched_fork() dependency in do_fork Sukadev Bhattiprolu reported a kernel crash with control groups. There are couple of problems discovered by Suka's test: - The test requires the cgroup filesystem to be mounted with atleast the cpu and ns options (i.e both namespace and cpu controllers are active in the same hierarchy). # mkdir /dev/cpuctl # mount -t cgroup -ocpu,ns none cpuctl (or simply) # mount -t cgroup none cpuctl -> Will activate all controllers in same hierarchy. - The test invokes clone() with CLONE_NEWNS set. This causes a a new child to be created, also a new group (do_fork->copy_namespaces->ns_cgroup_clone-> cgroup_clone) and the child is attached to the new group (cgroup_clone-> attach_task->sched_move_task). At this point in time, the child's scheduler related fields are uninitialized (including its on_rq field, which it has inherited from parent). As a result sched_move_task thinks its on runqueue, when it isn't. As a solution to this problem, I moved sched_fork() call, which initializes scheduler related fields on a new task, before copy_namespaces(). I am not sure though whether moving up will cause other side-effects. Do you see any issue? - The second problem exposed by this test is that task_new_fair() assumes that parent and child will be part of the same group (which needn't be as this test shows). As a result, cfs_rq->curr can be NULL for the child. The solution is to test for curr pointer being NULL in task_new_fair(). With the patch below, I could run ns_exec() fine w/o a crash. Reported-by: Sukadev Bhattiprolu Signed-off-by: Srivatsa Vaddagiri Signed-off-by: Ingo Molnar --- kernel/fork.c | 6 +++--- kernel/sched_fair.c | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 28a74015198..8ca1a14cdc8 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1123,6 +1123,9 @@ static struct task_struct *copy_process(unsigned long clone_flags, p->blocked_on = NULL; /* not blocked yet */ #endif + /* Perform scheduler related setup. Assign this task to a CPU. */ + sched_fork(p, clone_flags); + if ((retval = security_task_alloc(p))) goto bad_fork_cleanup_policy; if ((retval = audit_alloc(p))) @@ -1212,9 +1215,6 @@ static struct task_struct *copy_process(unsigned long clone_flags, INIT_LIST_HEAD(&p->ptrace_children); INIT_LIST_HEAD(&p->ptrace_list); - /* Perform scheduler related setup. Assign this task to a CPU. */ - sched_fork(p, clone_flags); - /* Now that the task is set up, run cgroup callbacks if * necessary. We need to run them before the task is visible * on the tasklist. */ diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 6c361472cc7..d3c03070872 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -1067,8 +1067,9 @@ static void task_new_fair(struct rq *rq, struct task_struct *p) update_curr(cfs_rq); place_entity(cfs_rq, se, 1); + /* 'curr' will be NULL if the child belongs to a different group */ if (sysctl_sched_child_runs_first && this_cpu == task_cpu(p) && - curr->vruntime < se->vruntime) { + curr && curr->vruntime < se->vruntime) { /* * Upon rescheduling, sched_class::put_prev_task() will place * 'current' within the tree based on its new key value. -- cgit v1.2.3 From b82d9fdd848abfbe7263a4ecd9bbb55e575100a6 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 9 Nov 2007 22:39:39 +0100 Subject: sched: avoid large irq-latencies in smp-balancing SMP balancing is done with IRQs disabled and can iterate the full rq. When rqs are large this can cause large irq-latencies. Limit the nr of iterations on each run. This fixes a scheduling latency regression reported by the -rt folks. Signed-off-by: Peter Zijlstra Acked-by: Steven Rostedt Tested-by: Gregory Haskins Signed-off-by: Ingo Molnar --- kernel/sched.c | 15 ++++++++++----- kernel/sysctl.c | 8 ++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 2a107e4ad5e..e195a422941 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -471,6 +471,12 @@ const_debug unsigned int sysctl_sched_features = #define sched_feat(x) (sysctl_sched_features & SCHED_FEAT_##x) +/* + * Number of tasks to iterate in a single balance run. + * Limited because this is done with IRQs disabled. + */ +const_debug unsigned int sysctl_sched_nr_migrate = 32; + /* * For kernel-internal use: high-speed (but slightly incorrect) per-cpu * clock constructed from sched_clock(): @@ -2235,7 +2241,7 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, enum cpu_idle_type idle, int *all_pinned, int *this_best_prio, struct rq_iterator *iterator) { - int pulled = 0, pinned = 0, skip_for_load; + int loops = 0, pulled = 0, pinned = 0, skip_for_load; struct task_struct *p; long rem_load_move = max_load_move; @@ -2249,10 +2255,10 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, */ p = iterator->start(iterator->arg); next: - if (!p) + if (!p || loops++ > sysctl_sched_nr_migrate) goto out; /* - * To help distribute high priority tasks accross CPUs we don't + * To help distribute high priority tasks across CPUs we don't * skip a task if it will be the highest priority task (i.e. smallest * prio value) on its new queue regardless of its load weight */ @@ -2269,8 +2275,7 @@ next: rem_load_move -= p->se.load.weight; /* - * We only want to steal up to the prescribed number of tasks - * and the prescribed amount of weighted load. + * We only want to steal up to the prescribed amount of weighted load. */ if (rem_load_move > 0) { if (p->prio < *this_best_prio) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index adddf682d4c..3a1744fed2b 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -301,6 +301,14 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = &proc_dointvec, }, + { + .ctl_name = CTL_UNNUMBERED, + .procname = "sched_nr_migrate", + .data = &sysctl_sched_nr_migrate, + .maxlen = sizeof(unsigned int), + .mode = 644, + .proc_handler = &proc_dointvec, + }, #endif { .ctl_name = CTL_UNNUMBERED, -- cgit v1.2.3 From e6fe6649b4ec11aa3075e394b4d8743eebe1f64c Mon Sep 17 00:00:00 2001 From: Adrian Bunk Date: Fri, 9 Nov 2007 22:39:39 +0100 Subject: sched: proper prototype for kernel/sched.c:migration_init() This patch adds a proper prototype for migration_init() in include/linux/sched.h Since there's no point in always returning 0 to a caller that doesn't check the return value it also changes the function to return void. Signed-off-by: Adrian Bunk Signed-off-by: Andrew Morton Signed-off-by: Ingo Molnar --- kernel/sched.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index e195a422941..b18f231a487 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -5650,7 +5650,7 @@ static struct notifier_block __cpuinitdata migration_notifier = { .priority = 10 }; -int __init migration_init(void) +void __init migration_init(void) { void *cpu = (void *)(long)smp_processor_id(); int err; @@ -5660,8 +5660,6 @@ int __init migration_init(void) BUG_ON(err == NOTIFY_BAD); migration_call(&migration_notifier, CPU_ONLINE, cpu); register_cpu_notifier(&migration_notifier); - - return 0; } #endif -- cgit v1.2.3 From 3c5fd9c77d609b51c0bab682c9d40cbb496ec6f1 Mon Sep 17 00:00:00 2001 From: David Miller Date: Tue, 6 Nov 2007 21:13:56 -0800 Subject: [FUTEX] Fix address computation in compat code. compat_exit_robust_list() computes a pointer to the futex entry in userspace as follows: (void __user *)entry + futex_offset 'entry' is a 'struct robust_list __user *', and 'futex_offset' is a 'compat_long_t' (typically a 's32'). Things explode if the 32-bit sign bit is set in futex_offset. Type promotion sign extends futex_offset to a 64-bit value before adding it to 'entry'. This triggered a problem on sparc64 running 32-bit applications which would lock up a cpu looping forever in the fault handling for the userspace load in handle_futex_death(). Compat userspace runs with address masking (wherein the cpu zeros out the top 32-bits of every effective address given to a memory operation instruction) so the sparc64 fault handler accounts for this by zero'ing out the top 32-bits of the fault address too. Since the kernel properly uses the compat_uptr interfaces, kernel side accesses to compat userspace work too since they will only use addresses with the top 32-bit clear. Because of this compat futex layer bug we get into the following loop when executing the get_user() load near the top of handle_futex_death(): 1) load from address '0xfffffffff7f16bd8', FAULT 2) fault handler clears upper 32-bits, processes fault for address '0xf7f16bd8' which succeeds 3) goto #1 I want to thank Bernd Zeimetz, Josip Rodin, and Fabio Massimo Di Nitto for their tireless efforts helping me track down this bug. Signed-off-by: David S. Miller Signed-off-by: Linus Torvalds --- kernel/futex_compat.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/futex_compat.c b/kernel/futex_compat.c index 00b572666cc..0a43def6fee 100644 --- a/kernel/futex_compat.c +++ b/kernel/futex_compat.c @@ -30,6 +30,15 @@ fetch_robust_entry(compat_uptr_t *uentry, struct robust_list __user **entry, return 0; } +static void __user *futex_uaddr(struct robust_list *entry, + compat_long_t futex_offset) +{ + compat_uptr_t base = ptr_to_compat(entry); + void __user *uaddr = compat_ptr(base + futex_offset); + + return uaddr; +} + /* * Walk curr->robust_list (very carefully, it's a userspace list!) * and mark any locks found there dead, and notify any waiters. @@ -76,11 +85,12 @@ void compat_exit_robust_list(struct task_struct *curr) * A pending lock might already be on the list, so * dont process it twice: */ - if (entry != pending) - if (handle_futex_death((void __user *)entry + futex_offset, - curr, pi)) - return; + if (entry != pending) { + void __user *uaddr = futex_uaddr(entry, futex_offset); + if (handle_futex_death(uaddr, curr, pi)) + return; + } if (rc) return; uentry = next_uentry; @@ -94,9 +104,11 @@ void compat_exit_robust_list(struct task_struct *curr) cond_resched(); } - if (pending) - handle_futex_death((void __user *)pending + futex_offset, - curr, pip); + if (pending) { + void __user *uaddr = futex_uaddr(pending, futex_offset); + + handle_futex_death(uaddr, curr, pip); + } } asmlinkage long -- cgit v1.2.3 From 325d22df7b19e0116aff3391d3a03f73d0634ded Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Mon, 12 Nov 2007 15:41:55 -0800 Subject: sigwait eats blocked default-ignore signals While a signal is blocked, it must be posted even if its action is SIG_IGN or is SIG_DFL with the default action to ignore. This works right most of the time, but is broken when a sigwait (rt_sigtimedwait) is in progress. This changes the early-discard check to respect real_blocked. ~blocked is the set to check for "should wake up now", but ~(blocked|real_blocked) is the set for "blocked" semantics as defined by POSIX. This fixes bugzilla entry 9347, see http://bugzilla.kernel.org/show_bug.cgi?id=9347 Signed-off-by: Roland McGrath Signed-off-by: Linus Torvalds --- kernel/signal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/signal.c b/kernel/signal.c index 909a0cc6bc7..afa4f781f92 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -55,7 +55,7 @@ static int sig_ignored(struct task_struct *t, int sig) * signal handler may change by the time it is * unblocked. */ - if (sigismember(&t->blocked, sig)) + if (sigismember(&t->blocked, sig) || sigismember(&t->real_blocked, sig)) return 0; /* Is it explicitly or implicitly ignored? */ -- cgit v1.2.3 From ce1d18e0064d55106a7042c07cfca97cad66f407 Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Tue, 13 Nov 2007 21:15:24 -0800 Subject: [SYSCTL]: Fix warning for token-ring from sysctl checker As seen when booting ppc64_defconfig: sysctl table check failed: /net/token-ring .3.14 procname does not match binary path procname Signed-off-by: Olof Johansson Signed-off-by: David S. Miller --- kernel/sysctl_check.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index 5a2f2b2bf88..4abc6d2306f 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c @@ -738,7 +738,7 @@ static struct trans_ctl_table trans_net_table[] = { { NET_ROSE, "rose", trans_net_rose_table }, { NET_IPV6, "ipv6", trans_net_ipv6_table }, { NET_X25, "x25", trans_net_x25_table }, - { NET_TR, "tr", trans_net_tr_table }, + { NET_TR, "token-ring", trans_net_tr_table }, { NET_DECNET, "decnet", trans_net_decnet_table }, /* NET_ECONET not used */ { NET_SCTP, "sctp", trans_net_sctp_table }, -- cgit v1.2.3 From 6fc48af82cef55546d640778698943b6227b7fb0 Mon Sep 17 00:00:00 2001 From: Tetsuo Handa Date: Wed, 14 Nov 2007 16:58:38 -0800 Subject: sysctl: check length at deprecated_sysctl_warning Original patch assumed args->nlen < CTL_MAXNAME, but it can be false. Signed-off-by: Tetsuo Handa Cc: "Eric W. Biederman" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/sysctl.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 3a1744fed2b..0deed82a615 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2620,6 +2620,10 @@ static int deprecated_sysctl_warning(struct __sysctl_args *args) int name[CTL_MAXNAME]; int i; + /* Check args->nlen. */ + if (args->nlen < 0 || args->nlen > CTL_MAXNAME) + return -ENOTDIR; + /* Read in the sysctl name for better debug message logging */ for (i = 0; i < args->nlen; i++) if (get_user(name[i], args->name + i)) -- cgit v1.2.3 From cfe36bde59bc1ae868e775ad82386c3acaabb738 Mon Sep 17 00:00:00 2001 From: Diego Calleja Date: Wed, 14 Nov 2007 16:58:54 -0800 Subject: Improve cgroup printks When I boot with the 'quiet' parameter, I see on the screen: [ 0.000000] Initializing cgroup subsys cpuset [ 0.000000] Initializing cgroup subsys cpu [ 39.036026] Initializing cgroup subsys cpuacct [ 39.036080] Initializing cgroup subsys debug [ 39.036118] Initializing cgroup subsys ns This patch lowers the priority of those messages, adds a "cgroup: " prefix to another couple of printks and kills the useless reference to the source file. Signed-off-by: Diego Calleja Cc: Paul Menage Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/cgroup.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup.c b/kernel/cgroup.c index 3fe21e19c96..1a3c23936d4 100644 --- a/kernel/cgroup.c +++ b/kernel/cgroup.c @@ -1,6 +1,4 @@ /* - * kernel/cgroup.c - * * Generic process-grouping system. * * Based originally on the cpuset system, extracted by Paul Menage @@ -2200,7 +2198,8 @@ static void cgroup_init_subsys(struct cgroup_subsys *ss) { struct cgroup_subsys_state *css; struct list_head *l; - printk(KERN_ERR "Initializing cgroup subsys %s\n", ss->name); + + printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name); /* Create the top cgroup state for this subsystem */ ss->root = &rootnode; @@ -2273,7 +2272,7 @@ int __init cgroup_init_early(void) BUG_ON(!ss->create); BUG_ON(!ss->destroy); if (ss->subsys_id != i) { - printk(KERN_ERR "Subsys %s id == %d\n", + printk(KERN_ERR "cgroup: Subsys %s id == %d\n", ss->name, ss->subsys_id); BUG(); } @@ -2605,7 +2604,7 @@ int cgroup_clone(struct task_struct *tsk, struct cgroup_subsys *subsys) dentry = lookup_one_len(nodename, parent->dentry, strlen(nodename)); if (IS_ERR(dentry)) { printk(KERN_INFO - "Couldn't allocate dentry for %s: %ld\n", nodename, + "cgroup: Couldn't allocate dentry for %s: %ld\n", nodename, PTR_ERR(dentry)); ret = PTR_ERR(dentry); goto out_release; -- cgit v1.2.3 From 887c3cb18865a4f9e0786e5a5b3ef47ff469b956 Mon Sep 17 00:00:00 2001 From: Yasunori Goto Date: Wed, 14 Nov 2007 16:59:20 -0800 Subject: Add IORESOUCE_BUSY flag for System RAM i386 and x86-64 registers System RAM as IORESOURCE_MEM | IORESOURCE_BUSY. But ia64 registers it as IORESOURCE_MEM only. In addition, memory hotplug code registers new memory as IORESOURCE_MEM too. This difference causes a failure of memory unplug of x86-64. This patch fixes it. This patch adds IORESOURCE_BUSY to avoid potential overlap mapping by PCI device. Signed-off-by: Yasunori Goto Signed-off-by: Badari Pulavarty Cc: Luck, Tony" Cc: Thomas Gleixner Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/resource.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/resource.c b/kernel/resource.c index a358142ff48..2eb553d9b51 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -277,7 +277,7 @@ walk_memory_resource(unsigned long start_pfn, unsigned long nr_pages, void *arg, int ret = -1; res.start = (u64) start_pfn << PAGE_SHIFT; res.end = ((u64)(start_pfn + nr_pages) << PAGE_SHIFT) - 1; - res.flags = IORESOURCE_MEM; + res.flags = IORESOURCE_MEM | IORESOURCE_BUSY; orig_end = res.end; while ((res.start < res.end) && (find_next_system_ram(&res) >= 0)) { pfn = (unsigned long)(res.start >> PAGE_SHIFT); -- cgit v1.2.3 From cfb5285660aad4931b2ebbfa902ea48a37dfffa1 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Wed, 14 Nov 2007 16:59:45 -0800 Subject: revert "Task Control Groups: example CPU accounting subsystem" Revert 62d0df64065e7c135d0002f069444fbdfc64768f. This was originally intended as a simple initial example of how to create a control groups subsystem; it wasn't intended for mainline, but I didn't make this clear enough to Andrew. The CFS cgroup subsystem now has better functionality for the per-cgroup usage accounting (based directly on CFS stats) than the "usage" status file in this patch, and the "load" status file is rather simplistic - although having a per-cgroup load average report would be a useful feature, I don't believe this patch actually provides it. If it gets into the final 2.6.24 we'd probably have to support this interface for ever. Cc: Paul Menage Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/Makefile | 1 - kernel/cpu_acct.c | 186 ------------------------------------------------------ kernel/sched.c | 14 +--- 3 files changed, 3 insertions(+), 198 deletions(-) delete mode 100644 kernel/cpu_acct.c (limited to 'kernel') diff --git a/kernel/Makefile b/kernel/Makefile index f60afe74259..dfa96956dae 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -40,7 +40,6 @@ obj-$(CONFIG_COMPAT) += compat.o obj-$(CONFIG_CGROUPS) += cgroup.o obj-$(CONFIG_CGROUP_DEBUG) += cgroup_debug.o obj-$(CONFIG_CPUSETS) += cpuset.o -obj-$(CONFIG_CGROUP_CPUACCT) += cpu_acct.o obj-$(CONFIG_CGROUP_NS) += ns_cgroup.o obj-$(CONFIG_IKCONFIG) += configs.o obj-$(CONFIG_STOP_MACHINE) += stop_machine.o diff --git a/kernel/cpu_acct.c b/kernel/cpu_acct.c deleted file mode 100644 index 731e47e7f16..00000000000 --- a/kernel/cpu_acct.c +++ /dev/null @@ -1,186 +0,0 @@ -/* - * kernel/cpu_acct.c - CPU accounting cgroup subsystem - * - * Copyright (C) Google Inc, 2006 - * - * Developed by Paul Menage (menage@google.com) and Balbir Singh - * (balbir@in.ibm.com) - * - */ - -/* - * Example cgroup subsystem for reporting total CPU usage of tasks in a - * cgroup, along with percentage load over a time interval - */ - -#include -#include -#include -#include - -#include - -struct cpuacct { - struct cgroup_subsys_state css; - spinlock_t lock; - /* total time used by this class */ - cputime64_t time; - - /* time when next load calculation occurs */ - u64 next_interval_check; - - /* time used in current period */ - cputime64_t current_interval_time; - - /* time used in last period */ - cputime64_t last_interval_time; -}; - -struct cgroup_subsys cpuacct_subsys; - -static inline struct cpuacct *cgroup_ca(struct cgroup *cont) -{ - return container_of(cgroup_subsys_state(cont, cpuacct_subsys_id), - struct cpuacct, css); -} - -static inline struct cpuacct *task_ca(struct task_struct *task) -{ - return container_of(task_subsys_state(task, cpuacct_subsys_id), - struct cpuacct, css); -} - -#define INTERVAL (HZ * 10) - -static inline u64 next_interval_boundary(u64 now) -{ - /* calculate the next interval boundary beyond the - * current time */ - do_div(now, INTERVAL); - return (now + 1) * INTERVAL; -} - -static struct cgroup_subsys_state *cpuacct_create( - struct cgroup_subsys *ss, struct cgroup *cont) -{ - struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL); - - if (!ca) - return ERR_PTR(-ENOMEM); - spin_lock_init(&ca->lock); - ca->next_interval_check = next_interval_boundary(get_jiffies_64()); - return &ca->css; -} - -static void cpuacct_destroy(struct cgroup_subsys *ss, - struct cgroup *cont) -{ - kfree(cgroup_ca(cont)); -} - -/* Lazily update the load calculation if necessary. Called with ca locked */ -static void cpuusage_update(struct cpuacct *ca) -{ - u64 now = get_jiffies_64(); - - /* If we're not due for an update, return */ - if (ca->next_interval_check > now) - return; - - if (ca->next_interval_check <= (now - INTERVAL)) { - /* If it's been more than an interval since the last - * check, then catch up - the last interval must have - * been zero load */ - ca->last_interval_time = 0; - ca->next_interval_check = next_interval_boundary(now); - } else { - /* If a steal takes the last interval time negative, - * then we just ignore it */ - if ((s64)ca->current_interval_time > 0) - ca->last_interval_time = ca->current_interval_time; - else - ca->last_interval_time = 0; - ca->next_interval_check += INTERVAL; - } - ca->current_interval_time = 0; -} - -static u64 cpuusage_read(struct cgroup *cont, struct cftype *cft) -{ - struct cpuacct *ca = cgroup_ca(cont); - u64 time; - - spin_lock_irq(&ca->lock); - cpuusage_update(ca); - time = cputime64_to_jiffies64(ca->time); - spin_unlock_irq(&ca->lock); - - /* Convert 64-bit jiffies to seconds */ - time *= 1000; - do_div(time, HZ); - return time; -} - -static u64 load_read(struct cgroup *cont, struct cftype *cft) -{ - struct cpuacct *ca = cgroup_ca(cont); - u64 time; - - /* Find the time used in the previous interval */ - spin_lock_irq(&ca->lock); - cpuusage_update(ca); - time = cputime64_to_jiffies64(ca->last_interval_time); - spin_unlock_irq(&ca->lock); - - /* Convert time to a percentage, to give the load in the - * previous period */ - time *= 100; - do_div(time, INTERVAL); - - return time; -} - -static struct cftype files[] = { - { - .name = "usage", - .read_uint = cpuusage_read, - }, - { - .name = "load", - .read_uint = load_read, - } -}; - -static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cont) -{ - return cgroup_add_files(cont, ss, files, ARRAY_SIZE(files)); -} - -void cpuacct_charge(struct task_struct *task, cputime_t cputime) -{ - - struct cpuacct *ca; - unsigned long flags; - - if (!cpuacct_subsys.active) - return; - rcu_read_lock(); - ca = task_ca(task); - if (ca) { - spin_lock_irqsave(&ca->lock, flags); - cpuusage_update(ca); - ca->time = cputime64_add(ca->time, cputime); - ca->current_interval_time = - cputime64_add(ca->current_interval_time, cputime); - spin_unlock_irqrestore(&ca->lock, flags); - } - rcu_read_unlock(); -} - -struct cgroup_subsys cpuacct_subsys = { - .name = "cpuacct", - .create = cpuacct_create, - .destroy = cpuacct_destroy, - .populate = cpuacct_populate, - .subsys_id = cpuacct_subsys_id, -}; diff --git a/kernel/sched.c b/kernel/sched.c index b18f231a487..4fb3532dd7e 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -52,7 +52,6 @@ #include #include #include -#include #include #include #include @@ -3338,13 +3337,9 @@ void account_user_time(struct task_struct *p, cputime_t cputime) { struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; cputime64_t tmp; - struct rq *rq = this_rq(); p->utime = cputime_add(p->utime, cputime); - if (p != rq->idle) - cpuacct_charge(p, cputime); - /* Add user time to cpustat. */ tmp = cputime_to_cputime64(cputime); if (TASK_NICE(p) > 0) @@ -3408,10 +3403,9 @@ void account_system_time(struct task_struct *p, int hardirq_offset, cpustat->irq = cputime64_add(cpustat->irq, tmp); else if (softirq_count()) cpustat->softirq = cputime64_add(cpustat->softirq, tmp); - else if (p != rq->idle) { + else if (p != rq->idle) cpustat->system = cputime64_add(cpustat->system, tmp); - cpuacct_charge(p, cputime); - } else if (atomic_read(&rq->nr_iowait) > 0) + else if (atomic_read(&rq->nr_iowait) > 0) cpustat->iowait = cputime64_add(cpustat->iowait, tmp); else cpustat->idle = cputime64_add(cpustat->idle, tmp); @@ -3447,10 +3441,8 @@ void account_steal_time(struct task_struct *p, cputime_t steal) cpustat->iowait = cputime64_add(cpustat->iowait, tmp); else cpustat->idle = cputime64_add(cpustat->idle, tmp); - } else { + } else cpustat->steal = cputime64_add(cpustat->steal, tmp); - cpuacct_charge(p, -tmp); - } } /* -- cgit v1.2.3 From 314de8a9e17f70243eacc80d2dd22a5d74b09fce Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Wed, 14 Nov 2007 16:59:48 -0800 Subject: Linux Kernel Markers: fix marker mutex not taken upon module load Upon module load, we must take the markers mutex. It implies that the marker mutex must be nested inside the module mutex. It implies changing the nesting order : now the marker mutex nests inside the module mutex. Make the necessary changes to reverse the order in which the mutexes are taken. Includes some cleanup from Dave Hansen . Signed-off-by: Mathieu Desnoyers Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/marker.c | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) (limited to 'kernel') diff --git a/kernel/marker.c b/kernel/marker.c index ccb48d9a365..5323cfaedbc 100644 --- a/kernel/marker.c +++ b/kernel/marker.c @@ -28,7 +28,7 @@ extern struct marker __start___markers[]; extern struct marker __stop___markers[]; /* - * module_mutex nests inside markers_mutex. Markers mutex protects the builtin + * markers_mutex nests inside module_mutex. Markers mutex protects the builtin * and module markers, the hash table and deferred_sync. */ static DEFINE_MUTEX(markers_mutex); @@ -257,7 +257,6 @@ static void disable_marker(struct marker *elem) * @refcount: number of references left to the given probe_module (out) * * Updates the probe callback corresponding to a range of markers. - * Must be called with markers_mutex held. */ void marker_update_probe_range(struct marker *begin, struct marker *end, struct module *probe_module, @@ -266,6 +265,7 @@ void marker_update_probe_range(struct marker *begin, struct marker *iter; struct marker_entry *mark_entry; + mutex_lock(&markers_mutex); for (iter = begin; iter < end; iter++) { mark_entry = get_marker(iter->name); if (mark_entry && mark_entry->refcount) { @@ -281,6 +281,7 @@ void marker_update_probe_range(struct marker *begin, disable_marker(iter); } } + mutex_unlock(&markers_mutex); } /* @@ -293,7 +294,6 @@ static void marker_update_probes(struct module *probe_module) { int refcount = 0; - mutex_lock(&markers_mutex); /* Core kernel markers */ marker_update_probe_range(__start___markers, __stop___markers, probe_module, &refcount); @@ -303,7 +303,6 @@ static void marker_update_probes(struct module *probe_module) synchronize_sched(); deferred_sync = 0; } - mutex_unlock(&markers_mutex); } /** @@ -320,7 +319,7 @@ int marker_probe_register(const char *name, const char *format, marker_probe_func *probe, void *private) { struct marker_entry *entry; - int ret = 0, need_update = 0; + int ret = 0; mutex_lock(&markers_mutex); entry = get_marker(name); @@ -335,11 +334,11 @@ int marker_probe_register(const char *name, const char *format, ret = add_marker(name, format, probe, private); if (ret) goto end; - need_update = 1; + mutex_unlock(&markers_mutex); + marker_update_probes(NULL); + return ret; end: mutex_unlock(&markers_mutex); - if (need_update) - marker_update_probes(NULL); return ret; } EXPORT_SYMBOL_GPL(marker_probe_register); @@ -355,7 +354,6 @@ void *marker_probe_unregister(const char *name) struct module *probe_module; struct marker_entry *entry; void *private; - int need_update = 0; mutex_lock(&markers_mutex); entry = get_marker(name); @@ -368,11 +366,11 @@ void *marker_probe_unregister(const char *name) probe_module = __module_text_address((unsigned long)entry->probe); private = remove_marker(name); deferred_sync = 1; - need_update = 1; + mutex_unlock(&markers_mutex); + marker_update_probes(probe_module); + return private; end: mutex_unlock(&markers_mutex); - if (need_update) - marker_update_probes(probe_module); return private; } EXPORT_SYMBOL_GPL(marker_probe_unregister); @@ -392,7 +390,6 @@ void *marker_probe_unregister_private_data(void *private) struct marker_entry *entry; int found = 0; unsigned int i; - int need_update = 0; mutex_lock(&markers_mutex); for (i = 0; i < MARKER_TABLE_SIZE; i++) { @@ -414,11 +411,11 @@ iter_end: probe_module = __module_text_address((unsigned long)entry->probe); private = remove_marker(entry->name); deferred_sync = 1; - need_update = 1; + mutex_unlock(&markers_mutex); + marker_update_probes(probe_module); + return private; end: mutex_unlock(&markers_mutex); - if (need_update) - marker_update_probes(probe_module); return private; } EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data); @@ -434,7 +431,7 @@ EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data); int marker_arm(const char *name) { struct marker_entry *entry; - int ret = 0, need_update = 0; + int ret = 0; mutex_lock(&markers_mutex); entry = get_marker(name); @@ -447,11 +444,9 @@ int marker_arm(const char *name) */ if (entry->refcount++) goto end; - need_update = 1; end: mutex_unlock(&markers_mutex); - if (need_update) - marker_update_probes(NULL); + marker_update_probes(NULL); return ret; } EXPORT_SYMBOL_GPL(marker_arm); @@ -467,7 +462,7 @@ EXPORT_SYMBOL_GPL(marker_arm); int marker_disarm(const char *name) { struct marker_entry *entry; - int ret = 0, need_update = 0; + int ret = 0; mutex_lock(&markers_mutex); entry = get_marker(name); @@ -486,11 +481,9 @@ int marker_disarm(const char *name) ret = -EPERM; goto end; } - need_update = 1; end: mutex_unlock(&markers_mutex); - if (need_update) - marker_update_probes(NULL); + marker_update_probes(NULL); return ret; } EXPORT_SYMBOL_GPL(marker_disarm); -- cgit v1.2.3 From 22800a2830ec07e7cc5c837999890ac47cc7f5de Mon Sep 17 00:00:00 2001 From: Jan Kiszka Date: Wed, 14 Nov 2007 17:00:08 -0800 Subject: fix param_sysfs_builtin name length check Commit faf8c714f4508207a9c81cc94dafc76ed6680b44 caused a regression: parameter names longer than MAX_KBUILD_MODNAME will now be rejected, although we just need to keep the module name part that short. This patch restores the old behaviour while still avoiding that memchr is called with its length parameter larger than the total string length. Signed-off-by: Jan Kiszka Cc: Dave Young Cc: Greg KH Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/params.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/params.c b/kernel/params.c index 16f269e9ddc..2a4c51487e7 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -592,19 +592,16 @@ static void __init param_sysfs_builtin(void) for (i=0; i < __stop___param - __start___param; i++) { char *dot; - size_t kplen; + size_t max_name_len; kp = &__start___param[i]; - kplen = strlen(kp->name); + max_name_len = + min_t(size_t, MAX_KBUILD_MODNAME, strlen(kp->name)); - /* We do not handle args without periods. */ - if (kplen > MAX_KBUILD_MODNAME) { - DEBUGP("kernel parameter name is too long: %s\n", kp->name); - continue; - } - dot = memchr(kp->name, '.', kplen); + dot = memchr(kp->name, '.', max_name_len); if (!dot) { - DEBUGP("couldn't find period in %s\n", kp->name); + DEBUGP("couldn't find period in first %d characters " + "of %s\n", MAX_KBUILD_MODNAME, kp->name); continue; } name_len = dot - kp->name; -- cgit v1.2.3 From 57d5f66b86079efac5c9a7843cce2a9bcbe58fb8 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Wed, 14 Nov 2007 17:00:13 -0800 Subject: pidns: Place under CONFIG_EXPERIMENTAL This is my trivial patch to swat innumerable little bugs with a single blow. After some intensive review (my apologies for not having gotten to this sooner) what we have looks like a good base to build on with the current pid namespace code but it is not complete, and it is still much to simple to find issues where the kernel does the wrong thing outside of the initial pid namespace. Until the dust settles and we are certain we have the ABI and the implementation is as correct as humanly possible let's keep process ID namespaces behind CONFIG_EXPERIMENTAL. Allowing us the option of fixing any ABI or other bugs we find as long as they are minor. Allowing users of the kernel to avoid those bugs simply by ensuring their kernel does not have support for multiple pid namespaces. [akpm@linux-foundation.org: coding-style cleanups] Signed-off-by: Eric W. Biederman Cc: Cedric Le Goater Cc: Adrian Bunk Cc: Jeremy Fitzhardinge Cc: Kir Kolyshkin Cc: Kirill Korotaev Cc: Pavel Emelyanov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/pid.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/pid.c b/kernel/pid.c index d1db36b9467..f815455431b 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -537,6 +537,7 @@ err_alloc: return NULL; } +#ifdef CONFIG_PID_NS static struct pid_namespace *create_pid_namespace(int level) { struct pid_namespace *ns; @@ -621,6 +622,7 @@ void free_pid_ns(struct kref *kref) if (parent != NULL) put_pid_ns(parent); } +#endif /* CONFIG_PID_NS */ void zap_pid_ns_processes(struct pid_namespace *pid_ns) { -- cgit v1.2.3 From c642b8391cf8efc3622cc97329a0f46e7cbb70b8 Mon Sep 17 00:00:00 2001 From: Russ Anderson Date: Wed, 14 Nov 2007 17:00:15 -0800 Subject: __do_IRQ does not check IRQ_DISABLED when IRQ_PER_CPU is set In __do_IRQ(), the normal case is that IRQ_DISABLED is checked and if set the handler (handle_IRQ_event()) is not called. Earlier in __do_IRQ(), if IRQ_PER_CPU is set the code does not check IRQ_DISABLED and calls the handler even though IRQ_DISABLED is set. This behavior seems unintentional. One user encountering this behavior is the CPE handler (in arch/ia64/kernel/mca.c). When the CPE handler encounters too many CPEs (such as a solid single bit error), it sets up a polling timer and disables the CPE interrupt (to avoid excessive overhead logging the stream of single bit errors). disable_irq_nosync() is called which sets IRQ_DISABLED. The IRQ_PER_CPU flag was previously set (in ia64_mca_late_init()). The net result is the CPE handler gets called even though it is marked disabled. If the behavior of not checking IRQ_DISABLED when IRQ_PER_CPU is set is intentional, it would be worthy of a comment describing the intended behavior. disable_irq_nosync() does call chip->disable() to provide a chipset specifiec interface for disabling the interrupt, which avoids this issue when used. Signed-off-by: Russ Anderson Cc: "Luck, Tony" Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Bjorn Helgaas Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/irq/handle.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index e391cbb1f56..dc335ad2752 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c @@ -178,9 +178,11 @@ fastcall unsigned int __do_IRQ(unsigned int irq) */ if (desc->chip->ack) desc->chip->ack(irq); - action_ret = handle_IRQ_event(irq, desc->action); - if (!noirqdebug) - note_interrupt(irq, desc, action_ret); + if (likely(!(desc->status & IRQ_DISABLED))) { + action_ret = handle_IRQ_event(irq, desc->action); + if (!noirqdebug) + note_interrupt(irq, desc, action_ret); + } desc->chip->end(irq); return 1; } -- cgit v1.2.3 From 60a0d23386eab0559ad32ae50b200cc58545f327 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Wed, 14 Nov 2007 17:00:16 -0800 Subject: hibernate: fix lockdep report Lockdep reports a circular locking dependency in the hibernate code because - during system boot hibernate code (from an initcall) locks pm_mutex and then a sysfs buffer mutex via name_to_dev_t - during regular operation hibernate code locks pm_mutex under a sysfs buffer mutex because it's called from sysfs methods. The deadlock can never happen because during initcall invocation nothing can write to sysfs yet. This removes the lockdep report by marking the initcall locking as being in a different class. Signed-off-by: Johannes Berg Cc: "Rafael J. Wysocki" Cc: Alan Stern Acked-by: Peter Zijlstra Cc: Pavel Machek Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/power/disk.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/power/disk.c b/kernel/power/disk.c index 8b15f777010..05b64790fe8 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c @@ -456,7 +456,17 @@ static int software_resume(void) int error; unsigned int flags; - mutex_lock(&pm_mutex); + /* + * name_to_dev_t() below takes a sysfs buffer mutex when sysfs + * is configured into the kernel. Since the regular hibernate + * trigger path is via sysfs which takes a buffer mutex before + * calling hibernate functions (which take pm_mutex) this can + * cause lockdep to complain about a possible ABBA deadlock + * which cannot happen since we're in the boot code here and + * sysfs can't be invoked yet. Therefore, we use a subclass + * here to avoid lockdep complaining. + */ + mutex_lock_nested(&pm_mutex, SINGLE_DEPTH_NESTING); if (!swsusp_resume_device) { if (!strlen(resume_file)) { mutex_unlock(&pm_mutex); -- cgit v1.2.3 From f96159840bc5f605aca5113ab2d24308d3dc2eff Mon Sep 17 00:00:00 2001 From: Adrian Bunk Date: Wed, 14 Nov 2007 17:00:37 -0800 Subject: kernel/taskstats.c: fix bogus nlmsg_free() We'd better not nlmsg_free on a pointer containing an undefined value (and without having anything allocated). Spotted by the Coverity checker. Signed-off-by: Adrian Bunk Acked-by: Balbir Singh Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/taskstats.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'kernel') diff --git a/kernel/taskstats.c b/kernel/taskstats.c index 354e74bc17c..07e86a82807 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c @@ -398,31 +398,31 @@ static int cgroupstats_user_cmd(struct sk_buff *skb, struct genl_info *info) fd = nla_get_u32(info->attrs[CGROUPSTATS_CMD_ATTR_FD]); file = fget_light(fd, &fput_needed); - if (file) { - size = nla_total_size(sizeof(struct cgroupstats)); + if (!file) + return 0; - rc = prepare_reply(info, CGROUPSTATS_CMD_NEW, &rep_skb, - size); - if (rc < 0) - goto err; + size = nla_total_size(sizeof(struct cgroupstats)); - na = nla_reserve(rep_skb, CGROUPSTATS_TYPE_CGROUP_STATS, - sizeof(struct cgroupstats)); - stats = nla_data(na); - memset(stats, 0, sizeof(*stats)); + rc = prepare_reply(info, CGROUPSTATS_CMD_NEW, &rep_skb, + size); + if (rc < 0) + goto err; - rc = cgroupstats_build(stats, file->f_dentry); - if (rc < 0) - goto err; + na = nla_reserve(rep_skb, CGROUPSTATS_TYPE_CGROUP_STATS, + sizeof(struct cgroupstats)); + stats = nla_data(na); + memset(stats, 0, sizeof(*stats)); - fput_light(file, fput_needed); - return send_reply(rep_skb, info->snd_pid); + rc = cgroupstats_build(stats, file->f_dentry); + if (rc < 0) { + nlmsg_free(rep_skb); + goto err; } + rc = send_reply(rep_skb, info->snd_pid); + err: - if (file) - fput_light(file, fput_needed); - nlmsg_free(rep_skb); + fput_light(file, fput_needed); return rc; } -- cgit v1.2.3 From a3474224e6a01924be40a8255636ea5522c1023a Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Tue, 13 Nov 2007 22:11:50 -0800 Subject: wait_task_stopped: Check p->exit_state instead of TASK_TRACED The original meaning of the old test (p->state > TASK_STOPPED) was "not dead", since it was before TASK_TRACED existed and before the state/exit_state split. It was a wrong correction in commit 14bf01bb0599c89fc7f426d20353b76e12555308 to make this test for TASK_TRACED instead. It should have been changed when TASK_TRACED was introducted and again when exit_state was introduced. Signed-off-by: Roland McGrath Cc: Oleg Nesterov Cc: Alexey Dobriyan Cc: Kees Cook Acked-by: Scott James Remnant Signed-off-by: Linus Torvalds --- kernel/exit.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/exit.c b/kernel/exit.c index f1aec27f1df..cd0f1d4137a 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -1386,8 +1386,7 @@ static int wait_task_stopped(struct task_struct *p, int delayed_group_leader, int why = (p->ptrace & PT_PTRACED) ? CLD_TRAPPED : CLD_STOPPED; exit_code = p->exit_code; - if (unlikely(!exit_code) || - unlikely(p->state & TASK_TRACED)) + if (unlikely(!exit_code) || unlikely(p->exit_state)) goto bail_ref; return wait_noreap_copyout(p, pid, uid, why, (exit_code << 8) | 0x7f, -- cgit v1.2.3 From 9778385db35a799d410039be123044a0d3e917a2 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Thu, 15 Nov 2007 20:57:39 +0100 Subject: sched: fix accounting of interrupts during guest execution on s390 Currently the scheduler checks for PF_VCPU to decide if this timeslice has to be accounted as guest time. On s390 host interrupts are not disabled during guest execution. This causes theses interrupts to be accounted as guest time if CONFIG_VIRT_CPU_ACCOUNTING is set. Solution is to check if an interrupt triggered account_system_time. As the tick is timer interrupt based, we have to subtract hardirq_offset. I tested the patch on s390 with CONFIG_VIRT_CPU_ACCOUNTING and on x86_64. Seems to work. CC: Avi Kivity CC: Laurent Vivier Signed-off-by: Christian Borntraeger Signed-off-by: Ingo Molnar --- kernel/sched.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 4fb3532dd7e..908335e1781 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -3390,10 +3390,8 @@ void account_system_time(struct task_struct *p, int hardirq_offset, struct rq *rq = this_rq(); cputime64_t tmp; - if (p->flags & PF_VCPU) { - account_guest_time(p, cputime); - return; - } + if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) + return account_guest_time(p, cputime); p->stime = cputime_add(p->stime, cputime); -- cgit v1.2.3 From dae51f56204d33444f61d9e7af3ee70aef55daa4 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Thu, 15 Nov 2007 20:57:40 +0100 Subject: sched: fix SCHED_FIFO tasks & FAIR_GROUP_SCHED Suppose that the SCHED_FIFO task does switch_uid(new_user); Now, p->se.cfs_rq and p->se.parent both point into the old user_struct->tg because sched_move_task() doesn't call set_task_cfs_rq() for !fair_sched_class case. Suppose that old user_struct/task_group is freed/reused, and the task does sched_setscheduler(SCHED_NORMAL); __setscheduler() sets fair_sched_class, but doesn't update ->se.cfs_rq/parent which point to the freed memory. This means that check_preempt_wakeup() doing while (!is_same_group(se, pse)) { se = parent_entity(se); pse = parent_entity(pse); } may OOPS in a similar way if rq->curr or p did something like above. Perhaps we need something like the patch below, note that __setscheduler() can't do set_task_cfs_rq(). Signed-off-by: Oleg Nesterov Signed-off-by: Ingo Molnar --- kernel/sched.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 908335e1781..d62b759753f 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -7087,8 +7087,10 @@ void sched_move_task(struct task_struct *tsk) rq = task_rq_lock(tsk, &flags); - if (tsk->sched_class != &fair_sched_class) + if (tsk->sched_class != &fair_sched_class) { + set_task_cfs_rq(tsk); goto done; + } update_rq_clock(rq); -- cgit v1.2.3 From ce96b5ac742801718ae86d2adf0500c5abef3782 Mon Sep 17 00:00:00 2001 From: Dmitry Adamushko Date: Thu, 15 Nov 2007 20:57:40 +0100 Subject: sched: fix __set_task_cpu() SMP race Grant Wilson has reported rare SCHED_FAIR_USER crashes on his quad-core system, which crashes can only be explained via runqueue corruption. there is a narrow SMP race in __set_task_cpu(): after ->cpu is set up to a new value, task_rq_lock(p, ...) can be successfuly executed on another CPU. We must ensure that updates of per-task data have been completed by this moment. this bug has been hiding in the Linux scheduler for an eternity (we never had any explicit barrier for task->cpu in set_task_cpu() - so the bug was introduced in 2.5.1), but only became visible via set_task_cfs_rq() being accidentally put after the task->cpu update. It also probably needs a sufficiently out-of-order CPU to trigger. Reported-by: Grant Wilson Signed-off-by: Dmitry Adamushko Signed-off-by: Ingo Molnar --- kernel/sched.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index d62b759753f..db1f71e3131 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -216,15 +216,15 @@ static inline struct task_group *task_group(struct task_struct *p) } /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ -static inline void set_task_cfs_rq(struct task_struct *p) +static inline void set_task_cfs_rq(struct task_struct *p, unsigned int cpu) { - p->se.cfs_rq = task_group(p)->cfs_rq[task_cpu(p)]; - p->se.parent = task_group(p)->se[task_cpu(p)]; + p->se.cfs_rq = task_group(p)->cfs_rq[cpu]; + p->se.parent = task_group(p)->se[cpu]; } #else -static inline void set_task_cfs_rq(struct task_struct *p) { } +static inline void set_task_cfs_rq(struct task_struct *p, unsigned int cpu) { } #endif /* CONFIG_FAIR_GROUP_SCHED */ @@ -1022,10 +1022,16 @@ unsigned long weighted_cpuload(const int cpu) static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) { + set_task_cfs_rq(p, cpu); #ifdef CONFIG_SMP + /* + * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be + * successfuly executed on another CPU. We must ensure that updates of + * per-task data have been completed by this moment. + */ + smp_wmb(); task_thread_info(p)->cpu = cpu; #endif - set_task_cfs_rq(p); } #ifdef CONFIG_SMP @@ -7088,7 +7094,7 @@ void sched_move_task(struct task_struct *tsk) rq = task_rq_lock(tsk, &flags); if (tsk->sched_class != &fair_sched_class) { - set_task_cfs_rq(tsk); + set_task_cfs_rq(tsk, task_cpu(tsk)); goto done; } @@ -7103,7 +7109,7 @@ void sched_move_task(struct task_struct *tsk) tsk->sched_class->put_prev_task(rq, tsk); } - set_task_cfs_rq(tsk); + set_task_cfs_rq(tsk, task_cpu(tsk)); if (on_rq) { if (unlikely(running)) -- cgit v1.2.3 From 94bc9a7bd97efdda4dfbe61d0df32ce19d41c0a9 Mon Sep 17 00:00:00 2001 From: Dmitry Adamushko Date: Thu, 15 Nov 2007 20:57:40 +0100 Subject: sched: remove activate_idle_task() cpu_down() code is ok wrt sched_idle_next() placing the 'idle' task not at the beginning of the queue. So get rid of activate_idle_task() and make use of activate_task() instead. It is the same as activate_task(), except for the update_rq_clock(rq) call that is redundant. Code size goes down: text data bss dec hex filename 47853 3934 336 52123 cb9b sched.o.before 47828 3934 336 52098 cb82 sched.o.after Signed-off-by: Dmitry Adamushko Signed-off-by: Ingo Molnar --- kernel/sched.c | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index db1f71e3131..cc8cb6f7d82 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -5281,24 +5281,10 @@ static void migrate_live_tasks(int src_cpu) read_unlock(&tasklist_lock); } -/* - * activate_idle_task - move idle task to the _front_ of runqueue. - */ -static void activate_idle_task(struct task_struct *p, struct rq *rq) -{ - update_rq_clock(rq); - - if (p->state == TASK_UNINTERRUPTIBLE) - rq->nr_uninterruptible--; - - enqueue_task(rq, p, 0); - inc_nr_running(p, rq); -} - /* * Schedules idle task to be the next runnable task on current CPU. - * It does so by boosting its priority to highest possible and adding it to - * the _front_ of the runqueue. Used by CPU offline code. + * It does so by boosting its priority to highest possible. + * Used by CPU offline code. */ void sched_idle_next(void) { @@ -5318,8 +5304,8 @@ void sched_idle_next(void) __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1); - /* Add idle task to the _front_ of its priority queue: */ - activate_idle_task(p, rq); + update_rq_clock(rq); + activate_task(rq, p, 0); spin_unlock_irqrestore(&rq->lock, flags); } -- cgit v1.2.3 From 518b22e990a9071bf508ca67e31b37e7590f499c Mon Sep 17 00:00:00 2001 From: Adrian Bunk Date: Thu, 15 Nov 2007 20:57:40 +0100 Subject: sched: make sched_nr_latency static sched_nr_latency can now become static. Signed-off-by: Adrian Bunk Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index d3c03070872..ee00da284b1 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -43,7 +43,7 @@ unsigned int sysctl_sched_min_granularity = 1000000ULL; /* * is kept at sysctl_sched_latency / sysctl_sched_min_granularity */ -unsigned int sched_nr_latency = 20; +static unsigned int sched_nr_latency = 20; /* * After fork, child runs first. (default) If set to 0 then -- cgit v1.2.3 From 9612633a21ae8424531caf977f0560f64285bf36 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Thu, 15 Nov 2007 20:57:40 +0100 Subject: sched: reorder SCHED_FEAT_ bits reorder SCHED_FEAT_ bits so that the used ones come first. Makes tuning instructions easier. Signed-off-by: Ingo Molnar --- kernel/sched.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index cc8cb6f7d82..38933cafea8 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -455,18 +455,18 @@ static void update_rq_clock(struct rq *rq) */ enum { SCHED_FEAT_NEW_FAIR_SLEEPERS = 1, - SCHED_FEAT_START_DEBIT = 2, - SCHED_FEAT_TREE_AVG = 4, - SCHED_FEAT_APPROX_AVG = 8, - SCHED_FEAT_WAKEUP_PREEMPT = 16, + SCHED_FEAT_WAKEUP_PREEMPT = 2, + SCHED_FEAT_START_DEBIT = 4, + SCHED_FEAT_TREE_AVG = 8, + SCHED_FEAT_APPROX_AVG = 16, }; const_debug unsigned int sysctl_sched_features = SCHED_FEAT_NEW_FAIR_SLEEPERS * 1 | + SCHED_FEAT_WAKEUP_PREEMPT * 1 | SCHED_FEAT_START_DEBIT * 1 | SCHED_FEAT_TREE_AVG * 0 | - SCHED_FEAT_APPROX_AVG * 0 | - SCHED_FEAT_WAKEUP_PREEMPT * 1; + SCHED_FEAT_APPROX_AVG * 0; #define sched_feat(x) (sysctl_sched_features & SCHED_FEAT_##x) -- cgit v1.2.3 From 4307d1e5ada595c87f9a4d16db16ba5edb70dcb1 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 7 Nov 2007 18:37:48 +0100 Subject: x86: ignore the sys_getcpu() tcache parameter dont use the vgetcpu tcache - it's causing problems with tasks migrating, they'll see the old cache up to a jiffy after the migration, further increasing the costs of the migration. In the worst case they see a complete bogus information from the tcache, when a sys_getcpu() call "invalidated" the cache info by incrementing the jiffies _and_ the cpuid info in the cache and the following vdso_getcpu() call happens after vdso_jiffies have been incremented. Signed-off-by: Ingo Molnar Signed-off-by: Ulrich Drepper Signed-off-by: Thomas Gleixner --- kernel/sys.c | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) (limited to 'kernel') diff --git a/kernel/sys.c b/kernel/sys.c index 304b5410d74..d1fe71eb454 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1750,7 +1750,7 @@ asmlinkage long sys_prctl(int option, unsigned long arg2, unsigned long arg3, } asmlinkage long sys_getcpu(unsigned __user *cpup, unsigned __user *nodep, - struct getcpu_cache __user *cache) + struct getcpu_cache __user *unused) { int err = 0; int cpu = raw_smp_processor_id(); @@ -1758,24 +1758,6 @@ asmlinkage long sys_getcpu(unsigned __user *cpup, unsigned __user *nodep, err |= put_user(cpu, cpup); if (nodep) err |= put_user(cpu_to_node(cpu), nodep); - if (cache) { - /* - * The cache is not needed for this implementation, - * but make sure user programs pass something - * valid. vsyscall implementations can instead make - * good use of the cache. Only use t0 and t1 because - * these are available in both 32bit and 64bit ABI (no - * need for a compat_getcpu). 32bit has enough - * padding - */ - unsigned long t0, t1; - get_user(t0, &cache->blob[0]); - get_user(t1, &cache->blob[1]); - t0++; - t1++; - put_user(t0, &cache->blob[0]); - put_user(t1, &cache->blob[1]); - } return err ? -EFAULT : 0; } -- cgit v1.2.3 From fa6a1a554b50cbb7763f6907e6fef927ead480d9 Mon Sep 17 00:00:00 2001 From: "David P. Reed" Date: Wed, 14 Nov 2007 17:49:21 -0500 Subject: ntp: fix typo that makes sync_cmos_clock erratic Fix a typo in ntp.c that has caused updating of the persistent (RTC) clock when synced to NTP to behave erratically. When debugging a freeze that arises on my AMD64 machines when I run the ntpd service, I added a number of printk's to monitor the sync_cmos_clock procedure. I discovered that it was not syncing to cmos RTC every 11 minutes as documented, but instead would keep trying every second for hours at a time. The reason turned out to be a typo in sync_cmos_clock, where it attempts to ensure that update_persistent_clock is called very close to 500 msec. after a 1 second boundary (required by the PC RTC's spec). That typo referred to "xtime" in one spot, rather than "now", which is derived from "xtime" but not equal to it. This makes the test erratic, creating a "coin-flip" that decides when update_persistent_clock is called - when it is called, which is rarely, it may be at any time during the one second period, rather than close to 500 msec, so the value written is needlessly incorrect, too. Signed-off-by: David P. Reed Signed-off-by: Thomas Gleixner --- kernel/time/ntp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index de6a2d6b3eb..14a2ecf2b31 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -205,7 +205,7 @@ static void sync_cmos_clock(unsigned long dummy) return; getnstimeofday(&now); - if (abs(xtime.tv_nsec - (NSEC_PER_SEC / 2)) <= tick_nsec / 2) + if (abs(now.tv_nsec - (NSEC_PER_SEC / 2)) <= tick_nsec / 2) fail = update_persistent_clock(now); next.tv_nsec = (NSEC_PER_SEC / 2) - now.tv_nsec; -- cgit v1.2.3 From 9a4b9708f1f2eaf5edd619df578cf3afec36eb82 Mon Sep 17 00:00:00 2001 From: Matti Linnanvuori Date: Thu, 8 Nov 2007 08:37:38 -0800 Subject: module: fix and elaborate comments Fix and elaborate comments. Signed-off-by: Matti Linnanvuori Signed-off-by: Rusty Russell --- kernel/module.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index 3202c995007..91fe6958b6e 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -81,7 +81,8 @@ int unregister_module_notifier(struct notifier_block * nb) } EXPORT_SYMBOL(unregister_module_notifier); -/* We require a truly strong try_module_get() */ +/* We require a truly strong try_module_get(): 0 means failure due to + ongoing or failed initialization etc. */ static inline int strong_try_module_get(struct module *mod) { if (mod && mod->state == MODULE_STATE_COMING) @@ -952,7 +953,8 @@ static unsigned long resolve_symbol(Elf_Shdr *sechdrs, ret = __find_symbol(name, &owner, &crc, !(mod->taints & TAINT_PROPRIETARY_MODULE)); if (ret) { - /* use_module can fail due to OOM, or module unloading */ + /* use_module can fail due to OOM, + or module initialization or unloading */ if (!check_version(sechdrs, versindex, name, mod, crc) || !use_module(mod, owner)) ret = 0; @@ -1369,7 +1371,7 @@ dup: return ret; } -/* Change all symbols so that sh_value encodes the pointer directly. */ +/* Change all symbols so that st_value encodes the pointer directly. */ static int simplify_symbols(Elf_Shdr *sechdrs, unsigned int symindex, const char *strtab, -- cgit v1.2.3 From 611cd55b155a89d9a0ce5f92a9cbabc5e284d0d4 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Mon, 19 Nov 2007 21:49:25 -0800 Subject: [IPVS]: Fix sysctl warnings about missing strategy Running the latest git code I get the following messages during boot: sysctl table check failed: /net/ipv4/vs/drop_entry .3.5.21.4 Missing strategy [...] sysctl table check failed: /net/ipv4/vs/drop_packet .3.5.21.5 Missing strategy [...] sysctl table check failed: /net/ipv4/vs/secure_tcp .3.5.21.6 Missing strategy [...] sysctl table check failed: /net/ipv4/vs/sync_threshold .3.5.21.24 Missing strategy I removed the binary sysctl handler for those messages and also removed the definitions in ip_vs.h. The alternative would be to implement a proper strategy handler, but syscall sysctl is deprecated. There are other sysctl definitions that are commented out or work with the default sysctl_data strategy. I did not touch these. Signed-off-by: Christian Borntraeger Acked-by: Simon Horman Signed-off-by: David S. Miller --- kernel/sysctl_check.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index 4abc6d2306f..9e174976037 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c @@ -242,9 +242,6 @@ static struct trans_ctl_table trans_net_ipv4_vs_table[] = { { NET_IPV4_VS_AMEMTHRESH, "amemthresh" }, { NET_IPV4_VS_DEBUG_LEVEL, "debug_level" }, { NET_IPV4_VS_AMDROPRATE, "am_droprate" }, - { NET_IPV4_VS_DROP_ENTRY, "drop_entry" }, - { NET_IPV4_VS_DROP_PACKET, "drop_packet" }, - { NET_IPV4_VS_SECURE_TCP, "secure_tcp" }, { NET_IPV4_VS_TO_ES, "timeout_established" }, { NET_IPV4_VS_TO_SS, "timeout_synsent" }, { NET_IPV4_VS_TO_SR, "timeout_synrecv" }, @@ -260,7 +257,6 @@ static struct trans_ctl_table trans_net_ipv4_vs_table[] = { { NET_IPV4_VS_CACHE_BYPASS, "cache_bypass" }, { NET_IPV4_VS_EXPIRE_NODEST_CONN, "expire_nodest_conn" }, { NET_IPV4_VS_EXPIRE_QUIESCENT_TEMPLATE, "expire_quiescent_template" }, - { NET_IPV4_VS_SYNC_THRESHOLD, "sync_threshold" }, { NET_IPV4_VS_NAT_ICMP_SEND, "nat_icmp_send" }, { NET_IPV4_VS_LBLC_EXPIRE, "lblc_expiration" }, { NET_IPV4_VS_LBLCR_EXPIRE, "lblcr_expiration" }, -- cgit v1.2.3 From 9e103fa6bd53147e228e941256803a6b8927cdb9 Mon Sep 17 00:00:00 2001 From: Simon Horman Date: Mon, 19 Nov 2007 21:50:21 -0800 Subject: [IPVS]: Fix sysctl warnings about missing strategy in schedulers sysctl table check failed: /net/ipv4/vs/lblc_expiration .3.5.21.19 Missing strategy [...] sysctl table check failed: /net/ipv4/vs/lblcr_expiration .3.5.21.20 Missing strategy Switch these entried over to use CTL_UNNUMBERED as clearly the sys_syscal portion wasn't working. This is along the same lines as Christian Borntraeger's patch that fixes up entries with no stratergy in net/ipv4/ipvs/ip_vs_ctl.c Signed-off-by: Simon Horman Signed-off-by: David S. Miller --- kernel/sysctl_check.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index 9e174976037..cffb4adf138 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c @@ -258,8 +258,6 @@ static struct trans_ctl_table trans_net_ipv4_vs_table[] = { { NET_IPV4_VS_EXPIRE_NODEST_CONN, "expire_nodest_conn" }, { NET_IPV4_VS_EXPIRE_QUIESCENT_TEMPLATE, "expire_quiescent_template" }, { NET_IPV4_VS_NAT_ICMP_SEND, "nat_icmp_send" }, - { NET_IPV4_VS_LBLC_EXPIRE, "lblc_expiration" }, - { NET_IPV4_VS_LBLCR_EXPIRE, "lblcr_expiration" }, {} }; -- cgit v1.2.3 From 9055fa1f3ded5ad858a55ae18439ed55227ee7eb Mon Sep 17 00:00:00 2001 From: Simon Horman Date: Mon, 19 Nov 2007 21:51:13 -0800 Subject: [IPVS]: Move remaining sysctl handlers over to CTL_UNNUMBERED Switch the remaining IPVS sysctl entries over to to use CTL_UNNUMBERED, I stronly doubt that anyone is using the sys_sysctl interface to these variables. Signed-off-by: Simon Horman Signed-off-by: David S. Miller --- kernel/sysctl_check.c | 25 ------------------------- 1 file changed, 25 deletions(-) (limited to 'kernel') diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index cffb4adf138..fdfca0dd990 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c @@ -237,30 +237,6 @@ static struct trans_ctl_table trans_net_ipv4_conf_table[] = { {} }; - -static struct trans_ctl_table trans_net_ipv4_vs_table[] = { - { NET_IPV4_VS_AMEMTHRESH, "amemthresh" }, - { NET_IPV4_VS_DEBUG_LEVEL, "debug_level" }, - { NET_IPV4_VS_AMDROPRATE, "am_droprate" }, - { NET_IPV4_VS_TO_ES, "timeout_established" }, - { NET_IPV4_VS_TO_SS, "timeout_synsent" }, - { NET_IPV4_VS_TO_SR, "timeout_synrecv" }, - { NET_IPV4_VS_TO_FW, "timeout_finwait" }, - { NET_IPV4_VS_TO_TW, "timeout_timewait" }, - { NET_IPV4_VS_TO_CL, "timeout_close" }, - { NET_IPV4_VS_TO_CW, "timeout_closewait" }, - { NET_IPV4_VS_TO_LA, "timeout_lastack" }, - { NET_IPV4_VS_TO_LI, "timeout_listen" }, - { NET_IPV4_VS_TO_SA, "timeout_synack" }, - { NET_IPV4_VS_TO_UDP, "timeout_udp" }, - { NET_IPV4_VS_TO_ICMP, "timeout_icmp" }, - { NET_IPV4_VS_CACHE_BYPASS, "cache_bypass" }, - { NET_IPV4_VS_EXPIRE_NODEST_CONN, "expire_nodest_conn" }, - { NET_IPV4_VS_EXPIRE_QUIESCENT_TEMPLATE, "expire_quiescent_template" }, - { NET_IPV4_VS_NAT_ICMP_SEND, "nat_icmp_send" }, - {} -}; - static struct trans_ctl_table trans_net_neigh_vars_table[] = { { NET_NEIGH_MCAST_SOLICIT, "mcast_solicit" }, { NET_NEIGH_UCAST_SOLICIT, "ucast_solicit" }, @@ -335,7 +311,6 @@ static struct trans_ctl_table trans_net_ipv4_table[] = { { NET_IPV4_ROUTE, "route", trans_net_ipv4_route_table }, /* NET_IPV4_FIB_HASH unused */ { NET_IPV4_NETFILTER, "netfilter", trans_net_ipv4_netfilter_table }, - { NET_IPV4_VS, "vs", trans_net_ipv4_vs_table }, { NET_IPV4_TCP_TIMESTAMPS, "tcp_timestamps" }, { NET_IPV4_TCP_WINDOW_SCALING, "tcp_window_scaling" }, -- cgit v1.2.3 From 43ebbf119a9670d8f08b9e57968e109c770f8636 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 20 Nov 2007 11:13:33 +0100 Subject: [S390] cmm: remove unused binary sysctls. Remove binary sysctls that never worked due to missing strategy functions. Cc: Christian Borntraeger Signed-off-by: Heiko Carstens Signed-off-by: "Eric W. Biederman" Signed-off-by: Andrew Morton Signed-off-by: Martin Schwidefsky --- kernel/sysctl_check.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index 4abc6d2306f..cde6d780b0e 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c @@ -140,9 +140,6 @@ static struct trans_ctl_table trans_vm_table[] = { { VM_PANIC_ON_OOM, "panic_on_oom" }, { VM_VDSO_ENABLED, "vdso_enabled" }, { VM_MIN_SLAB, "min_slab_ratio" }, - { VM_CMM_PAGES, "cmm_pages" }, - { VM_CMM_TIMED_PAGES, "cmm_timed_pages" }, - { VM_CMM_TIMEOUT, "cmm_timeout" }, {} }; -- cgit v1.2.3 From 37e3a6ac5a30468021a2f366e497d455bbcb5d21 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 20 Nov 2007 11:13:34 +0100 Subject: [S390] appldata: remove unused binary sysctls. Remove binary sysctls that never worked due to missing strategy functions. Cc: "Eric W. Biederman" Cc: Christian Borntraeger Cc: Gerald Schaefer Signed-off-by: Heiko Carstens Signed-off-by: Martin Schwidefsky --- kernel/sysctl_check.c | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'kernel') diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index cde6d780b0e..8f5baac1eb0 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c @@ -1216,16 +1216,6 @@ static struct trans_ctl_table trans_arlan_table[] = { {} }; -static struct trans_ctl_table trans_appldata_table[] = { - { CTL_APPLDATA_TIMER, "timer" }, - { CTL_APPLDATA_INTERVAL, "interval" }, - { CTL_APPLDATA_OS, "os" }, - { CTL_APPLDATA_NET_SUM, "net_sum" }, - { CTL_APPLDATA_MEM, "mem" }, - {} - -}; - static struct trans_ctl_table trans_s390dbf_table[] = { { 5678 /* CTL_S390DBF_STOPPABLE */, "debug_stoppable" }, { 5679 /* CTL_S390DBF_ACTIVE */, "debug_active" }, @@ -1270,7 +1260,6 @@ static struct trans_ctl_table trans_root_table[] = { { CTL_ABI, "abi" }, /* CTL_CPU not used */ { CTL_ARLAN, "arlan", trans_arlan_table }, - { CTL_APPLDATA, "appldata", trans_appldata_table }, { CTL_S390DBF, "s390dbf", trans_s390dbf_table }, { CTL_SUNRPC, "sunrpc", trans_sunrpc_table }, { CTL_PM, "pm", trans_pm_table }, -- cgit v1.2.3 From 52bfb36050c8529d9031d2c2513b281a360922ec Mon Sep 17 00:00:00 2001 From: John Stultz Date: Mon, 26 Nov 2007 20:42:19 +0100 Subject: time: add ADJ_OFFSET_SS_READ Michael Kerrisk reported that a long standing bug in the adjtimex() system call causes glibc's adjtime(3) function to deliver the wrong results if 'delta' is NULL. add the ADJ_OFFSET_SS_READ API detail, which will be used by glibc to fix this API compatibility bug. Also see: http://bugzilla.kernel.org/show_bug.cgi?id=6761 [ mingo@elte.hu: added patch description and made it backwards compatible ] NOTE: the new flag is defined 0xa001 so that it returns -EINVAL on older kernels - this way glibc can use it safely. Suggested by Ulrich Drepper. Acked-by: Ulrich Drepper Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/time/ntp.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 14a2ecf2b31..e64efaf957e 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -249,10 +249,12 @@ int do_adjtimex(struct timex *txc) /* Now we validate the data before disabling interrupts */ - if ((txc->modes & ADJ_OFFSET_SINGLESHOT) == ADJ_OFFSET_SINGLESHOT) + if ((txc->modes & ADJ_OFFSET_SINGLESHOT) == ADJ_OFFSET_SINGLESHOT) { /* singleshot must not be used with any other mode bits */ - if (txc->modes != ADJ_OFFSET_SINGLESHOT) + if (txc->modes != ADJ_OFFSET_SINGLESHOT && + txc->modes != ADJ_OFFSET_SS_READ) return -EINVAL; + } if (txc->modes != ADJ_OFFSET_SINGLESHOT && (txc->modes & ADJ_OFFSET)) /* adjustment Offset limited to +- .512 seconds */ @@ -372,7 +374,8 @@ int do_adjtimex(struct timex *txc) leave: if ((time_status & (STA_UNSYNC|STA_CLOCKERR)) != 0) result = TIME_ERROR; - if ((txc->modes & ADJ_OFFSET_SINGLESHOT) == ADJ_OFFSET_SINGLESHOT) + if ((txc->modes == ADJ_OFFSET_SINGLESHOT) || + (txc->modes == ADJ_OFFSET_SS_READ)) txc->offset = save_adjust; else txc->offset = ((long)shift_right(time_offset, SHIFT_UPDATE)) * -- cgit v1.2.3 From 5e8869bb699d50be5c0733edfc71cfcd5b43e10a Mon Sep 17 00:00:00 2001 From: Pavel Emelyanov Date: Mon, 26 Nov 2007 21:21:49 +0100 Subject: sched: don't forget to unlock uids_mutex on error paths The commit commit 5cb350baf580017da38199625b7365b1763d7180 Author: Dhaval Giani Date: Mon Oct 15 17:00:14 2007 +0200 sched: group scheduling, sysfs tunables introduced the uids_mutex and the helpers to lock/unlock it. Unfortunately, the error paths of alloc_uid() were not patched to unlock it. Signed-off-by: Pavel Emelyanov Acked-by: Dhaval Giani Signed-off-by: Ingo Molnar --- kernel/user.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/user.c b/kernel/user.c index 0f3aa023410..8320a87f3e5 100644 --- a/kernel/user.c +++ b/kernel/user.c @@ -337,8 +337,11 @@ struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid) struct user_struct *new; new = kmem_cache_alloc(uid_cachep, GFP_KERNEL); - if (!new) + if (!new) { + uids_mutex_unlock(); return NULL; + } + new->uid = uid; atomic_set(&new->__count, 1); atomic_set(&new->processes, 0); @@ -355,6 +358,7 @@ struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid) if (alloc_uid_keyring(new, current) < 0) { kmem_cache_free(uid_cachep, new); + uids_mutex_unlock(); return NULL; } @@ -362,6 +366,7 @@ struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid) key_put(new->uid_keyring); key_put(new->session_keyring); kmem_cache_free(uid_cachep, new); + uids_mutex_unlock(); return NULL; } -- cgit v1.2.3 From bcbe4a076609e15ea84cbebd9cd8f317ed70ce92 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 26 Nov 2007 21:21:49 +0100 Subject: sched: fix kernel/acct.c comment fix kernel/acct.c comment. noticed by Lin Tan. Comment suggested by Olaf Kirch. also see: http://bugzilla.kernel.org/show_bug.cgi?id=8220 Reported-by: tammy000@gmail.com Signed-off-by: Ingo Molnar --- kernel/acct.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/acct.c b/kernel/acct.c index fce53d8df8a..cf19547cc9e 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -413,7 +413,7 @@ static u32 encode_float(u64 value) * The acct_process() call is the workhorse of the process * accounting system. The struct acct is built here and then written * into the accounting file. This function should only be called from - * do_exit(). + * do_exit() or when switching to a different output file. */ /* -- cgit v1.2.3 From 722aab0c3bbd7648d66790515c14d95d10a15bf3 Mon Sep 17 00:00:00 2001 From: Zou Nan hai Date: Mon, 26 Nov 2007 21:21:49 +0100 Subject: sched: fix minimum granularity tunings increase the default minimum granularity some more - this gives us more performance in aim7 benchmarks. also correct some comments: we scale with ilog(ncpus) + 1. Signed-off-by: Zou Nan hai Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index ee00da284b1..2f16e15c022 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -22,7 +22,7 @@ /* * Targeted preemption latency for CPU-bound tasks: - * (default: 20ms * ilog(ncpus), units: nanoseconds) + * (default: 20ms * (1 + ilog(ncpus)), units: nanoseconds) * * NOTE: this latency value is not the same as the concept of * 'timeslice length' - timeslices in CFS are of variable length @@ -36,14 +36,14 @@ unsigned int sysctl_sched_latency = 20000000ULL; /* * Minimal preemption granularity for CPU-bound tasks: - * (default: 1 msec * ilog(ncpus), units: nanoseconds) + * (default: 4 msec * (1 + ilog(ncpus)), units: nanoseconds) */ -unsigned int sysctl_sched_min_granularity = 1000000ULL; +unsigned int sysctl_sched_min_granularity = 4000000ULL; /* * is kept at sysctl_sched_latency / sysctl_sched_min_granularity */ -static unsigned int sched_nr_latency = 20; +static unsigned int sched_nr_latency = 5; /* * After fork, child runs first. (default) If set to 0 then @@ -61,7 +61,7 @@ unsigned int __read_mostly sysctl_sched_compat_yield; /* * SCHED_BATCH wake-up granularity. - * (default: 10 msec * ilog(ncpus), units: nanoseconds) + * (default: 10 msec * (1 + ilog(ncpus)), units: nanoseconds) * * This option delays the preemption effects of decoupled workloads * and reduces their over-scheduling. Synchronous workloads will still @@ -71,7 +71,7 @@ unsigned int sysctl_sched_batch_wakeup_granularity = 10000000UL; /* * SCHED_OTHER wake-up granularity. - * (default: 10 msec * ilog(ncpus), units: nanoseconds) + * (default: 10 msec * (1 + ilog(ncpus)), units: nanoseconds) * * This option delays the preemption effects of decoupled workloads * and reduces their over-scheduling. Synchronous workloads will still -- cgit v1.2.3 From f7b9329e556a8bdb9e07292cddbbe484c7a2b8c5 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 26 Nov 2007 21:21:49 +0100 Subject: sched: bump version of kernel/sched_debug.c bump version of kernel/sched_debug.c and remove CFS version information from it. Signed-off-by: Ingo Molnar --- kernel/sched_debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index ca198a797bf..5d0d623a546 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c @@ -199,7 +199,7 @@ static int sched_debug_show(struct seq_file *m, void *v) u64 now = ktime_to_ns(ktime_get()); int cpu; - SEQ_printf(m, "Sched Debug Version: v0.06-v22, %s %.*s\n", + SEQ_printf(m, "Sched Debug Version: v0.07, %s %.*s\n", init_utsname()->release, (int)strcspn(init_utsname()->version, " "), init_utsname()->version); -- cgit v1.2.3 From d3938204468dccae16be0099a2abf53db4ed0505 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 28 Nov 2007 15:52:56 +0100 Subject: softlockup: fix false positives on CONFIG_NOHZ David Miller reported soft lockup false-positives that trigger on NOHZ due to CPUs idling for more than 10 seconds. The solution is touch the softlockup watchdog when we return from idle. (by definition we are not 'locked up' when we were idle) http://bugzilla.kernel.org/show_bug.cgi?id=9409 Reported-by: David Miller Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- kernel/time/tick-sched.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 27a2338deb4..cb89fa8db11 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -133,6 +133,8 @@ void tick_nohz_update_jiffies(void) if (!ts->tick_stopped) return; + touch_softlockup_watchdog(); + cpu_clear(cpu, nohz_cpu_mask); now = ktime_get(); -- cgit v1.2.3 From 9a4e715914f07e56cbfa6f2b544a68365d51c3ef Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 28 Nov 2007 15:52:56 +0100 Subject: sched: clean up sd_alloc_ctl_cpu_table() definition clean up sd_alloc_ctl_cpu_table() definition. Signed-off-by: Ingo Molnar --- kernel/sched.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 38933cafea8..89cb2dac8bf 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -5466,7 +5466,7 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd) return table; } -static ctl_table * sd_alloc_ctl_cpu_table(int cpu) +static ctl_table *sd_alloc_ctl_cpu_table(int cpu) { struct ctl_table *entry, *table; struct sched_domain *sd; -- cgit v1.2.3 From deaf2227ddf657a260e923db44b6f0974d9bb782 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 28 Nov 2007 15:52:56 +0100 Subject: sched: clean up, move __sched_text_start/end to sched.h move __sched_text_start/end to sched.h. No code changed: text data bss dec hex filename 26582 2310 28 28920 70f8 sched.o.before 26582 2310 28 28920 70f8 sched.o.after Signed-off-by: Ingo Molnar --- kernel/sched.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 89cb2dac8bf..98dcdf272db 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -6708,9 +6708,6 @@ void __init sched_init_smp(void) int in_sched_functions(unsigned long addr) { - /* Linker adds these: start and end of __sched functions */ - extern char __sched_text_start[], __sched_text_end[]; - return in_lock_functions(addr) || (addr >= (unsigned long)__sched_text_start && addr < (unsigned long)__sched_text_end); -- cgit v1.2.3 From c1a89740da168d3431f2f4e7c3b03daacbb55be1 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 28 Nov 2007 15:52:56 +0100 Subject: sched: clean up overlong line in kernel/sched_debug.c clean up overlong line in kernel/sched_debug.c. Signed-off-by: Ingo Molnar --- kernel/sched_debug.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index 5d0d623a546..d30467b47dd 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c @@ -327,10 +327,12 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) avg_atom = -1LL; avg_per_cpu = p->se.sum_exec_runtime; - if (p->se.nr_migrations) - avg_per_cpu = div64_64(avg_per_cpu, p->se.nr_migrations); - else + if (p->se.nr_migrations) { + avg_per_cpu = div64_64(avg_per_cpu, + p->se.nr_migrations); + } else { avg_per_cpu = -1LL; + } __PN(avg_atom); __PN(avg_per_cpu); -- cgit v1.2.3 From f95e0d1c2ad668c77aa4b272c076faf3aa0d631c Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 28 Nov 2007 15:52:56 +0100 Subject: sched: clean up kernel/sched_stat.h clean up kernel/sched_stat.h. Signed-off-by: Ingo Molnar --- kernel/sched_stats.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched_stats.h b/kernel/sched_stats.h index 630178e53bb..5b32433e7ee 100644 --- a/kernel/sched_stats.h +++ b/kernel/sched_stats.h @@ -52,7 +52,8 @@ static int show_schedstat(struct seq_file *seq, void *v) sd->lb_nobusyq[itype], sd->lb_nobusyg[itype]); } - seq_printf(seq, " %u %u %u %u %u %u %u %u %u %u %u %u\n", + seq_printf(seq, + " %u %u %u %u %u %u %u %u %u %u %u %u\n", sd->alb_count, sd->alb_failed, sd->alb_pushed, sd->sbe_count, sd->sbe_balanced, sd->sbe_pushed, sd->sbf_count, sd->sbf_balanced, sd->sbf_pushed, -- cgit v1.2.3 From c895078355b6b6e05c60aa205892526dd3390f0a Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 28 Nov 2007 16:21:24 -0800 Subject: wait_task_stopped(): don't use task_pid_nr_ns() lockless wait_task_stopped(WNOWAIT) does task_pid_nr_ns() without tasklist/rcu lock, we can read an already freed memory. Use the cached pid_t value. Signed-off-by: Oleg Nesterov Looks-good-to: Roland McGrath Acked-by: Pavel Emelyanov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/exit.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/exit.c b/kernel/exit.c index cd0f1d4137a..0a4a382ecf2 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -1357,7 +1357,7 @@ static int wait_task_stopped(struct task_struct *p, int delayed_group_leader, int __user *stat_addr, struct rusage __user *ru) { int retval, exit_code; - struct pid_namespace *ns; + pid_t pid; if (!p->exit_code) return 0; @@ -1376,12 +1376,11 @@ static int wait_task_stopped(struct task_struct *p, int delayed_group_leader, * keep holding onto the tasklist_lock while we call getrusage and * possibly take page faults for user memory. */ - ns = current->nsproxy->pid_ns; + pid = task_pid_nr_ns(p, current->nsproxy->pid_ns); get_task_struct(p); read_unlock(&tasklist_lock); if (unlikely(noreap)) { - pid_t pid = task_pid_nr_ns(p, ns); uid_t uid = p->uid; int why = (p->ptrace & PT_PTRACED) ? CLD_TRAPPED : CLD_STOPPED; @@ -1451,11 +1450,11 @@ bail_ref: if (!retval && infop) retval = put_user(exit_code, &infop->si_status); if (!retval && infop) - retval = put_user(task_pid_nr_ns(p, ns), &infop->si_pid); + retval = put_user(pid, &infop->si_pid); if (!retval && infop) retval = put_user(p->uid, &infop->si_uid); if (!retval) - retval = task_pid_nr_ns(p, ns); + retval = pid; put_task_struct(p); BUG_ON(!retval); -- cgit v1.2.3 From 32df81cbd5b41d281cc3d7e7ff6a98ac6201e197 Mon Sep 17 00:00:00 2001 From: Pavel Emelyanov Date: Wed, 28 Nov 2007 16:21:38 -0800 Subject: Isolate the UTS namespace's domainname and hostname back Commit 7d69a1f4a72b18876c99c697692b78339d491568 ("remove CONFIG_UTS_NS and CONFIG_IPC_NS") by Cedric Le Goater accidentally removed the code that prevented the uts->hostname and uts->domainname values from being overwritten from another namespace. In other words, setting hostname/domainname via sysfs (echo xxx > /proc/sys/kernel/(host|domain)name) cased the new value to be set in init UTS namespace only. Return the isolation back. Signed-off-by: Pavel Emelyanov Acked-by: Cedric Le Goater Acked-by: Serge Hallyn Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/utsname_sysctl.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/utsname_sysctl.c b/kernel/utsname_sysctl.c index c76c06466bf..fe3a56c2256 100644 --- a/kernel/utsname_sysctl.c +++ b/kernel/utsname_sysctl.c @@ -18,6 +18,10 @@ static void *get_uts(ctl_table *table, int write) { char *which = table->data; + struct uts_namespace *uts_ns; + + uts_ns = current->nsproxy->uts_ns; + which = (which - (char *)&init_uts_ns) + (char *)uts_ns; if (!write) down_read(&uts_sem); -- cgit v1.2.3 From 9e6c1e633355b69803094ecbac4cecc96e00965c Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 28 Nov 2007 16:22:04 -0800 Subject: FRV: fix the extern declaration of kallsyms_num_syms Fix the extern declaration of kallsyms_num_syms to indicate that the symbol does not reside in the small-data storage space, and so may not be accessed relative to the small data base register. Signed-off-by: David Howells Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/kallsyms.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 474219a4192..2fc25810509 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -32,9 +32,14 @@ /* These will be re-linked against their real values during the second link stage */ extern const unsigned long kallsyms_addresses[] __attribute__((weak)); -extern const unsigned long kallsyms_num_syms __attribute__((weak)); extern const u8 kallsyms_names[] __attribute__((weak)); +/* tell the compiler that the count isn't in the small data section if the arch + * has one (eg: FRV) + */ +extern const unsigned long kallsyms_num_syms +__attribute__((weak, section(".rodata"))); + extern const u8 kallsyms_token_table[] __attribute__((weak)); extern const u16 kallsyms_token_index[] __attribute__((weak)); -- cgit v1.2.3 From e6ceb32aa25fc33f21af84cc7a32fe289b3e860c Mon Sep 17 00:00:00 2001 From: Scott James Remnant Date: Wed, 28 Nov 2007 16:22:07 -0800 Subject: wait_task_stopped(): pass correct exit_code to wait_noreap_copyout() In wait_task_stopped() exit_code already contains the right value for the si_status member of siginfo, and this is simply set in the non WNOWAIT case. If you call waitid() with a stopped or traced process, you'll get the signal in siginfo.si_status as expected -- however if you call waitid(WNOWAIT) at the same time, you'll get the signal << 8 | 0x7f Pass it unchanged to wait_noreap_copyout(); we would only need to shift it and add 0x7f if we were returning it in the user status field and that isn't used for any function that permits WNOWAIT. Signed-off-by: Scott James Remnant Signed-off-by: Oleg Nesterov Cc: Roland McGrath Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/exit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/exit.c b/kernel/exit.c index 0a4a382ecf2..549c0558ba6 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -1388,7 +1388,7 @@ static int wait_task_stopped(struct task_struct *p, int delayed_group_leader, if (unlikely(!exit_code) || unlikely(p->exit_state)) goto bail_ref; return wait_noreap_copyout(p, pid, uid, - why, (exit_code << 8) | 0x7f, + why, exit_code, infop, ru); } -- cgit v1.2.3 From d842de871c8c5e2110c7e4f3f29bbe7b1a519ab8 Mon Sep 17 00:00:00 2001 From: Srivatsa Vaddagiri Date: Sun, 2 Dec 2007 20:04:49 +0100 Subject: sched: cpu accounting controller (V2) Commit cfb5285660aad4931b2ebbfa902ea48a37dfffa1 removed a useful feature for us, which provided a cpu accounting resource controller. This feature would be useful if someone wants to group tasks only for accounting purpose and doesnt really want to exercise any control over their cpu consumption. The patch below reintroduces the feature. It is based on Paul Menage's original patch (Commit 62d0df64065e7c135d0002f069444fbdfc64768f), with these differences: - Removed load average information. I felt it needs more thought (esp to deal with SMP and virtualized platforms) and can be added for 2.6.25 after more discussions. - Convert group cpu usage to be nanosecond accurate (as rest of the cfs stats are) and invoke cpuacct_charge() from the respective scheduler classes - Make accounting scalable on SMP systems by splitting the usage counter to be per-cpu - Move the code from kernel/cpu_acct.c to kernel/sched.c (since the code is not big enough to warrant a new file and also this rightly needs to live inside the scheduler. Also things like accessing rq->lock while reading cpu usage becomes easier if the code lived in kernel/sched.c) The patch also modifies the cpu controller not to provide the same accounting information. Tested-by: Balbir Singh Tested the patches on top of 2.6.24-rc3. The patches work fine. Ran some simple tests like cpuspin (spin on the cpu), ran several tasks in the same group and timed them. Compared their time stamps with cpuacct.usage. Signed-off-by: Srivatsa Vaddagiri Signed-off-by: Balbir Singh Signed-off-by: Ingo Molnar --- kernel/sched.c | 155 +++++++++++++++++++++++++++++++++++++++++++--------- kernel/sched_fair.c | 6 ++ kernel/sched_rt.c | 1 + 3 files changed, 136 insertions(+), 26 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 98dcdf272db..59ff6b140ed 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -854,6 +854,12 @@ iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest, struct rq_iterator *iterator); #endif +#ifdef CONFIG_CGROUP_CPUACCT +static void cpuacct_charge(struct task_struct *tsk, u64 cputime); +#else +static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {} +#endif + #include "sched_stats.h" #include "sched_idletask.c" #include "sched_fair.c" @@ -7221,38 +7227,12 @@ static u64 cpu_shares_read_uint(struct cgroup *cgrp, struct cftype *cft) return (u64) tg->shares; } -static u64 cpu_usage_read(struct cgroup *cgrp, struct cftype *cft) -{ - struct task_group *tg = cgroup_tg(cgrp); - unsigned long flags; - u64 res = 0; - int i; - - for_each_possible_cpu(i) { - /* - * Lock to prevent races with updating 64-bit counters - * on 32-bit arches. - */ - spin_lock_irqsave(&cpu_rq(i)->lock, flags); - res += tg->se[i]->sum_exec_runtime; - spin_unlock_irqrestore(&cpu_rq(i)->lock, flags); - } - /* Convert from ns to ms */ - do_div(res, NSEC_PER_MSEC); - - return res; -} - static struct cftype cpu_files[] = { { .name = "shares", .read_uint = cpu_shares_read_uint, .write_uint = cpu_shares_write_uint, }, - { - .name = "usage", - .read_uint = cpu_usage_read, - }, }; static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont) @@ -7272,3 +7252,126 @@ struct cgroup_subsys cpu_cgroup_subsys = { }; #endif /* CONFIG_FAIR_CGROUP_SCHED */ + +#ifdef CONFIG_CGROUP_CPUACCT + +/* + * CPU accounting code for task groups. + * + * Based on the work by Paul Menage (menage@google.com) and Balbir Singh + * (balbir@in.ibm.com). + */ + +/* track cpu usage of a group of tasks */ +struct cpuacct { + struct cgroup_subsys_state css; + /* cpuusage holds pointer to a u64-type object on every cpu */ + u64 *cpuusage; +}; + +struct cgroup_subsys cpuacct_subsys; + +/* return cpu accounting group corresponding to this container */ +static inline struct cpuacct *cgroup_ca(struct cgroup *cont) +{ + return container_of(cgroup_subsys_state(cont, cpuacct_subsys_id), + struct cpuacct, css); +} + +/* return cpu accounting group to which this task belongs */ +static inline struct cpuacct *task_ca(struct task_struct *tsk) +{ + return container_of(task_subsys_state(tsk, cpuacct_subsys_id), + struct cpuacct, css); +} + +/* create a new cpu accounting group */ +static struct cgroup_subsys_state *cpuacct_create( + struct cgroup_subsys *ss, struct cgroup *cont) +{ + struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL); + + if (!ca) + return ERR_PTR(-ENOMEM); + + ca->cpuusage = alloc_percpu(u64); + if (!ca->cpuusage) { + kfree(ca); + return ERR_PTR(-ENOMEM); + } + + return &ca->css; +} + +/* destroy an existing cpu accounting group */ +static void cpuacct_destroy(struct cgroup_subsys *ss, + struct cgroup *cont) +{ + struct cpuacct *ca = cgroup_ca(cont); + + free_percpu(ca->cpuusage); + kfree(ca); +} + +/* return total cpu usage (in nanoseconds) of a group */ +static u64 cpuusage_read(struct cgroup *cont, struct cftype *cft) +{ + struct cpuacct *ca = cgroup_ca(cont); + u64 totalcpuusage = 0; + int i; + + for_each_possible_cpu(i) { + u64 *cpuusage = percpu_ptr(ca->cpuusage, i); + + /* + * Take rq->lock to make 64-bit addition safe on 32-bit + * platforms. + */ + spin_lock_irq(&cpu_rq(i)->lock); + totalcpuusage += *cpuusage; + spin_unlock_irq(&cpu_rq(i)->lock); + } + + return totalcpuusage; +} + +static struct cftype files[] = { + { + .name = "usage", + .read_uint = cpuusage_read, + }, +}; + +static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cont) +{ + return cgroup_add_files(cont, ss, files, ARRAY_SIZE(files)); +} + +/* + * charge this task's execution time to its accounting group. + * + * called with rq->lock held. + */ +static void cpuacct_charge(struct task_struct *tsk, u64 cputime) +{ + struct cpuacct *ca; + + if (!cpuacct_subsys.active) + return; + + ca = task_ca(tsk); + if (ca) { + u64 *cpuusage = percpu_ptr(ca->cpuusage, task_cpu(tsk)); + + *cpuusage += cputime; + } +} + +struct cgroup_subsys cpuacct_subsys = { + .name = "cpuacct", + .create = cpuacct_create, + .destroy = cpuacct_destroy, + .populate = cpuacct_populate, + .subsys_id = cpuacct_subsys_id, +}; +#endif /* CONFIG_CGROUP_CPUACCT */ diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 2f16e15c022..37bb265598d 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -351,6 +351,12 @@ static void update_curr(struct cfs_rq *cfs_rq) __update_curr(cfs_rq, curr, delta_exec); curr->exec_start = now; + + if (entity_is_task(curr)) { + struct task_struct *curtask = task_of(curr); + + cpuacct_charge(curtask, delta_exec); + } } static inline void diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 8abd752a0eb..ee9c8b6529e 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -23,6 +23,7 @@ static void update_curr_rt(struct rq *rq) curr->se.sum_exec_runtime += delta_exec; curr->se.exec_start = rq->clock; + cpuacct_charge(curr, delta_exec); } static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup) -- cgit v1.2.3 From b00296fb781acfafa93687000cdef72b8922bb40 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 1 Dec 2007 12:16:29 -0800 Subject: uml: add !UML dependencies The previous commit ("uml: keep UML Kconfig in sync with x86") is not enough, unfortunately. If we go that way, we need to add dependencies on !UML for several options. Signed-off-by: Al Viro Signed-off-by: Jeff Dike Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/Kconfig.instrumentation | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/Kconfig.instrumentation b/kernel/Kconfig.instrumentation index f5f2c769d95..2ea1e347df4 100644 --- a/kernel/Kconfig.instrumentation +++ b/kernel/Kconfig.instrumentation @@ -21,7 +21,7 @@ config PROFILING config OPROFILE tristate "OProfile system profiling (EXPERIMENTAL)" depends on PROFILING - depends on ALPHA || ARM || BLACKFIN || X86_32 || IA64 || M32R || MIPS || PARISC || PPC || S390 || SUPERH || SPARC || X86_64 + depends on (ALPHA || ARM || BLACKFIN || X86_32 || IA64 || M32R || MIPS || PARISC || PPC || S390 || SUPERH || SPARC || X86_64) && !UML help OProfile is a profiling system capable of profiling the whole system, include the kernel, kernel modules, libraries, @@ -32,7 +32,7 @@ config OPROFILE config KPROBES bool "Kprobes" depends on KALLSYMS && MODULES - depends on X86_32 || IA64 || PPC || S390 || SPARC64 || X86_64 || AVR32 + depends on (X86_32 || IA64 || PPC || S390 || SPARC64 || X86_64 || AVR32) && !UML help Kprobes allows you to trap at almost any kernel address and execute a callback function. register_kprobe() establishes -- cgit v1.2.3 From 77034937dc4575ca0a76bf209838ecd39e804089 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 4 Dec 2007 17:04:39 +0100 Subject: sched: fix crash in sys_sched_rr_get_interval() Luiz Fernando N. Capitulino reported that sched_rr_get_interval() crashes for SCHED_OTHER tasks that are on an idle runqueue. The fix is to return a 0 timeslice for tasks that are on an idle runqueue. (and which are not running, obviously) this also shrinks the code a bit: text data bss dec hex filename 47903 3934 336 52173 cbcd sched.o.before 47885 3934 336 52155 cbbb sched.o.after Reported-by: Luiz Fernando N. Capitulino Signed-off-by: Ingo Molnar --- kernel/sched.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 59ff6b140ed..b062856b946 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -4850,17 +4850,21 @@ long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval) if (retval) goto out_unlock; - if (p->policy == SCHED_FIFO) - time_slice = 0; - else if (p->policy == SCHED_RR) + /* + * Time slice is 0 for SCHED_FIFO tasks and for SCHED_OTHER + * tasks that are on an otherwise idle runqueue: + */ + time_slice = 0; + if (p->policy == SCHED_RR) { time_slice = DEF_TIMESLICE; - else { + } else { struct sched_entity *se = &p->se; unsigned long flags; struct rq *rq; rq = task_rq_lock(p, &flags); - time_slice = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se)); + if (rq->cfs.load.weight) + time_slice = NS_TO_JIFFIES(sched_slice(&rq->cfs, se)); task_rq_unlock(rq, &flags); } read_unlock(&tasklist_lock); -- cgit v1.2.3 From db292ca302e83534f5f0f7139e13d7e6976e51f9 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 4 Dec 2007 17:04:39 +0100 Subject: sched: default to more agressive yield for SCHED_BATCH tasks do more agressive yield for SCHED_BATCH tuned tasks: they are all about throughput anyway. This allows a gentler migration path for any apps that relied on stronger yield. Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 37bb265598d..c33f0ceb3de 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -799,8 +799,9 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep) */ static void yield_task_fair(struct rq *rq) { - struct cfs_rq *cfs_rq = task_cfs_rq(rq->curr); - struct sched_entity *rightmost, *se = &rq->curr->se; + struct task_struct *curr = rq->curr; + struct cfs_rq *cfs_rq = task_cfs_rq(curr); + struct sched_entity *rightmost, *se = &curr->se; /* * Are we the only task in the tree? @@ -808,7 +809,7 @@ static void yield_task_fair(struct rq *rq) if (unlikely(cfs_rq->nr_running == 1)) return; - if (likely(!sysctl_sched_compat_yield)) { + if (likely(!sysctl_sched_compat_yield) && curr->policy != SCHED_BATCH) { __update_rq_clock(rq); /* * Update run-time statistics of the 'current'. -- cgit v1.2.3 From 874a5f87f53f80b798140b07fcf81f8d3718b3cc Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 19 Nov 2007 21:35:42 -0800 Subject: [SYSCTL_CHECK]: Fix typo in KERN_SPARC_SCONS_PWROFF entry string. Based upon a report by Mikael Pettersson. Signed-off-by: David S. Miller --- kernel/sysctl_check.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index 6972f26c65f..bed939f82c3 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c @@ -96,7 +96,7 @@ static struct trans_ctl_table trans_kern_table[] = { { KERN_PTY, "pty", trans_pty_table }, { KERN_NGROUPS_MAX, "ngroups_max" }, - { KERN_SPARC_SCONS_PWROFF, "scons_poweroff" }, + { KERN_SPARC_SCONS_PWROFF, "scons-poweroff" }, { KERN_HZ_TIMER, "hz_timer" }, { KERN_UNKNOWN_NMI_PANIC, "unknown_nmi_panic" }, { KERN_BOOTLOADER_TYPE, "bootloader_type" }, -- cgit v1.2.3 From ce6bd420f43b28038a2c6e8fbb86ad24014727b6 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Wed, 5 Dec 2007 15:46:09 +0100 Subject: futex: fix for futex_wait signal stack corruption David Holmes found a bug in the -rt tree with respect to pthread_cond_timedwait. After trying his test program on the latest git from mainline, I found the bug was there too. The bug he was seeing that his test program showed, was that if one were to do a "Ctrl-Z" on a process that was in the pthread_cond_timedwait, and then did a "bg" on that process, it would return with a "-ETIMEDOUT" but early. That is, the timer would go off early. Looking into this, I found the source of the problem. And it is a rather nasty bug at that. Here's the relevant code from kernel/futex.c: (not in order in the file) [...] smlinkage long sys_futex(u32 __user *uaddr, int op, u32 val, struct timespec __user *utime, u32 __user *uaddr2, u32 val3) { struct timespec ts; ktime_t t, *tp = NULL; u32 val2 = 0; int cmd = op & FUTEX_CMD_MASK; if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI)) { if (copy_from_user(&ts, utime, sizeof(ts)) != 0) return -EFAULT; if (!timespec_valid(&ts)) return -EINVAL; t = timespec_to_ktime(ts); if (cmd == FUTEX_WAIT) t = ktime_add(ktime_get(), t); tp = &t; } [...] return do_futex(uaddr, op, val, tp, uaddr2, val2, val3); } [...] long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, u32 __user *uaddr2, u32 val2, u32 val3) { int ret; int cmd = op & FUTEX_CMD_MASK; struct rw_semaphore *fshared = NULL; if (!(op & FUTEX_PRIVATE_FLAG)) fshared = ¤t->mm->mmap_sem; switch (cmd) { case FUTEX_WAIT: ret = futex_wait(uaddr, fshared, val, timeout); [...] static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared, u32 val, ktime_t *abs_time) { [...] struct restart_block *restart; restart = ¤t_thread_info()->restart_block; restart->fn = futex_wait_restart; restart->arg0 = (unsigned long)uaddr; restart->arg1 = (unsigned long)val; restart->arg2 = (unsigned long)abs_time; restart->arg3 = 0; if (fshared) restart->arg3 |= ARG3_SHARED; return -ERESTART_RESTARTBLOCK; [...] static long futex_wait_restart(struct restart_block *restart) { u32 __user *uaddr = (u32 __user *)restart->arg0; u32 val = (u32)restart->arg1; ktime_t *abs_time = (ktime_t *)restart->arg2; struct rw_semaphore *fshared = NULL; restart->fn = do_no_restart_syscall; if (restart->arg3 & ARG3_SHARED) fshared = ¤t->mm->mmap_sem; return (long)futex_wait(uaddr, fshared, val, abs_time); } So when the futex_wait is interrupt by a signal we break out of the hrtimer code and set up or return from signal. This code does not return back to userspace, so we set up a RESTARTBLOCK. The bug here is that we save the "abs_time" which is a pointer to the stack variable "ktime_t t" from sys_futex. This returns and unwinds the stack before we get to call our signal. On return from the signal we go to futex_wait_restart, where we update all the parameters for futex_wait and call it. But here we have a problem where abs_time is no longer valid. I verified this with print statements, and sure enough, what abs_time was set to ends up being garbage when we get to futex_wait_restart. The solution I did to solve this (with input from Linus Torvalds) was to add unions to the restart_block to allow system calls to use the restart with specific parameters. This way the futex code now saves the time in a 64bit value in the restart block instead of storing it on the stack. Note: I'm a bit nervious to add "linux/types.h" and use u32 and u64 in thread_info.h, when there's a #ifdef __KERNEL__ just below that. Not sure what that is there for. If this turns out to be a problem, I've tested this with using "unsigned int" for u32 and "unsigned long long" for u64 and it worked just the same. I'm using u32 and u64 just to be consistent with what the futex code uses. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner Acked-by: Linus Torvalds --- kernel/futex.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/futex.c b/kernel/futex.c index 9dc591ab681..e8fbdd7d95a 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -1149,9 +1149,9 @@ static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q, /* * In case we must use restart_block to restart a futex_wait, - * we encode in the 'arg3' shared capability + * we encode in the 'flags' shared capability */ -#define ARG3_SHARED 1 +#define FLAGS_SHARED 1 static long futex_wait_restart(struct restart_block *restart); @@ -1290,12 +1290,13 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared, struct restart_block *restart; restart = ¤t_thread_info()->restart_block; restart->fn = futex_wait_restart; - restart->arg0 = (unsigned long)uaddr; - restart->arg1 = (unsigned long)val; - restart->arg2 = (unsigned long)abs_time; - restart->arg3 = 0; + restart->futex.uaddr = (u32 *)uaddr; + restart->futex.val = val; + restart->futex.time = abs_time->tv64; + restart->futex.flags = 0; + if (fshared) - restart->arg3 |= ARG3_SHARED; + restart->futex.flags |= FLAGS_SHARED; return -ERESTART_RESTARTBLOCK; } @@ -1310,15 +1311,15 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared, static long futex_wait_restart(struct restart_block *restart) { - u32 __user *uaddr = (u32 __user *)restart->arg0; - u32 val = (u32)restart->arg1; - ktime_t *abs_time = (ktime_t *)restart->arg2; + u32 __user *uaddr = (u32 __user *)restart->futex.uaddr; struct rw_semaphore *fshared = NULL; + ktime_t t; + t.tv64 = restart->futex.time; restart->fn = do_no_restart_syscall; - if (restart->arg3 & ARG3_SHARED) + if (restart->futex.flags & FLAGS_SHARED) fshared = ¤t->mm->mmap_sem; - return (long)futex_wait(uaddr, fshared, val, abs_time); + return (long)futex_wait(uaddr, fshared, restart->futex.val, &t); } -- cgit v1.2.3 From 41a2d6cfa3f77ec469e7e5f06b4d7ffd031f9c0e Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 5 Dec 2007 15:46:09 +0100 Subject: sched: style cleanups style cleanup of various changes that were done recently. no code changed: text data bss dec hex filename 23680 2542 28 26250 668a sched.o.before 23680 2542 28 26250 668a sched.o.after Signed-off-by: Ingo Molnar --- kernel/sched.c | 132 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 68 insertions(+), 64 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index b062856b946..67d9d1799d8 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -209,9 +209,8 @@ static inline struct task_group *task_group(struct task_struct *p) tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id), struct task_group, css); #else - tg = &init_task_group; + tg = &init_task_group; #endif - return tg; } @@ -249,15 +248,16 @@ struct cfs_rq { #ifdef CONFIG_FAIR_GROUP_SCHED struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */ - /* leaf cfs_rqs are those that hold tasks (lowest schedulable entity in + /* + * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in * a hierarchy). Non-leaf lrqs hold other higher schedulable entities * (like users, containers etc.) * * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This * list is used during load balance. */ - struct list_head leaf_cfs_rq_list; /* Better name : task_cfs_rq_list? */ - struct task_group *tg; /* group that "owns" this runqueue */ + struct list_head leaf_cfs_rq_list; + struct task_group *tg; /* group that "owns" this runqueue */ #endif }; @@ -300,7 +300,7 @@ struct rq { /* list of leaf cfs_rq on this cpu: */ struct list_head leaf_cfs_rq_list; #endif - struct rt_rq rt; + struct rt_rq rt; /* * This is part of a global counter where only the total sum @@ -457,8 +457,8 @@ enum { SCHED_FEAT_NEW_FAIR_SLEEPERS = 1, SCHED_FEAT_WAKEUP_PREEMPT = 2, SCHED_FEAT_START_DEBIT = 4, - SCHED_FEAT_TREE_AVG = 8, - SCHED_FEAT_APPROX_AVG = 16, + SCHED_FEAT_TREE_AVG = 8, + SCHED_FEAT_APPROX_AVG = 16, }; const_debug unsigned int sysctl_sched_features = @@ -591,7 +591,7 @@ static inline struct rq *__task_rq_lock(struct task_struct *p) /* * task_rq_lock - lock the runqueue a given task resides on and disable - * interrupts. Note the ordering: we can safely lookup the task_rq without + * interrupts. Note the ordering: we can safely lookup the task_rq without * explicitly disabling preemption. */ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags) @@ -779,7 +779,7 @@ static inline void update_load_sub(struct load_weight *lw, unsigned long dec) * To aid in avoiding the subversion of "niceness" due to uneven distribution * of tasks with abnormal "nice" values across CPUs the contribution that * each task makes to its run queue's load is weighted according to its - * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a + * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a * scaled version of the new time slice allocation that they receive on time * slice expiry etc. */ @@ -1854,7 +1854,7 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev, * and do any other architecture-specific cleanup actions. * * Note that we may have delayed dropping an mm in context_switch(). If - * so, we finish that here outside of the runqueue lock. (Doing it + * so, we finish that here outside of the runqueue lock. (Doing it * with the lock held can cause deadlocks; see schedule() for * details.) */ @@ -2136,7 +2136,7 @@ static void double_lock_balance(struct rq *this_rq, struct rq *busiest) /* * If dest_cpu is allowed for this process, migrate the task to it. * This is accomplished by forcing the cpu_allowed mask to only - * allow dest_cpu, which will force the cpu onto dest_cpu. Then + * allow dest_cpu, which will force the cpu onto dest_cpu. Then * the cpu_allowed mask is restored. */ static void sched_migrate_task(struct task_struct *p, int dest_cpu) @@ -2581,7 +2581,7 @@ group_next: * tasks around. Thus we look for the minimum possible imbalance. * Negative imbalances (*we* are more loaded than anyone else) will * be counted as no imbalance for these purposes -- we can't fix that - * by pulling tasks to us. Be careful of negative numbers as they'll + * by pulling tasks to us. Be careful of negative numbers as they'll * appear as very large values with unsigned longs. */ if (max_load <= busiest_load_per_task) @@ -3016,7 +3016,7 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu) /* * This condition is "impossible", if it occurs - * we need to fix it. Originally reported by + * we need to fix it. Originally reported by * Bjorn Helgaas on a 128-cpu setup. */ BUG_ON(busiest_rq == target_rq); @@ -3048,7 +3048,7 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu) #ifdef CONFIG_NO_HZ static struct { atomic_t load_balancer; - cpumask_t cpu_mask; + cpumask_t cpu_mask; } nohz ____cacheline_aligned = { .load_balancer = ATOMIC_INIT(-1), .cpu_mask = CPU_MASK_NONE, @@ -3552,7 +3552,7 @@ static noinline void __schedule_bug(struct task_struct *prev) static inline void schedule_debug(struct task_struct *prev) { /* - * Test if we are atomic. Since do_exit() needs to call into + * Test if we are atomic. Since do_exit() needs to call into * schedule() atomically, we ignore that path for now. * Otherwise, whine if we are scheduling when we should not be. */ @@ -3674,7 +3674,7 @@ EXPORT_SYMBOL(schedule); #ifdef CONFIG_PREEMPT /* * this is the entry point to schedule() from in-kernel preemption - * off of preempt_enable. Kernel preemptions off return from interrupt + * off of preempt_enable. Kernel preemptions off return from interrupt * occur there and call schedule directly. */ asmlinkage void __sched preempt_schedule(void) @@ -3686,7 +3686,7 @@ asmlinkage void __sched preempt_schedule(void) #endif /* * If there is a non-zero preempt_count or interrupts are disabled, - * we do not want to preempt the current task. Just return.. + * we do not want to preempt the current task. Just return.. */ if (likely(ti->preempt_count || irqs_disabled())) return; @@ -3772,12 +3772,12 @@ int default_wake_function(wait_queue_t *curr, unsigned mode, int sync, EXPORT_SYMBOL(default_wake_function); /* - * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just - * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve + * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just + * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve * number) then we wake all the non-exclusive tasks and one exclusive task. * * There are circumstances in which we can try to wake a task which has already - * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns + * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns * zero in this (rare) case, and we handle it by continuing to scan the queue. */ static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, @@ -4390,8 +4390,8 @@ do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) * @policy: new policy. * @param: structure containing the new RT priority. */ -asmlinkage long sys_sched_setscheduler(pid_t pid, int policy, - struct sched_param __user *param) +asmlinkage long +sys_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) { /* negative values for policy are not valid */ if (policy < 0) @@ -4491,7 +4491,7 @@ long sched_setaffinity(pid_t pid, cpumask_t new_mask) /* * It is not safe to call set_cpus_allowed with the - * tasklist_lock held. We will bump the task_struct's + * tasklist_lock held. We will bump the task_struct's * usage count and then drop tasklist_lock. */ get_task_struct(p); @@ -4687,7 +4687,7 @@ EXPORT_SYMBOL(cond_resched); * cond_resched_lock() - if a reschedule is pending, drop the given lock, * call schedule, and on return reacquire the lock. * - * This works OK both with and without CONFIG_PREEMPT. We do strange low-level + * This works OK both with and without CONFIG_PREEMPT. We do strange low-level * operations here to prevent schedule() from being called twice (once via * spin_unlock(), once by hand). */ @@ -4741,7 +4741,7 @@ void __sched yield(void) EXPORT_SYMBOL(yield); /* - * This task is about to go to sleep on IO. Increment rq->nr_iowait so + * This task is about to go to sleep on IO. Increment rq->nr_iowait so * that process accounting knows that this is a task in IO wait state. * * But don't do that if it is a deliberate, throttling IO wait (this task @@ -5050,7 +5050,7 @@ static inline void sched_init_granularity(void) * is removed from the allowed bitmask. * * NOTE: the caller must have a valid reference to the task, the - * task must not exit() & deallocate itself prematurely. The + * task must not exit() & deallocate itself prematurely. The * call is not atomic; no spinlocks may be held. */ int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask) @@ -5087,7 +5087,7 @@ out: EXPORT_SYMBOL_GPL(set_cpus_allowed); /* - * Move (not current) task off this cpu, onto dest cpu. We're doing + * Move (not current) task off this cpu, onto dest cpu. We're doing * this because either it can't run here any more (set_cpus_allowed() * away from this CPU, or CPU going down), or because we're * attempting to rebalance this task on exec (sched_exec). @@ -5232,7 +5232,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) * Try to stay on the same cpuset, where the * current cpuset may be a subset of all cpus. * The cpuset_cpus_allowed_locked() variant of - * cpuset_cpus_allowed() will not block. It must be + * cpuset_cpus_allowed() will not block. It must be * called within calls to cpuset_lock/cpuset_unlock. */ rq = task_rq_lock(p, &flags); @@ -5245,10 +5245,11 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) * kernel threads (both mm NULL), since they never * leave kernel. */ - if (p->mm && printk_ratelimit()) + if (p->mm && printk_ratelimit()) { printk(KERN_INFO "process %d (%s) no " "longer affine to cpu%d\n", - task_pid_nr(p), p->comm, dead_cpu); + task_pid_nr(p), p->comm, dead_cpu); + } } } while (!__migrate_task_irq(p, dead_cpu, dest_cpu)); } @@ -5350,7 +5351,7 @@ static void migrate_dead(unsigned int dead_cpu, struct task_struct *p) /* * Drop lock around migration; if someone else moves it, - * that's OK. No task can be added to this CPU, so iteration is + * that's OK. No task can be added to this CPU, so iteration is * fine. */ spin_unlock_irq(&rq->lock); @@ -5414,7 +5415,7 @@ static void sd_free_ctl_entry(struct ctl_table **tablep) /* * In the intermediate directories, both the child directory and * procname are dynamically allocated and could fail but the mode - * will always be set. In the lowest directory the names are + * will always be set. In the lowest directory the names are * static strings and all have proc handlers. */ for (entry = *tablep; entry->mode; entry++) { @@ -5585,7 +5586,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) case CPU_UP_CANCELED_FROZEN: if (!cpu_rq(cpu)->migration_thread) break; - /* Unbind it from offline cpu so it can run. Fall thru. */ + /* Unbind it from offline cpu so it can run. Fall thru. */ kthread_bind(cpu_rq(cpu)->migration_thread, any_online_cpu(cpu_online_map)); kthread_stop(cpu_rq(cpu)->migration_thread); @@ -5612,9 +5613,11 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) migrate_nr_uninterruptible(rq); BUG_ON(rq->nr_running != 0); - /* No need to migrate the tasks: it was best-effort if - * they didn't take sched_hotcpu_mutex. Just wake up - * the requestors. */ + /* + * No need to migrate the tasks: it was best-effort if + * they didn't take sched_hotcpu_mutex. Just wake up + * the requestors. + */ spin_lock_irq(&rq->lock); while (!list_empty(&rq->migration_queue)) { struct migration_req *req; @@ -5922,7 +5925,7 @@ init_sched_build_groups(cpumask_t span, const cpumask_t *cpu_map, * @node: node whose sched_domain we're building * @used_nodes: nodes already in the sched_domain * - * Find the next node to include in a given scheduling domain. Simply + * Find the next node to include in a given scheduling domain. Simply * finds the closest node not already in the @used_nodes map. * * Should use nodemask_t. @@ -5962,7 +5965,7 @@ static int find_next_best_node(int node, unsigned long *used_nodes) * @node: node whose cpumask we're constructing * @size: number of nodes to include in this span * - * Given a node, construct a good cpumask for its sched_domain to span. It + * Given a node, construct a good cpumask for its sched_domain to span. It * should be one that prevents unnecessary balancing, but also spreads tasks * out optimally. */ @@ -5999,8 +6002,8 @@ int sched_smt_power_savings = 0, sched_mc_power_savings = 0; static DEFINE_PER_CPU(struct sched_domain, cpu_domains); static DEFINE_PER_CPU(struct sched_group, sched_group_cpus); -static int cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, - struct sched_group **sg) +static int +cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg) { if (sg) *sg = &per_cpu(sched_group_cpus, cpu); @@ -6017,8 +6020,8 @@ static DEFINE_PER_CPU(struct sched_group, sched_group_core); #endif #if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) -static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map, - struct sched_group **sg) +static int +cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg) { int group; cpumask_t mask = per_cpu(cpu_sibling_map, cpu); @@ -6029,8 +6032,8 @@ static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map, return group; } #elif defined(CONFIG_SCHED_MC) -static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map, - struct sched_group **sg) +static int +cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg) { if (sg) *sg = &per_cpu(sched_group_core, cpu); @@ -6041,8 +6044,8 @@ static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map, static DEFINE_PER_CPU(struct sched_domain, phys_domains); static DEFINE_PER_CPU(struct sched_group, sched_group_phys); -static int cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, - struct sched_group **sg) +static int +cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg) { int group; #ifdef CONFIG_SCHED_MC @@ -6222,7 +6225,7 @@ static int build_sched_domains(const cpumask_t *cpu_map) * Allocate the per-node list of sched groups */ sched_group_nodes = kcalloc(MAX_NUMNODES, sizeof(struct sched_group *), - GFP_KERNEL); + GFP_KERNEL); if (!sched_group_nodes) { printk(KERN_WARNING "Can not alloc sched group node list\n"); return -ENOMEM; @@ -6469,7 +6472,7 @@ static int ndoms_cur; /* number of sched domains in 'doms_cur' */ static cpumask_t fallback_doms; /* - * Set up scheduler domains and groups. Callers must hold the hotplug lock. + * Set up scheduler domains and groups. Callers must hold the hotplug lock. * For now this just excludes isolated cpus, but could be used to * exclude other special cases in the future. */ @@ -6511,19 +6514,19 @@ static void detach_destroy_domains(const cpumask_t *cpu_map) /* * Partition sched domains as specified by the 'ndoms_new' - * cpumasks in the array doms_new[] of cpumasks. This compares + * cpumasks in the array doms_new[] of cpumasks. This compares * doms_new[] to the current sched domain partitioning, doms_cur[]. * It destroys each deleted domain and builds each new domain. * * 'doms_new' is an array of cpumask_t's of length 'ndoms_new'. - * The masks don't intersect (don't overlap.) We should setup one - * sched domain for each mask. CPUs not in any of the cpumasks will - * not be load balanced. If the same cpumask appears both in the + * The masks don't intersect (don't overlap.) We should setup one + * sched domain for each mask. CPUs not in any of the cpumasks will + * not be load balanced. If the same cpumask appears both in the * current 'doms_cur' domains and in the new 'doms_new', we can leave * it as it is. * - * The passed in 'doms_new' should be kmalloc'd. This routine takes - * ownership of it and will kfree it when done with it. If the caller + * The passed in 'doms_new' should be kmalloc'd. This routine takes + * ownership of it and will kfree it when done with it. If the caller * failed the kmalloc call, then it can pass in doms_new == NULL, * and partition_sched_domains() will fallback to the single partition * 'fallback_doms'. @@ -6653,7 +6656,7 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls) #endif /* - * Force a reinitialization of the sched domains hierarchy. The domains + * Force a reinitialization of the sched domains hierarchy. The domains * and groups cannot be updated in place without racing with the balancing * code, so we temporarily attach all running cpus to the NULL domain * which will prevent rebalancing while the sched domains are recalculated. @@ -6943,8 +6946,8 @@ struct task_struct *curr_task(int cpu) * @p: the task pointer to set. * * Description: This function must only be used when non-maskable interrupts - * are serviced on a separate stack. It allows the architecture to switch the - * notion of the current task on a cpu in a non-blocking manner. This function + * are serviced on a separate stack. It allows the architecture to switch the + * notion of the current task on a cpu in a non-blocking manner. This function * must be called with all CPU's synchronized, and interrupts disabled, the * and caller must save the original value of the current task (see * curr_task() above) and restore that value before reenabling interrupts and @@ -7193,16 +7196,17 @@ cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp) return &tg->css; } -static void cpu_cgroup_destroy(struct cgroup_subsys *ss, - struct cgroup *cgrp) +static void +cpu_cgroup_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp) { struct task_group *tg = cgroup_tg(cgrp); sched_destroy_group(tg); } -static int cpu_cgroup_can_attach(struct cgroup_subsys *ss, - struct cgroup *cgrp, struct task_struct *tsk) +static int +cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp, + struct task_struct *tsk) { /* We don't support RT-tasks being in separate groups */ if (tsk->sched_class != &fair_sched_class) @@ -7308,8 +7312,8 @@ static struct cgroup_subsys_state *cpuacct_create( } /* destroy an existing cpu accounting group */ -static void cpuacct_destroy(struct cgroup_subsys *ss, - struct cgroup *cont) +static void +cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cont) { struct cpuacct *ca = cgroup_ca(cont); -- cgit v1.2.3 From 856848737bd944c1db3ce0a66bbf67e56bd6f77d Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 5 Dec 2007 15:46:09 +0100 Subject: lockdep: fix debug_show_all_locks() fix the oops that can be seen in: http://bugzilla.kernel.org/attachment.cgi?id=13828&action=view it is not safe to print the locks of running tasks. (even with this fix we have a small race - but this is a debug function after all.) Signed-off-by: Ingo Molnar Signed-off-by: Peter Zijlstra --- kernel/lockdep.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'kernel') diff --git a/kernel/lockdep.c b/kernel/lockdep.c index ed38bbfc48a..7e2ca7c9d99 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -3173,6 +3173,13 @@ retry: printk(" locked it.\n"); do_each_thread(g, p) { + /* + * It's not reliable to print a task's held locks + * if it's not sleeping (or if it's not the current + * task): + */ + if (p->state == TASK_RUNNING && p != current) + continue; if (p->lockdep_depth) lockdep_print_held_locks(p); if (!unlock) -- cgit v1.2.3 From 54561783ee99d73a086f3abbda3e44f87f6bf65b Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 5 Dec 2007 15:46:09 +0100 Subject: lockdep: in_range() fix Torsten Kaiser wrote: | static inline int in_range(const void *start, const void *addr, const void *end) | { | return addr >= start && addr <= end; | } | This will return true, if addr is in the range of start (including) | to end (including). | | But debug_check_no_locks_freed() seems does: | const void *mem_to = mem_from + mem_len | -> mem_to is the last byte of the freed range, that fits in_range | lock_from = (void *)hlock->instance; | -> first byte of the lock | lock_to = (void *)(hlock->instance + 1); | -> first byte of the next lock, not last byte of the lock that is being checked! | | The test is: | if (!in_range(mem_from, lock_from, mem_to) && | !in_range(mem_from, lock_to, mem_to)) | continue; | So it tests, if the first byte of the lock is in the range that is freed ->OK | And if the first byte of the *next* lock is in the range that is freed | -> Not OK. We can also simplify in_range checks, we need only 2 comparisons, not 4. If the lock is not in memory range, it should be either at the left of range or at the right. Signed-off-by: Oleg Nesterov Signed-off-by: Ingo Molnar Signed-off-by: Peter Zijlstra --- kernel/lockdep.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 7e2ca7c9d99..0f389621bb6 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -3054,11 +3054,6 @@ void __init lockdep_info(void) #endif } -static inline int in_range(const void *start, const void *addr, const void *end) -{ - return addr >= start && addr <= end; -} - static void print_freed_lock_bug(struct task_struct *curr, const void *mem_from, const void *mem_to, struct held_lock *hlock) @@ -3080,6 +3075,13 @@ print_freed_lock_bug(struct task_struct *curr, const void *mem_from, dump_stack(); } +static inline int not_in_range(const void* mem_from, unsigned long mem_len, + const void* lock_from, unsigned long lock_len) +{ + return lock_from + lock_len <= mem_from || + mem_from + mem_len <= lock_from; +} + /* * Called when kernel memory is freed (or unmapped), or if a lock * is destroyed or reinitialized - this code checks whether there is @@ -3087,7 +3089,6 @@ print_freed_lock_bug(struct task_struct *curr, const void *mem_from, */ void debug_check_no_locks_freed(const void *mem_from, unsigned long mem_len) { - const void *mem_to = mem_from + mem_len, *lock_from, *lock_to; struct task_struct *curr = current; struct held_lock *hlock; unsigned long flags; @@ -3100,14 +3101,11 @@ void debug_check_no_locks_freed(const void *mem_from, unsigned long mem_len) for (i = 0; i < curr->lockdep_depth; i++) { hlock = curr->held_locks + i; - lock_from = (void *)hlock->instance; - lock_to = (void *)(hlock->instance + 1); - - if (!in_range(mem_from, lock_from, mem_to) && - !in_range(mem_from, lock_to, mem_to)) + if (not_in_range(mem_from, mem_len, hlock->instance, + sizeof(*hlock->instance))) continue; - print_freed_lock_bug(curr, mem_from, mem_to, hlock); + print_freed_lock_bug(curr, mem_from, mem_from + mem_len, hlock); break; } local_irq_restore(flags); -- cgit v1.2.3 From cde898fa80a45bb23eab2a060fc79d0913081409 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 5 Dec 2007 15:46:09 +0100 Subject: futex: correctly return -EFAULT not -EINVAL return -EFAULT not -EINVAL. Found by review. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- kernel/futex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/futex.c b/kernel/futex.c index e8fbdd7d95a..172a1aeeafd 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -658,7 +658,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) if (curval == -EFAULT) ret = -EFAULT; - if (curval != uval) + else if (curval != uval) ret = -EINVAL; if (ret) { spin_unlock(&pi_state->pi_mutex.wait_lock); -- cgit v1.2.3 From 5cd17569fd0eeca510735e63a6061291e3971bf6 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Tue, 4 Dec 2007 23:45:04 -0800 Subject: fix clone(CLONE_NEWPID) Currently we are complicating the code in copy_process, the clone ABI, and if we fix the bugs sys_setsid itself, with an unnecessary open coded version of sys_setsid. So just simplify everything and don't special case the session and pgrp of the initial process in a pid namespace. Having this special case actually presents to user space the classic linux startup conditions with session == pgrp == 0 for /sbin/init. We already handle sending signals to processes in a child pid namespace. We need to handle sending signals to processes in a parent pid namespace for cases like SIGCHILD and SIGIO. This makes nothing extra visible inside a pid namespace. So this extra special case appears to have no redeeming merits. Further removing this special case increases the flexibility of how we can use pid namespaces, by not requiring the initial process in a pid namespace to be a daemon. Signed-off-by: Eric W. Biederman Cc: Oleg Nesterov Cc: Pavel Emelyanov Cc: Sukadev Bhattiprolu Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/fork.c | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 8ca1a14cdc8..8dd8ff28100 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1292,23 +1292,14 @@ static struct task_struct *copy_process(unsigned long clone_flags, __ptrace_link(p, current->parent); if (thread_group_leader(p)) { - if (clone_flags & CLONE_NEWPID) { + if (clone_flags & CLONE_NEWPID) p->nsproxy->pid_ns->child_reaper = p; - p->signal->tty = NULL; - set_task_pgrp(p, p->pid); - set_task_session(p, p->pid); - attach_pid(p, PIDTYPE_PGID, pid); - attach_pid(p, PIDTYPE_SID, pid); - } else { - p->signal->tty = current->signal->tty; - set_task_pgrp(p, task_pgrp_nr(current)); - set_task_session(p, task_session_nr(current)); - attach_pid(p, PIDTYPE_PGID, - task_pgrp(current)); - attach_pid(p, PIDTYPE_SID, - task_session(current)); - } + p->signal->tty = current->signal->tty; + set_task_pgrp(p, task_pgrp_nr(current)); + set_task_session(p, task_session_nr(current)); + attach_pid(p, PIDTYPE_PGID, task_pgrp(current)); + attach_pid(p, PIDTYPE_SID, task_session(current)); list_add_tail_rcu(&p->tasks, &init_task.tasks); __get_cpu_var(process_counts)++; } -- cgit v1.2.3 From f1dad166e88a5ddca0acf8f11dea0e2bd92d8bf3 Mon Sep 17 00:00:00 2001 From: Pavel Emelyanov Date: Tue, 4 Dec 2007 23:45:24 -0800 Subject: Avoid potential NULL dereference in unregister_sysctl_table register_sysctl_table() can return NULL sometimes, e.g. when kmalloc() returns NULL or when sysctl check fails. I've also noticed, that many (most?) code in the kernel doesn't check for the return value from register_sysctl_table() and later simply calls the unregister_sysctl_table() with potentially NULL argument. This is unlikely on a common kernel configuration, but in case we're dealing with modules and/or fault-injection support, there's a slight possibility of an OOPS. Changing all the users to check for return code from the registering does not look like a good solution - there are too many code doing this and failure in sysctl tables registration is not a good reason to abort module loading (in most of the cases). So I think, that we can just have this check in unregister_sysctl_table just to avoid accidental OOPS-es (actually, the unregister_sysctl_table() did exactly this, before the start_unregistering() appeared). Signed-off-by: Pavel Emelyanov Cc: "Eric W. Biederman" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/sysctl.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 0deed82a615..8ac51714b08 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1588,6 +1588,10 @@ struct ctl_table_header *register_sysctl_table(struct ctl_table * table) void unregister_sysctl_table(struct ctl_table_header * header) { might_sleep(); + + if (header == NULL) + return; + spin_lock(&sysctl_lock); start_unregistering(header); spin_unlock(&sysctl_lock); -- cgit v1.2.3 From 00a5825332769706eb2e276c101dd20269a53992 Mon Sep 17 00:00:00 2001 From: Ralf Baechle Date: Thu, 6 Dec 2007 16:53:19 +0000 Subject: Fix oprofile configuration breakage The cleanup 09cadedbdc01f1a4bea1f427d4fb4642eaa19da9 broke the oprofile configuration for MIPS by allowing oprofile support to be built for kernel models where oprofile doesn't have a chance in hell to work. Just a dependecy list on a number of architectures is - surprise - broken and should as per past discussions probably in most considered to be broken in most cases. So I introduce a dependency for the oprofile configuration on ARCH_SUPPORTS_OPROFILE. Signed-off-by: Ralf Baechle Signed-off-by: Linus Torvalds --- kernel/Kconfig.instrumentation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/Kconfig.instrumentation b/kernel/Kconfig.instrumentation index 2ea1e347df4..12a9f74b626 100644 --- a/kernel/Kconfig.instrumentation +++ b/kernel/Kconfig.instrumentation @@ -21,7 +21,7 @@ config PROFILING config OPROFILE tristate "OProfile system profiling (EXPERIMENTAL)" depends on PROFILING - depends on (ALPHA || ARM || BLACKFIN || X86_32 || IA64 || M32R || MIPS || PARISC || PPC || S390 || SUPERH || SPARC || X86_64) && !UML + depends on (ARCH_SUPPORTS_OPROFILE || ALPHA || ARM || BLACKFIN || X86_32 || IA64 || M32R || PARISC || PPC || S390 || SUPERH || SPARC || X86_64) && !UML help OProfile is a profiling system capable of profiling the whole system, include the kernel, kernel modules, libraries, -- cgit v1.2.3 From 3743d33edf4e49376384822c57c4ee5cdf2d32f8 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Thu, 6 Dec 2007 09:41:12 -0800 Subject: Tiny clean-up of OPROFILE/KPROBES configuration Make the Kconfig.instrumentation file a bit easier on the eyes, and use the new ARCH_SUPPORTS_OPROFILE for x86[-64]. Signed-off-by: Linus Torvalds --- kernel/Kconfig.instrumentation | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/Kconfig.instrumentation b/kernel/Kconfig.instrumentation index 12a9f74b626..468f47ad750 100644 --- a/kernel/Kconfig.instrumentation +++ b/kernel/Kconfig.instrumentation @@ -20,8 +20,8 @@ config PROFILING config OPROFILE tristate "OProfile system profiling (EXPERIMENTAL)" - depends on PROFILING - depends on (ARCH_SUPPORTS_OPROFILE || ALPHA || ARM || BLACKFIN || X86_32 || IA64 || M32R || PARISC || PPC || S390 || SUPERH || SPARC || X86_64) && !UML + depends on PROFILING && !UML + depends on ARCH_SUPPORTS_OPROFILE || ALPHA || ARM || BLACKFIN || IA64 || M32R || PARISC || PPC || S390 || SUPERH || SPARC help OProfile is a profiling system capable of profiling the whole system, include the kernel, kernel modules, libraries, @@ -31,8 +31,8 @@ config OPROFILE config KPROBES bool "Kprobes" - depends on KALLSYMS && MODULES - depends on (X86_32 || IA64 || PPC || S390 || SPARC64 || X86_64 || AVR32) && !UML + depends on KALLSYMS && MODULES && !UML + depends on X86_32 || IA64 || PPC || S390 || SPARC64 || X86_64 || AVR32 help Kprobes allows you to trap at almost any kernel address and execute a callback function. register_kprobe() establishes -- cgit v1.2.3 From e64d66c8edf11629aa203328daf898775ee27dd4 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 6 Dec 2007 17:34:36 -0500 Subject: wait: Use TASK_NORMAL Also move wake_up_locked() to be with the related functions Signed-off-by: Matthew Wilcox --- kernel/wait.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/wait.c b/kernel/wait.c index 444ddbfaefc..f9876888a56 100644 --- a/kernel/wait.c +++ b/kernel/wait.c @@ -215,7 +215,7 @@ void fastcall __wake_up_bit(wait_queue_head_t *wq, void *word, int bit) { struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit); if (waitqueue_active(wq)) - __wake_up(wq, TASK_INTERRUPTIBLE|TASK_UNINTERRUPTIBLE, 1, &key); + __wake_up(wq, TASK_NORMAL, 1, &key); } EXPORT_SYMBOL(__wake_up_bit); -- cgit v1.2.3 From d2f60e5faa9aef9a08d0abaee2007e5a0800d2c5 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 6 Dec 2007 11:06:01 -0500 Subject: power: Use task_is_* Signed-off-by: Matthew Wilcox --- kernel/power/process.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/power/process.c b/kernel/power/process.c index 6533923e711..7c2118f9597 100644 --- a/kernel/power/process.c +++ b/kernel/power/process.c @@ -86,9 +86,9 @@ static void fake_signal_wake_up(struct task_struct *p, int resume) static void send_fake_signal(struct task_struct *p) { - if (p->state == TASK_STOPPED) + if (task_is_stopped(p)) force_sig_specific(SIGSTOP, p); - fake_signal_wake_up(p, p->state == TASK_STOPPED); + fake_signal_wake_up(p, task_is_stopped(p)); } static int has_mm(struct task_struct *p) @@ -182,7 +182,7 @@ static int try_to_freeze_tasks(int freeze_user_space) if (frozen(p) || !freezeable(p)) continue; - if (p->state == TASK_TRACED && frozen(p->parent)) { + if (task_is_traced(p) && frozen(p->parent)) { cancel_freezing(p); continue; } -- cgit v1.2.3 From 6618a3e275519e10001a2ac4669f46141d4c108b Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 6 Dec 2007 11:06:16 -0500 Subject: ptrace: Use task_is_* Signed-off-by: Matthew Wilcox --- kernel/ptrace.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 7c76f2ffaea..26f9923badd 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -51,7 +51,7 @@ void __ptrace_link(struct task_struct *child, struct task_struct *new_parent) void ptrace_untrace(struct task_struct *child) { spin_lock(&child->sighand->siglock); - if (child->state == TASK_TRACED) { + if (task_is_traced(child)) { if (child->signal->flags & SIGNAL_STOP_STOPPED) { child->state = TASK_STOPPED; } else { @@ -79,7 +79,7 @@ void __ptrace_unlink(struct task_struct *child) add_parent(child); } - if (child->state == TASK_TRACED) + if (task_is_traced(child)) ptrace_untrace(child); } @@ -103,9 +103,9 @@ int ptrace_check_attach(struct task_struct *child, int kill) && child->signal != NULL) { ret = 0; spin_lock_irq(&child->sighand->siglock); - if (child->state == TASK_STOPPED) { + if (task_is_stopped(child)) { child->state = TASK_TRACED; - } else if (child->state != TASK_TRACED && !kill) { + } else if (!task_is_traced(child) && !kill) { ret = -ESRCH; } spin_unlock_irq(&child->sighand->siglock); -- cgit v1.2.3 From d9514f6c6b95b5a747ba902858eff577281e8659 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 6 Dec 2007 11:07:07 -0500 Subject: sched: Use task_contributes_to_load, TASK_ALL and TASK_NORMAL Signed-off-by: Matthew Wilcox --- kernel/sched.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 67d9d1799d8..50a0faae585 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -992,7 +992,7 @@ static int effective_prio(struct task_struct *p) */ static void activate_task(struct rq *rq, struct task_struct *p, int wakeup) { - if (p->state == TASK_UNINTERRUPTIBLE) + if (task_contributes_to_load(p)) rq->nr_uninterruptible--; enqueue_task(rq, p, wakeup); @@ -1004,7 +1004,7 @@ static void activate_task(struct rq *rq, struct task_struct *p, int wakeup) */ static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep) { - if (p->state == TASK_UNINTERRUPTIBLE) + if (task_contributes_to_load(p)) rq->nr_uninterruptible++; dequeue_task(rq, p, sleep); @@ -1646,8 +1646,7 @@ out: int fastcall wake_up_process(struct task_struct *p) { - return try_to_wake_up(p, TASK_STOPPED | TASK_TRACED | - TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0); + return try_to_wake_up(p, TASK_ALL, 0); } EXPORT_SYMBOL(wake_up_process); @@ -3857,8 +3856,7 @@ void complete(struct completion *x) spin_lock_irqsave(&x->wait.lock, flags); x->done++; - __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, - 1, 0, NULL); + __wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL); spin_unlock_irqrestore(&x->wait.lock, flags); } EXPORT_SYMBOL(complete); @@ -3869,8 +3867,7 @@ void complete_all(struct completion *x) spin_lock_irqsave(&x->wait.lock, flags); x->done += UINT_MAX/2; - __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, - 0, 0, NULL); + __wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL); spin_unlock_irqrestore(&x->wait.lock, flags); } EXPORT_SYMBOL(complete_all); -- cgit v1.2.3 From e1abb39c60971590b6580c0d3a12119dcbad9c50 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 6 Dec 2007 11:07:35 -0500 Subject: signal: Use task_is_* Signed-off-by: Matthew Wilcox --- kernel/signal.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/signal.c b/kernel/signal.c index afa4f781f92..c5a401aa715 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -838,7 +838,7 @@ static inline int wants_signal(int sig, struct task_struct *p) return 0; if (sig == SIGKILL) return 1; - if (p->state & (TASK_STOPPED | TASK_TRACED)) + if (task_is_stopped_or_traced(p)) return 0; return task_curr(p) || !signal_pending(p); } @@ -1441,7 +1441,7 @@ void do_notify_parent(struct task_struct *tsk, int sig) BUG_ON(sig == -1); /* do_notify_parent_cldstop should have been called instead. */ - BUG_ON(tsk->state & (TASK_STOPPED|TASK_TRACED)); + BUG_ON(task_is_stopped_or_traced(tsk)); BUG_ON(!tsk->ptrace && (tsk->group_leader != tsk || !thread_group_empty(tsk))); @@ -1729,7 +1729,7 @@ static int do_signal_stop(int signr) * so this check has no races. */ if (!t->exit_state && - !(t->state & (TASK_STOPPED|TASK_TRACED))) { + !task_is_stopped_or_traced(t)) { stop_count++; signal_wake_up(t, 0); } -- cgit v1.2.3 From 338077e54e17e656da470a84724b773816207316 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 6 Dec 2007 11:09:35 -0500 Subject: exit: Use task_is_* Also restructure the loop in do_wait() Signed-off-by: Matthew Wilcox --- kernel/exit.c | 88 ++++++++++++++++++++++++++--------------------------------- 1 file changed, 39 insertions(+), 49 deletions(-) (limited to 'kernel') diff --git a/kernel/exit.c b/kernel/exit.c index 549c0558ba6..bfb1c0e940e 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -249,7 +249,7 @@ static int has_stopped_jobs(struct pid *pgrp) struct task_struct *p; do_each_pid_task(pgrp, PIDTYPE_PGID, p) { - if (p->state != TASK_STOPPED) + if (!task_is_stopped(p)) continue; retval = 1; break; @@ -614,7 +614,7 @@ reparent_thread(struct task_struct *p, struct task_struct *father, int traced) p->parent = p->real_parent; add_parent(p); - if (p->state == TASK_TRACED) { + if (task_is_traced(p)) { /* * If it was at a trace stop, turn it into * a normal stop since it's no longer being @@ -1563,60 +1563,51 @@ repeat: } allowed = 1; - switch (p->state) { - case TASK_TRACED: - /* - * When we hit the race with PTRACE_ATTACH, - * we will not report this child. But the - * race means it has not yet been moved to - * our ptrace_children list, so we need to - * set the flag here to avoid a spurious ECHILD - * when the race happens with the only child. - */ - flag = 1; - if (!my_ptrace_child(p)) - continue; - /*FALLTHROUGH*/ - case TASK_STOPPED: + if (task_is_stopped_or_traced(p)) { /* * It's stopped now, so it might later * continue, exit, or stop again. + * + * When we hit the race with PTRACE_ATTACH, we + * will not report this child. But the race + * means it has not yet been moved to our + * ptrace_children list, so we need to set the + * flag here to avoid a spurious ECHILD when + * the race happens with the only child. */ flag = 1; - if (!(options & WUNTRACED) && - !my_ptrace_child(p)) - continue; + + if (!my_ptrace_child(p)) { + if (task_is_traced(p)) + continue; + if (!(options & WUNTRACED)) + continue; + } + retval = wait_task_stopped(p, ret == 2, - (options & WNOWAIT), - infop, - stat_addr, ru); + (options & WNOWAIT), infop, + stat_addr, ru); if (retval == -EAGAIN) goto repeat; if (retval != 0) /* He released the lock. */ goto end; - break; - default: - // case EXIT_DEAD: - if (p->exit_state == EXIT_DEAD) + } else if (p->exit_state == EXIT_DEAD) { + continue; + } else if (p->exit_state == EXIT_ZOMBIE) { + /* + * Eligible but we cannot release it yet: + */ + if (ret == 2) + goto check_continued; + if (!likely(options & WEXITED)) continue; - // case EXIT_ZOMBIE: - if (p->exit_state == EXIT_ZOMBIE) { - /* - * Eligible but we cannot release - * it yet: - */ - if (ret == 2) - goto check_continued; - if (!likely(options & WEXITED)) - continue; - retval = wait_task_zombie( - p, (options & WNOWAIT), - infop, stat_addr, ru); - /* He released the lock. */ - if (retval != 0) - goto end; - break; - } + retval = wait_task_zombie(p, + (options & WNOWAIT), infop, + stat_addr, ru); + /* He released the lock. */ + if (retval != 0) + goto end; + } else { check_continued: /* * It's running now, so it might later @@ -1625,12 +1616,11 @@ check_continued: flag = 1; if (!unlikely(options & WCONTINUED)) continue; - retval = wait_task_continued( - p, (options & WNOWAIT), - infop, stat_addr, ru); + retval = wait_task_continued(p, + (options & WNOWAIT), infop, + stat_addr, ru); if (retval != 0) /* He released the lock. */ goto end; - break; } } if (!flag) { -- cgit v1.2.3 From f021a3c2b14d0dd082c2cee890c204d9e1dee52b Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 6 Dec 2007 11:13:16 -0500 Subject: Add TASK_WAKEKILL Set TASK_WAKEKILL for TASK_STOPPED and TASK_TRACED, add TASK_KILLABLE and use TASK_WAKEKILL in signal_wake_up() Signed-off-by: Matthew Wilcox --- kernel/signal.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/signal.c b/kernel/signal.c index c5a401aa715..fd4797f3062 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -456,15 +456,15 @@ void signal_wake_up(struct task_struct *t, int resume) set_tsk_thread_flag(t, TIF_SIGPENDING); /* - * For SIGKILL, we want to wake it up in the stopped/traced case. - * We don't check t->state here because there is a race with it + * For SIGKILL, we want to wake it up in the stopped/traced/killable + * case. We don't check t->state here because there is a race with it * executing another processor and just now entering stopped state. * By using wake_up_state, we ensure the process will wake up and * handle its death signal. */ mask = TASK_INTERRUPTIBLE; if (resume) - mask |= TASK_STOPPED | TASK_TRACED; + mask |= TASK_WAKEKILL; if (!wake_up_state(t, mask)) kick_process(t); } @@ -620,7 +620,7 @@ static void handle_stop_signal(int sig, struct task_struct *p) * Wake up the stopped thread _after_ setting * TIF_SIGPENDING */ - state = TASK_STOPPED; + state = __TASK_STOPPED; if (sig_user_defined(t, SIGCONT) && !sigismember(&t->blocked, SIGCONT)) { set_tsk_thread_flag(t, TIF_SIGPENDING); state |= TASK_INTERRUPTIBLE; -- cgit v1.2.3 From f776d12dd16da1b0cd55a1240002c1b31f315d5d Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 6 Dec 2007 11:15:50 -0500 Subject: Add fatal_signal_pending Like signal_pending, but it's only true for signals which are fatal to this process Signed-off-by: Matthew Wilcox --- kernel/signal.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/signal.c b/kernel/signal.c index fd4797f3062..657aa16d97c 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -994,6 +994,11 @@ void zap_other_threads(struct task_struct *p) } } +int fastcall __fatal_signal_pending(struct task_struct *tsk) +{ + return sigismember(&tsk->pending.signal, SIGKILL); +} + /* * Must be called under rcu_read_lock() or with tasklist_lock read-held. */ -- cgit v1.2.3 From ad776537cc6b4b936cfd11893e7b698dfa072666 Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Thu, 6 Dec 2007 17:37:59 -0500 Subject: Add mutex_lock_killable Similar to mutex_lock_interruptible, it can be interrupted by a fatal signal only. Signed-off-by: Liam R. Howlett Acked-by: Ingo Molnar Signed-off-by: Matthew Wilcox --- kernel/mutex.c | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/mutex.c b/kernel/mutex.c index d7fe50cc556..d9ec9b66625 100644 --- a/kernel/mutex.c +++ b/kernel/mutex.c @@ -166,9 +166,12 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, * got a signal? (This code gets eliminated in the * TASK_UNINTERRUPTIBLE case.) */ - if (unlikely(state == TASK_INTERRUPTIBLE && - signal_pending(task))) { - mutex_remove_waiter(lock, &waiter, task_thread_info(task)); + if (unlikely((state == TASK_INTERRUPTIBLE && + signal_pending(task)) || + (state == TASK_KILLABLE && + fatal_signal_pending(task)))) { + mutex_remove_waiter(lock, &waiter, + task_thread_info(task)); mutex_release(&lock->dep_map, 1, ip); spin_unlock_mutex(&lock->wait_lock, flags); @@ -210,6 +213,14 @@ mutex_lock_nested(struct mutex *lock, unsigned int subclass) EXPORT_SYMBOL_GPL(mutex_lock_nested); +int __sched +mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass) +{ + might_sleep(); + return __mutex_lock_common(lock, TASK_KILLABLE, subclass, _RET_IP_); +} +EXPORT_SYMBOL_GPL(mutex_lock_killable_nested); + int __sched mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass) { @@ -272,6 +283,9 @@ __mutex_unlock_slowpath(atomic_t *lock_count) * mutex_lock_interruptible() and mutex_trylock(). */ static int fastcall noinline __sched +__mutex_lock_killable_slowpath(atomic_t *lock_count); + +static noinline int fastcall __sched __mutex_lock_interruptible_slowpath(atomic_t *lock_count); /*** @@ -294,6 +308,14 @@ int fastcall __sched mutex_lock_interruptible(struct mutex *lock) EXPORT_SYMBOL(mutex_lock_interruptible); +int fastcall __sched mutex_lock_killable(struct mutex *lock) +{ + might_sleep(); + return __mutex_fastpath_lock_retval + (&lock->count, __mutex_lock_killable_slowpath); +} +EXPORT_SYMBOL(mutex_lock_killable); + static void fastcall noinline __sched __mutex_lock_slowpath(atomic_t *lock_count) { @@ -303,6 +325,14 @@ __mutex_lock_slowpath(atomic_t *lock_count) } static int fastcall noinline __sched +__mutex_lock_killable_slowpath(atomic_t *lock_count) +{ + struct mutex *lock = container_of(lock_count, struct mutex, count); + + return __mutex_lock_common(lock, TASK_KILLABLE, 0, _RET_IP_); +} + +static noinline int fastcall __sched __mutex_lock_interruptible_slowpath(atomic_t *lock_count) { struct mutex *lock = container_of(lock_count, struct mutex, count); -- cgit v1.2.3 From 294d5cc233d81ec4aec77ebc60dc5752a3d0082a Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 6 Dec 2007 11:59:46 -0500 Subject: Add schedule_timeout_killable Signed-off-by: Matthew Wilcox --- kernel/timer.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'kernel') diff --git a/kernel/timer.c b/kernel/timer.c index a05817c021d..66d7d8bca1a 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -1100,6 +1100,13 @@ signed long __sched schedule_timeout_interruptible(signed long timeout) } EXPORT_SYMBOL(schedule_timeout_interruptible); +signed long __sched schedule_timeout_killable(signed long timeout) +{ + __set_current_state(TASK_KILLABLE); + return schedule_timeout(timeout); +} +EXPORT_SYMBOL(schedule_timeout_killable); + signed long __sched schedule_timeout_uninterruptible(signed long timeout) { __set_current_state(TASK_UNINTERRUPTIBLE); -- cgit v1.2.3 From 009e577e079656d51d0fe9b15e61e41b00816c29 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 6 Dec 2007 12:29:54 -0500 Subject: Add wait_for_completion_killable Signed-off-by: Matthew Wilcox --- kernel/sched.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 50a0faae585..d2f77fab0f4 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -3881,8 +3881,10 @@ do_wait_for_common(struct completion *x, long timeout, int state) wait.flags |= WQ_FLAG_EXCLUSIVE; __add_wait_queue_tail(&x->wait, &wait); do { - if (state == TASK_INTERRUPTIBLE && - signal_pending(current)) { + if ((state == TASK_INTERRUPTIBLE && + signal_pending(current)) || + (state == TASK_KILLABLE && + fatal_signal_pending(current))) { __remove_wait_queue(&x->wait, &wait); return -ERESTARTSYS; } @@ -3942,6 +3944,15 @@ wait_for_completion_interruptible_timeout(struct completion *x, } EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); +int __sched wait_for_completion_killable(struct completion *x) +{ + long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); + if (t == -ERESTARTSYS) + return t; + return 0; +} +EXPORT_SYMBOL(wait_for_completion_killable); + static long __sched sleep_on_common(wait_queue_head_t *q, int state, long timeout) { -- cgit v1.2.3 From 5f9fa8a62d6a98f5cb2ee2e00b85bfe95e45888d Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 7 Dec 2007 19:02:47 +0100 Subject: lockdep: make cli/sti annotation warnings clearer make cli/sti annotation warnings easier to interpret. Signed-off-by: Ingo Molnar Signed-off-by: Peter Zijlstra --- kernel/lockdep.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 0f389621bb6..723bd9f9255 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -2654,10 +2654,15 @@ static void check_flags(unsigned long flags) if (!debug_locks) return; - if (irqs_disabled_flags(flags)) - DEBUG_LOCKS_WARN_ON(current->hardirqs_enabled); - else - DEBUG_LOCKS_WARN_ON(!current->hardirqs_enabled); + if (irqs_disabled_flags(flags)) { + if (DEBUG_LOCKS_WARN_ON(current->hardirqs_enabled)) { + printk("possible reason: unannotated irqs-off.\n"); + } + } else { + if (DEBUG_LOCKS_WARN_ON(!current->hardirqs_enabled)) { + printk("possible reason: unannotated irqs-on.\n"); + } + } /* * We dont accurately track softirq state in e.g. -- cgit v1.2.3 From 8ced5f69e4bc09adcc6442e090e2e64c197246cf Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 7 Dec 2007 19:02:47 +0100 Subject: sched: enable early use of sched_clock() some platforms have sched_clock() implementations that cannot be called very early during wakeup. If it's called it might hang or crash in hard to debug ways. So only call update_rq_clock() [which calls sched_clock()] if sched_init() has already been called. (rq->idle is NULL before the scheduler is initialized.) Signed-off-by: Ingo Molnar --- kernel/sched.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 67d9d1799d8..c6e551de795 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -488,7 +488,12 @@ unsigned long long cpu_clock(int cpu) local_irq_save(flags); rq = cpu_rq(cpu); - update_rq_clock(rq); + /* + * Only call sched_clock() if the scheduler has already been + * initialized (some code might call cpu_clock() very early): + */ + if (rq->idle) + update_rq_clock(rq); now = rq->clock; local_irq_restore(flags); -- cgit v1.2.3 From 62f0f61e6673e67151a7c8c0f9a09c7ea43fe2b5 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 7 Dec 2007 19:16:17 +0100 Subject: hrtimers: avoid overflow for large relative timeouts Relative hrtimers with a large timeout value might end up as negative timer values, when the current time is added in hrtimer_start(). This in turn is causing the clockevents_set_next() function to set an huge timeout and sleep for quite a long time when we have a clock source which is capable of long sleeps like HPET. With PIT this almost goes unnoticed as the maximum delta is ~27ms. The non-hrt/nohz code sorts this out in the next timer interrupt, so we never noticed that problem which has been there since the first day of hrtimers. This bug became more apparent in 2.6.24 which activates HPET on more hardware. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- kernel/hrtimer.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'kernel') diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 22a25142e4c..e65dd0b47cd 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -850,6 +850,14 @@ hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) #ifdef CONFIG_TIME_LOW_RES tim = ktime_add(tim, base->resolution); #endif + /* + * Careful here: User space might have asked for a + * very long sleep, so the add above might result in a + * negative number, which enqueues the timer in front + * of the queue. + */ + if (tim.tv64 < 0) + tim.tv64 = KTIME_MAX; } timer->expires = tim; -- cgit v1.2.3 From 167b1de3ee4e50d65a2bd0a2667c9cd48faf54f3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 7 Dec 2007 19:16:17 +0100 Subject: clockevents: warn once when program_event() is called with negative expiry The hrtimer problem with large relative timeouts resulting in a negative expiry time went unnoticed as there is no check in the clockevents_program_event() code. Put a check there with a WARN_ONCE to avoid such problems in the future. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- kernel/time/clockevents.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 822beebe664..5fb139fef9f 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -78,6 +78,11 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires, unsigned long long clc; int64_t delta; + if (unlikely(expires.tv64 < 0)) { + WARN_ON_ONCE(1); + return -ETIME; + } + delta = ktime_to_ns(ktime_sub(expires, now)); if (delta <= 0) -- cgit v1.2.3 From d1c3fb1f8f29c41b0d098d7cfb3c32939043631f Mon Sep 17 00:00:00 2001 From: Nishanth Aravamudan Date: Mon, 17 Dec 2007 16:20:12 -0800 Subject: hugetlb: introduce nr_overcommit_hugepages sysctl hugetlb: introduce nr_overcommit_hugepages sysctl While examining the code to support /proc/sys/vm/hugetlb_dynamic_pool, I became convinced that having a boolean sysctl was insufficient: 1) To support per-node control of hugepages, I have previously submitted patches to add a sysfs attribute related to nr_hugepages. However, with a boolean global value and per-mount quota enforcement constraining the dynamic pool, adding corresponding control of the dynamic pool on a per-node basis seems inconsistent to me. 2) Administration of the hugetlb dynamic pool with multiple hugetlbfs mount points is, arguably, more arduous than it needs to be. Each quota would need to be set separately, and the sum would need to be monitored. To ease the administration, and to help make the way for per-node control of the static & dynamic hugepage pool, I added a separate sysctl, nr_overcommit_hugepages. This value serves as a high watermark for the overall hugepage pool, while nr_hugepages serves as a low watermark. The boolean sysctl can then be removed, as the condition nr_overcommit_hugepages > 0 indicates the same administrative setting as hugetlb_dynamic_pool == 1 Quotas still serve as local enforcement of the size of the pool on a per-mount basis. A few caveats: 1) There is a race whereby the global surplus huge page counter is incremented before a hugepage has allocated. Another process could then try grow the pool, and fail to convert a surplus huge page to a normal huge page and instead allocate a fresh huge page. I believe this is benign, as no memory is leaked (the actual pages are still tracked correctly) and the counters won't go out of sync. 2) Shrinking the static pool while a surplus is in effect will allow the number of surplus huge pages to exceed the overcommit value. As long as this condition holds, however, no more surplus huge pages will be allowed on the system until one of the two sysctls are increased sufficiently, or the surplus huge pages go out of use and are freed. Successfully tested on x86_64 with the current libhugetlbfs snapshot, modified to use the new sysctl. Signed-off-by: Nishanth Aravamudan Acked-by: Adam Litke Cc: William Lee Irwin III Cc: Dave Hansen Cc: David Gibson Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/sysctl.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'kernel') diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 8ac51714b08..b85a1282605 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -912,6 +912,14 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = &proc_dointvec, }, + { + .ctl_name = CTL_UNNUMBERED, + .procname = "nr_overcommit_hugepages", + .data = &nr_overcommit_huge_pages, + .maxlen = sizeof(nr_overcommit_huge_pages), + .mode = 0644, + .proc_handler = &proc_doulongvec_minmax, + }, #endif { .ctl_name = VM_LOWMEM_RESERVE_RATIO, -- cgit v1.2.3 From 368d2c6358c3c62b3820a8a73f9fe9c8b540cdea Mon Sep 17 00:00:00 2001 From: Nishanth Aravamudan Date: Mon, 17 Dec 2007 16:20:22 -0800 Subject: Revert "hugetlb: Add hugetlb_dynamic_pool sysctl" This reverts commit 54f9f80d6543fb7b157d3b11e2e7911dc1379790 ("hugetlb: Add hugetlb_dynamic_pool sysctl") Given the new sysctl nr_overcommit_hugepages, the boolean dynamic pool sysctl is not needed, as its semantics can be expressed by 0 in the overcommit sysctl (no dynamic pool) and non-0 in the overcommit sysctl (pool enabled). (Needed in 2.6.24 since it reverts a post-2.6.23 userspace-visible change) Signed-off-by: Nishanth Aravamudan Acked-by: Adam Litke Cc: William Lee Irwin III Cc: Dave Hansen Cc: David Gibson Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/sysctl.c | 8 -------- 1 file changed, 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sysctl.c b/kernel/sysctl.c index b85a1282605..1135de73087 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -904,14 +904,6 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = &hugetlb_treat_movable_handler, }, - { - .ctl_name = CTL_UNNUMBERED, - .procname = "hugetlb_dynamic_pool", - .data = &hugetlb_dynamic_pool, - .maxlen = sizeof(hugetlb_dynamic_pool), - .mode = 0644, - .proc_handler = &proc_dointvec, - }, { .ctl_name = CTL_UNNUMBERED, .procname = "nr_overcommit_hugepages", -- cgit v1.2.3 From 64396accc2831fcbdc7d793edc25481a5ebc75b2 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Mon, 17 Dec 2007 16:20:28 -0800 Subject: sysctl: fix ax25 checks Fix: sysctl table check failed: /net/ax25/ax0/ax25_default_mode .3.9.1.2 Unknown sysctl binary path Pid: 2936, comm: kissattach Not tainted 2.6.24-rc5 #1 [] set_fail+0x3b/0x43 [] sysctl_check_table+0x408/0x456 [] sysctl_check_table+0x41c/0x456 [] sysctl_check_table+0x41c/0x456 ... Signed-off-by: Eric W. Biederman Cc: Bernard Pidoux Cc: "David S. Miller" Cc: "Rafael J. Wysocki" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/sysctl_check.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index bed939f82c3..a68425a5cc1 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c @@ -428,7 +428,7 @@ static struct trans_ctl_table trans_net_netrom_table[] = { {} }; -static struct trans_ctl_table trans_net_ax25_table[] = { +static struct trans_ctl_table trans_net_ax25_param_table[] = { { NET_AX25_IP_DEFAULT_MODE, "ip_default_mode" }, { NET_AX25_DEFAULT_MODE, "ax25_default_mode" }, { NET_AX25_BACKOFF_TYPE, "backoff_type" }, @@ -446,6 +446,11 @@ static struct trans_ctl_table trans_net_ax25_table[] = { {} }; +static struct trans_ctl_table trans_net_ax25_table[] = { + { 0, NULL, trans_net_ax25_param_table }, + {} +}; + static struct trans_ctl_table trans_net_bridge_table[] = { { NET_BRIDGE_NF_CALL_ARPTABLES, "bridge-nf-call-arptables" }, { NET_BRIDGE_NF_CALL_IPTABLES, "bridge-nf-call-iptables" }, -- cgit v1.2.3 From 051a1d1afa47206e23ae03f781c6795ce870e3d5 Mon Sep 17 00:00:00 2001 From: Dmitry Adamushko Date: Tue, 18 Dec 2007 15:21:13 +0100 Subject: sched: fix crash on ia64, introduce task_current() Some services (e.g. sched_setscheduler(), rt_mutex_setprio() and sched_move_task()) must handle a given task differently in case it's the 'rq->curr' task on its run-queue. The task_running() interface is not suitable for determining such tasks for platforms with one of the following options: #define __ARCH_WANT_UNLOCKED_CTXSW #define __ARCH_WANT_INTERRUPTS_ON_CTXSW Due to the fact that it makes use of 'p->oncpu == 1' as a criterion but such a task is not necessarily 'rq->curr'. The detailed explanation is available here: https://lists.linux-foundation.org/pipermail/containers/2007-December/009262.html Signed-off-by: Dmitry Adamushko Signed-off-by: Ingo Molnar Tested-by: Dhaval Giani Tested-by: KAMEZAWA Hiroyuki --- kernel/sched.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index c6e551de795..5ae0d4296e7 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -508,10 +508,15 @@ EXPORT_SYMBOL_GPL(cpu_clock); # define finish_arch_switch(prev) do { } while (0) #endif +static inline int task_current(struct rq *rq, struct task_struct *p) +{ + return rq->curr == p; +} + #ifndef __ARCH_WANT_UNLOCKED_CTXSW static inline int task_running(struct rq *rq, struct task_struct *p) { - return rq->curr == p; + return task_current(rq, p); } static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) @@ -540,7 +545,7 @@ static inline int task_running(struct rq *rq, struct task_struct *p) #ifdef CONFIG_SMP return p->oncpu; #else - return rq->curr == p; + return task_current(rq, p); #endif } @@ -3334,7 +3339,7 @@ unsigned long long task_sched_runtime(struct task_struct *p) rq = task_rq_lock(p, &flags); ns = p->se.sum_exec_runtime; - if (rq->curr == p) { + if (task_current(rq, p)) { update_rq_clock(rq); delta_exec = rq->clock - p->se.exec_start; if ((s64)delta_exec > 0) @@ -4021,7 +4026,7 @@ void rt_mutex_setprio(struct task_struct *p, int prio) oldprio = p->prio; on_rq = p->se.on_rq; - running = task_running(rq, p); + running = task_current(rq, p); if (on_rq) { dequeue_task(rq, p, 0); if (running) @@ -4332,7 +4337,7 @@ recheck: } update_rq_clock(rq); on_rq = p->se.on_rq; - running = task_running(rq, p); + running = task_current(rq, p); if (on_rq) { deactivate_task(rq, p, 0); if (running) @@ -7101,7 +7106,7 @@ void sched_move_task(struct task_struct *tsk) update_rq_clock(rq); - running = task_running(rq, tsk); + running = task_current(rq, tsk); on_rq = tsk->se.on_rq; if (on_rq) { -- cgit v1.2.3 From c7af77b584b02d3e321b00203a618a9c93782121 Mon Sep 17 00:00:00 2001 From: Livio Soares Date: Tue, 18 Dec 2007 15:21:13 +0100 Subject: sched: mark rwsem functions as __sched for wchan/profiling This following commit http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commitdiff;h=fdf8cb0909b531f9ae8f9b9d7e4eb35ba3505f07 un-inlined a low-level rwsem function, but did not mark it as __sched. The result is that it now shows up as thread wchan (which also affects /proc/profile stats). The following simple patch fixes this by properly marking rwsem_down_failed_common() as a __sched function. Also in this patch, which is up for discussion, marks down_read() and down_write() proper as __sched. For profiling, it is pretty much useless to know that a semaphore is beig help - it is necessary to know _which_ one. By going up another frame on the stack, the information becomes much more useful. In summary, the below change to lib/rwsem.c should be applied; the changes to kernel/rwsem.c could be applied if other kernel hackers agree with my proposal that down_read()/down_write() in the profile is not enough. [ akpm@linux-foundation.org: build fix ] Signed-off-by: Livio Soares Signed-off-by: Andrew Morton Signed-off-by: Ingo Molnar --- kernel/rwsem.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/rwsem.c b/kernel/rwsem.c index 1ec620c0306..cae050b05f5 100644 --- a/kernel/rwsem.c +++ b/kernel/rwsem.c @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -15,7 +16,7 @@ /* * lock for reading */ -void down_read(struct rw_semaphore *sem) +void __sched down_read(struct rw_semaphore *sem) { might_sleep(); rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); @@ -42,7 +43,7 @@ EXPORT_SYMBOL(down_read_trylock); /* * lock for writing */ -void down_write(struct rw_semaphore *sem) +void __sched down_write(struct rw_semaphore *sem) { might_sleep(); rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); -- cgit v1.2.3 From 73c4efd2c88a41c8a4810904266a34423b5584e5 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 18 Dec 2007 15:21:13 +0100 Subject: sched: sysctl, proc_dointvec_minmax() expects int values for min_sched_granularity_ns, max_sched_granularity_ns, min_wakeup_granularity_ns and max_wakeup_granularity_ns are declared "unsigned long". This is incorrect since proc_dointvec_minmax() expects plain "int" guard values. This bug only triggers on big endian 64 bit arches. Signed-off-by: Eric Dumazet Signed-off-by: Ingo Molnar --- kernel/sysctl.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 1135de73087..c68f68dcc60 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -225,10 +225,10 @@ static struct ctl_table root_table[] = { }; #ifdef CONFIG_SCHED_DEBUG -static unsigned long min_sched_granularity_ns = 100000; /* 100 usecs */ -static unsigned long max_sched_granularity_ns = NSEC_PER_SEC; /* 1 second */ -static unsigned long min_wakeup_granularity_ns; /* 0 usecs */ -static unsigned long max_wakeup_granularity_ns = NSEC_PER_SEC; /* 1 second */ +static int min_sched_granularity_ns = 100000; /* 100 usecs */ +static int max_sched_granularity_ns = NSEC_PER_SEC; /* 1 second */ +static int min_wakeup_granularity_ns; /* 0 usecs */ +static int max_wakeup_granularity_ns = NSEC_PER_SEC; /* 1 second */ #endif static struct ctl_table kern_table[] = { -- cgit v1.2.3 From 2bacec8c318ca0418c0ee9ac662ee44207765dd4 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 18 Dec 2007 15:21:13 +0100 Subject: sched: touch softlockup watchdog after idling touch softlockup watchdog after idling. Signed-off-by: Ingo Molnar --- kernel/sched.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 5ae0d4296e7..3df84ea6aba 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -668,6 +668,7 @@ void sched_clock_idle_wakeup_event(u64 delta_ns) struct rq *rq = cpu_rq(smp_processor_id()); u64 now = sched_clock(); + touch_softlockup_watchdog(); rq->idle_clock += delta_ns; /* * Override the previous timestamp and ignore all -- cgit v1.2.3 From 6cbf1c126cf6a727287d61b122fde00a8b827bfe Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 18 Dec 2007 15:21:13 +0100 Subject: sched: do not hurt SCHED_BATCH on wakeup measurements by Yanmin Zhang have shown that SCHED_BATCH tasks benefit if they run the same place_entity() logic as SCHED_OTHER tasks - so uniformize behavior in this area. Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index c33f0ceb3de..da7c061e720 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -511,8 +511,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) if (!initial) { /* sleeps upto a single latency don't count. */ - if (sched_feat(NEW_FAIR_SLEEPERS) && entity_is_task(se) && - task_of(se)->policy != SCHED_BATCH) + if (sched_feat(NEW_FAIR_SLEEPERS) && entity_is_task(se)) vruntime -= sysctl_sched_latency; /* ensure we never gain time by being placed backwards. */ -- cgit v1.2.3 From cdc6f27d9e3c2f7ca1a3e19c6eabb1ad6a2add5d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 18 Dec 2007 18:05:58 +0100 Subject: clockevents: fix reprogramming decision in oneshot broadcast Resolve the following regression of a choppy, almost unusable laptop: http://lkml.org/lkml/2007/12/7/299 http://bugzilla.kernel.org/show_bug.cgi?id=9525 A previous version of the code did the reprogramming of the broadcast device in the return from idle code. This was removed, but the logic in tick_handle_oneshot_broadcast() was kept the same. When a broadcast interrupt happens we signal the expiry to all CPUs which have an expired event. If none of the CPUs has an expired event, which can happen in dyntick mode, then we reprogram the broadcast device. We do not reprogram otherwise, but this is only correct if all CPUs, which are in the idle broadcast state have been woken up. The code ignores, that there might be pending not yet expired events on other CPUs, which are in the idle broadcast state. So the delivery of those events can be delayed for quite a time. Change the tick_handle_oneshot_broadcast() function to check for CPUs, which are in broadcast state and are not woken up by the current event, and enforce the rearming of the broadcast device for those CPUs. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- kernel/time/tick-broadcast.c | 56 +++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 35 deletions(-) (limited to 'kernel') diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index aa82d7bf478..5b86698faa0 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -383,33 +383,6 @@ int tick_resume_broadcast_oneshot(struct clock_event_device *bc) return 0; } -/* - * Reprogram the broadcast device: - * - * Called with tick_broadcast_lock held and interrupts disabled. - */ -static int tick_broadcast_reprogram(void) -{ - ktime_t expires = { .tv64 = KTIME_MAX }; - struct tick_device *td; - int cpu; - - /* - * Find the event which expires next: - */ - for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS; - cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) { - td = &per_cpu(tick_cpu_device, cpu); - if (td->evtdev->next_event.tv64 < expires.tv64) - expires = td->evtdev->next_event; - } - - if (expires.tv64 == KTIME_MAX) - return 0; - - return tick_broadcast_set_event(expires, 0); -} - /* * Handle oneshot mode broadcasting */ @@ -417,12 +390,13 @@ static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) { struct tick_device *td; cpumask_t mask; - ktime_t now; + ktime_t now, next_event; int cpu; spin_lock(&tick_broadcast_lock); again: dev->next_event.tv64 = KTIME_MAX; + next_event.tv64 = KTIME_MAX; mask = CPU_MASK_NONE; now = ktime_get(); /* Find all expired events */ @@ -431,19 +405,31 @@ again: td = &per_cpu(tick_cpu_device, cpu); if (td->evtdev->next_event.tv64 <= now.tv64) cpu_set(cpu, mask); + else if (td->evtdev->next_event.tv64 < next_event.tv64) + next_event.tv64 = td->evtdev->next_event.tv64; } /* - * Wakeup the cpus which have an expired event. The broadcast - * device is reprogrammed in the return from idle code. + * Wakeup the cpus which have an expired event. + */ + tick_do_broadcast(mask); + + /* + * Two reasons for reprogram: + * + * - The global event did not expire any CPU local + * events. This happens in dyntick mode, as the maximum PIT + * delta is quite small. + * + * - There are pending events on sleeping CPUs which were not + * in the event mask */ - if (!tick_do_broadcast(mask)) { + if (next_event.tv64 != KTIME_MAX) { /* - * The global event did not expire any CPU local - * events. This happens in dyntick mode, as the - * maximum PIT delta is quite small. + * Rearm the broadcast device. If event expired, + * repeat the above */ - if (tick_broadcast_reprogram()) + if (tick_broadcast_set_event(next_event, 0)) goto again; } spin_unlock(&tick_broadcast_lock); -- cgit v1.2.3 From b4be625852618636a6b54908c4f9d90fb29dc549 Mon Sep 17 00:00:00 2001 From: Adrian Bunk Date: Tue, 18 Dec 2007 18:05:58 +0100 Subject: timer: kernel/timer.c section fixes This patch fixes the following section mismatches with CONFIG_HOTPLUG=n, CONFIG_HOTPLUG_CPU=y: ... WARNING: vmlinux.o(.text+0x41cd3): Section mismatch: reference to .init.data:tvec_base_done.22610 (between 'timer_cpu_notify' and 'run_timer_softirq') WARNING: vmlinux.o(.text+0x41d67): Section mismatch: reference to .init.data:tvec_base_done.22610 (between 'timer_cpu_notify' and 'run_timer_softirq') ... Signed-off-by: Adrian Bunk Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/timer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/timer.c b/kernel/timer.c index a05817c021d..d4527dcef1a 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -1219,11 +1219,11 @@ asmlinkage long sys_sysinfo(struct sysinfo __user *info) */ static struct lock_class_key base_lock_keys[NR_CPUS]; -static int __devinit init_timers_cpu(int cpu) +static int __cpuinit init_timers_cpu(int cpu) { int j; tvec_base_t *base; - static char __devinitdata tvec_base_done[NR_CPUS]; + static char __cpuinitdata tvec_base_done[NR_CPUS]; if (!tvec_base_done[cpu]) { static char boot_done; -- cgit v1.2.3 From 971e5b35fb02c5088d49e6c024aab73582a35b71 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Tue, 18 Dec 2007 18:05:58 +0100 Subject: genirq: revert lazy irq disable for simple irqs In commit 76d2160147f43f982dfe881404cfde9fd0a9da21 lazy irq disabling was implemented, and the simple irq handler had a masking set to it. Remy Bohmer discovered that some devices in the ARM architecture would trigger the mask, but never unmask it. His patch to do the unmasking was questioned by Russell King about masking simple irqs to begin with. Looking further, it was discovered that the problems Remy was seeing was due to improper use of the simple handler by devices, and he later submitted patches to fix those. But the issue that was uncovered was that the simple handler should never mask. This patch reverts the masking in the simple handler. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner Acked-by: Russell King --- kernel/irq/chip.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index 9b5dff6b3f6..44019ce30a1 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -297,18 +297,13 @@ handle_simple_irq(unsigned int irq, struct irq_desc *desc) if (unlikely(desc->status & IRQ_INPROGRESS)) goto out_unlock; + desc->status &= ~(IRQ_REPLAY | IRQ_WAITING); kstat_cpu(cpu).irqs[irq]++; action = desc->action; - if (unlikely(!action || (desc->status & IRQ_DISABLED))) { - if (desc->chip->mask) - desc->chip->mask(irq); - desc->status &= ~(IRQ_REPLAY | IRQ_WAITING); - desc->status |= IRQ_PENDING; + if (unlikely(!action || (desc->status & IRQ_DISABLED))) goto out_unlock; - } - desc->status &= ~(IRQ_REPLAY | IRQ_WAITING | IRQ_PENDING); desc->status |= IRQ_INPROGRESS; spin_unlock(&desc->lock); -- cgit v1.2.3 From 67e2be02328b9a61a9c799fbdd4ec94d7da0c323 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 20 Dec 2007 15:01:17 +0100 Subject: sched: rt: account the cpu time during the tick Realtime tasks would not account their runtime during ticks. Which would lead to: struct sched_param param = { .sched_priority = 10 }; pthread_setschedparam(pthread_self(), SCHED_FIFO, ¶m); while (1) ; Not showing up in top. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/sched_rt.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index ee9c8b6529e..9ba3daa0347 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -208,6 +208,8 @@ move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, static void task_tick_rt(struct rq *rq, struct task_struct *p) { + update_curr_rt(rq); + /* * RR tasks need a special form of timeslice management. * FIFO tasks have no timeslices. -- cgit v1.2.3 From 2c3b20e91fe3a083c5d9bc79437c485866ea251c Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Thu, 20 Dec 2007 15:01:17 +0100 Subject: debug: add end-of-oops marker Right now it's nearly impossible for parsers that collect kernel crashes from logs or emails (such as www.kerneloops.org) to detect the end-of-oops condition. In addition, it's not currently possible to detect whether or not 2 oopses that look alike are actually the same oops reported twice, or are truly two unique oopses. This patch adds an end-of-oops marker, and makes the end marker include a very simple 64-bit random ID to be able to detect duplicate reports. Normally, this ID is calculated as a late_initcall() (in the hope that at that time there is enough entropy to get a unique enough ID); however for early oopses the oops_exit() function needs to generate the ID on the fly. We do this all at the _end_ of an oops printout, so this does not impact our ability to get the most important portions of a crash out to the console first. [ Sidenote: the already existing oopses-since-bootup counter we print during crashes serves as the differentiator between multiple oopses that trigger during the same bootup. ] Tested on 32-bit and 64-bit x86. Artificially injected very early crashes as well, as expected they result in this constant ID after multiple bootups: ---[ end trace ca143223eefdc828 ]--- ---[ end trace ca143223eefdc828 ]--- because the random pools are still all zero. But it all still works fine and causes no additional problems (which is the main goal of instrumentation code). Signed-off-by: Arjan van de Ven Signed-off-by: Ingo Molnar --- kernel/panic.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'kernel') diff --git a/kernel/panic.c b/kernel/panic.c index 6f6e03e9159..da4d6bac270 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -19,6 +19,7 @@ #include #include #include +#include int panic_on_oops; int tainted; @@ -265,6 +266,20 @@ void oops_enter(void) do_oops_enter_exit(); } +/* + * 64-bit random ID for oopses: + */ +static u64 oops_id; + +static int init_oops_id(void) +{ + if (!oops_id) + get_random_bytes(&oops_id, sizeof(oops_id)); + + return 0; +} +late_initcall(init_oops_id); + /* * Called when the architecture exits its oops handler, after printing * everything. @@ -272,6 +287,9 @@ void oops_enter(void) void oops_exit(void) { do_oops_enter_exit(); + init_oops_id(); + printk(KERN_WARNING "---[ end trace %016llx ]---\n", + (unsigned long long)oops_id); } #ifdef CONFIG_CC_STACKPROTECTOR -- cgit v1.2.3 From d172f4ef31bec924c6ebcb242c9d7d290811e1e5 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Sat, 22 Dec 2007 21:18:25 -0800 Subject: Modules: fix memory leak of module names Due to the change in kobject name handling, the module kobject needs to have a null release function to ensure that the name it previously set will be properly cleaned up. All of this wierdness goes away in 2.6.25 with the rework of the kobject name and cleanup logic, but this is required for 2.6.24. Thanks to Alexey Dobriyan for finding the problem, and to Kay Sievers for pointing out the simple way to fix it after I tried many complex ways. Cc: Alexey Dobriyan Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/params.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'kernel') diff --git a/kernel/params.c b/kernel/params.c index 2a4c51487e7..7686417ee00 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -697,8 +697,18 @@ static struct kset_uevent_ops module_uevent_ops = { decl_subsys(module, &module_ktype, &module_uevent_ops); int module_sysfs_initialized; +static void module_release(struct kobject *kobj) +{ + /* + * Stupid empty release function to allow the memory for the kobject to + * be properly cleaned up. This will not need to be present for 2.6.25 + * with the upcoming kobject core rework. + */ +} + static struct kobj_type module_ktype = { .sysfs_ops = &module_sysfs_ops, + .release = module_release, }; /* -- cgit v1.2.3 From fb445ee5f9bfc7cbef9e397556170c608dc02955 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sat, 29 Dec 2007 01:19:49 -0800 Subject: [SERIAL]: Fix section mismatches in Sun serial console drivers. We're exporting an __init function, oops :-) The core issue here is that add_preferred_console() is marked as __init, this makes it impossible to invoke this thing from a driver probe routine which is what the Sparc serial drivers need to do. There is no harm in dropping the __init marker. This code will actually work properly when invoked from a modular driver, except that init will probably not pick up the console change without some other support code. Then we can drop the __init from sunserial_console_match() and we're no longer exporting an __init function to modules. Signed-off-by: David S. Miller --- kernel/printk.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/printk.c b/kernel/printk.c index a30fe33de39..89011bf8c10 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -817,7 +817,7 @@ __setup("console=", console_setup); * commonly to provide a default console (ie from PROM variables) when * the user has not supplied one. */ -int __init add_preferred_console(char *name, int idx, char *options) +int add_preferred_console(char *name, int idx, char *options) { struct console_cmdline *c; int i; -- cgit v1.2.3 From 90b2628f1fe94a667330d425a7fb76ec8d2a49ec Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sun, 30 Dec 2007 17:24:35 +0100 Subject: sched: fix gcc warnings Meelis Roos reported these warnings on sparc64: CC kernel/sched.o In file included from kernel/sched.c:879: kernel/sched_debug.c: In function 'nsec_high': kernel/sched_debug.c:38: warning: comparison of distinct pointer types lacks a cast the debug check in do_div() is over-eager here, because the long long is always positive in these places. Mark this by casting them to unsigned long long. no change in code output: text data bss dec hex filename 51471 6582 376 58429 e43d sched.o.before 51471 6582 376 58429 e43d sched.o.after md5: 7f7729c111f185bf3ccea4d542abc049 sched.o.before.asm 7f7729c111f185bf3ccea4d542abc049 sched.o.after.asm Signed-off-by: Ingo Molnar --- kernel/sched_debug.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index d30467b47dd..80fbbfc0429 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c @@ -31,9 +31,9 @@ /* * Ease the printing of nsec fields: */ -static long long nsec_high(long long nsec) +static long long nsec_high(unsigned long long nsec) { - if (nsec < 0) { + if ((long long)nsec < 0) { nsec = -nsec; do_div(nsec, 1000000); return -nsec; @@ -43,9 +43,9 @@ static long long nsec_high(long long nsec) return nsec; } -static unsigned long nsec_low(long long nsec) +static unsigned long nsec_low(unsigned long long nsec) { - if (nsec < 0) + if ((long long)nsec < 0) nsec = -nsec; return do_div(nsec, 1000000); -- cgit v1.2.3 From 831830b5a2b5d413407adf380ef62fe17d6fcbf2 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 2 Jan 2008 14:09:57 +0000 Subject: restrict reading from /proc//maps to those who share ->mm or can ptrace pid Contents of /proc/*/maps is sensitive and may become sensitive after open() (e.g. if target originally shares our ->mm and later does exec on suid-root binary). Check at read() (actually, ->start() of iterator) time that mm_struct we'd grabbed and locked is - still the ->mm of target - equal to reader's ->mm or the target is ptracable by reader. Signed-off-by: Al Viro Acked-by: Rik van Riel Signed-off-by: Linus Torvalds --- kernel/ptrace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 7c76f2ffaea..0c65d306f41 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -120,7 +120,7 @@ int ptrace_check_attach(struct task_struct *child, int kill) return ret; } -static int may_attach(struct task_struct *task) +int __ptrace_may_attach(struct task_struct *task) { /* May we inspect the given task? * This check is used both for attaching with ptrace @@ -154,7 +154,7 @@ int ptrace_may_attach(struct task_struct *task) { int err; task_lock(task); - err = may_attach(task); + err = __ptrace_may_attach(task); task_unlock(task); return !err; } -- cgit v1.2.3 From b8c9a18712f7b617fda66d878ce3759c9e575ba0 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Wed, 2 Jan 2008 13:48:27 -0800 Subject: Fix kernel/ptrace.c compile problem (missing "may_attach()") The previous commit missed one use of "may_attach()" that had been renamed to __ptrace_may_attach(). Tssk, tssk, Al. Signed-off-by: Linus Torvalds --- kernel/ptrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 0c65d306f41..c25db863081 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -196,7 +196,7 @@ repeat: /* the same process cannot be attached many times */ if (task->ptrace & PT_PTRACED) goto bad; - retval = may_attach(task); + retval = __ptrace_may_attach(task); if (retval) goto bad; -- cgit v1.2.3 From b59f8197c5ddd0d5d74b663650be5449dacd34aa Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Mon, 7 Jan 2008 14:23:34 -0800 Subject: acct: real_parent ppid The ac_ppid field reported in process accounting records should match what getppid() would have returned to that process, regardless of whether a debugger is attached. Signed-off-by: Roland McGrath Signed-off-by: Linus Torvalds --- kernel/acct.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/acct.c b/kernel/acct.c index cf19547cc9e..521dfa53cb9 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -482,7 +482,7 @@ static void do_acct_process(struct file *file) #endif #if ACCT_VERSION==3 ac.ac_pid = current->tgid; - ac.ac_ppid = current->parent->tgid; + ac.ac_ppid = current->real_parent->tgid; #endif spin_lock_irq(¤t->sighand->siglock); -- cgit v1.2.3 From 83a08e7c6ed533a47631794e7f618a98094b4129 Mon Sep 17 00:00:00 2001 From: Ken'ichi Ohmichi Date: Tue, 8 Jan 2008 15:33:05 -0800 Subject: vmcoreinfo: add the array length of "free_list" for filtering free pages This patch adds the array length of "free_area.free_list" to the vmcoreinfo data so that makedumpfile (dump filtering command) can exclude all free pages in linux-2.6.24. makedumpfile creates a small dumpfile by excluding unnecessary pages for the analysis. To distinguish unnecessary pages, makedumpfile gets the vmcoreinfo data which has the minimum debugging information only for dump filtering. In 2.6.24-rc1 or later, the free_area.free_list is an array which has one list for each migrate types instead of a single list. makedumpfile needs the array length of "free_area.free_list" and the vmcoreinfo data should contain it. Signed-off-by: Huang Ying Tested-by: Ken'ichi Ohmichi Acked-by: Simon Horman Cc: David Rientjes Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/kexec.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/kexec.c b/kernel/kexec.c index aa74a1ef2da..9a26eec9eb0 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -1404,6 +1404,7 @@ static int __init crash_save_vmcoreinfo_init(void) VMCOREINFO_OFFSET(list_head, next); VMCOREINFO_OFFSET(list_head, prev); VMCOREINFO_LENGTH(zone.free_area, MAX_ORDER); + VMCOREINFO_LENGTH(free_area.free_list, MIGRATE_TYPES); VMCOREINFO_NUMBER(NR_FREE_PAGES); arch_crash_save_vmcoreinfo(); -- cgit v1.2.3 From cdf71a10c7b6432d9b48e292cca2c62a0b9fa6cf Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Jan 2008 19:47:38 +0100 Subject: futex: Prevent stale futex owner when interrupted/timeout Roland Westrelin did a great analysis of a long standing thinko in the return path of futex_lock_pi. While we fixed the lock steal case long ago, which was easy to trigger, we never had a test case which exposed this problem and stupidly never thought about the reverse lock stealing scenario and the return to user space with a stale state. When a blocked tasks returns from rt_mutex_timed_locked without holding the rt_mutex (due to a signal or timeout) and at the same time the task holding the futex is releasing the futex and assigning the ownership of the futex to the returning task, then it might happen that a third task acquires the rt_mutex before the final rt_mutex_trylock() of the returning task happens under the futex hash bucket lock. The returning task returns to user space with ETIMEOUT or EINTR, but the user space futex value is assigned to this task. The task which acquired the rt_mutex fixes the user space futex value right after the hash bucket lock has been released by the returning task, but for a short period of time the user space value is wrong. Detailed description is available at: https://bugzilla.redhat.com/show_bug.cgi?id=400541 The fix for this is the same as we do when the rt_mutex was acquired by a higher priority task via lock stealing from the designated new owner. In that case we already fix the user space value and the internal pi_state up before we return. This mechanism can be used to fixup the above corner case as well. When the returning task, which failed to acquire the rt_mutex, notices that it is the designated owner of the futex, then it fixes up the stale user space value and the pi_state, before returning to user space. This happens with the futex hash bucket lock held, so the task which acquired the rt_mutex is guaranteed to be blocked on the hash bucket lock. We can access the rt_mutex owner, which gives us the pid of the new owner, safely here as the owner is not able to modify (release) it while waiting on the hash bucket lock. Rename the "curr" argument of fixup_pi_state_owner() to "newowner" to avoid confusion with current and add the check for the stale state into the failure path of rt_mutex_trylock() in the return path of unlock_futex_pi(). If the situation is detected use fixup_pi_state_owner() to assign everything to the owner of the rt_mutex. Pointed-out-and-tested-by: Roland Westrelin Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner Signed-off-by: Linus Torvalds --- kernel/futex.c | 51 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/futex.c b/kernel/futex.c index 172a1aeeafd..db9824de8bf 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -1097,15 +1097,15 @@ static void unqueue_me_pi(struct futex_q *q) } /* - * Fixup the pi_state owner with current. + * Fixup the pi_state owner with the new owner. * * Must be called with hash bucket lock held and mm->sem held for non * private futexes. */ static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q, - struct task_struct *curr) + struct task_struct *newowner) { - u32 newtid = task_pid_vnr(curr) | FUTEX_WAITERS; + u32 newtid = task_pid_vnr(newowner) | FUTEX_WAITERS; struct futex_pi_state *pi_state = q->pi_state; u32 uval, curval, newval; int ret; @@ -1119,12 +1119,12 @@ static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q, } else newtid |= FUTEX_OWNER_DIED; - pi_state->owner = curr; + pi_state->owner = newowner; - spin_lock_irq(&curr->pi_lock); + spin_lock_irq(&newowner->pi_lock); WARN_ON(!list_empty(&pi_state->list)); - list_add(&pi_state->list, &curr->pi_state_list); - spin_unlock_irq(&curr->pi_lock); + list_add(&pi_state->list, &newowner->pi_state_list); + spin_unlock_irq(&newowner->pi_lock); /* * We own it, so we have to replace the pending owner @@ -1508,9 +1508,40 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared, * when we were on the way back before we locked the * hash bucket. */ - if (q.pi_state->owner == curr && - rt_mutex_trylock(&q.pi_state->pi_mutex)) { - ret = 0; + if (q.pi_state->owner == curr) { + /* + * Try to get the rt_mutex now. This might + * fail as some other task acquired the + * rt_mutex after we removed ourself from the + * rt_mutex waiters list. + */ + if (rt_mutex_trylock(&q.pi_state->pi_mutex)) + ret = 0; + else { + /* + * pi_state is incorrect, some other + * task did a lock steal and we + * returned due to timeout or signal + * without taking the rt_mutex. Too + * late. We can access the + * rt_mutex_owner without locking, as + * the other task is now blocked on + * the hash bucket lock. Fix the state + * up. + */ + struct task_struct *owner; + int res; + + owner = rt_mutex_owner(&q.pi_state->pi_mutex); + res = fixup_pi_state_owner(uaddr, &q, owner); + + WARN_ON(rt_mutex_owner(&q.pi_state->pi_mutex) != + owner); + + /* propagate -EFAULT, if the fixup failed */ + if (res) + ret = res; + } } else { /* * Paranoia check. If we did not take the lock -- cgit v1.2.3 From fcfd50afb6e94c8cf121ca4e7e3e7166bae7c6aa Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Wed, 9 Jan 2008 00:03:23 -0800 Subject: show_task: real_parent The show_task function invoked by sysrq-t et al displays the pid and parent's pid of each task. It seems more useful to show the actual process hierarchy here than who is using ptrace on each process. Signed-off-by: Roland McGrath Signed-off-by: Linus Torvalds --- kernel/sched.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 3df84ea6aba..37cf07aa416 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -4918,7 +4918,7 @@ static void show_task(struct task_struct *p) } #endif printk(KERN_CONT "%5lu %5d %6d\n", free, - task_pid_nr(p), task_pid_nr(p->parent)); + task_pid_nr(p), task_pid_nr(p->real_parent)); if (state != TASK_RUNNING) show_stack(p, NULL); -- cgit v1.2.3 From 9f9adecd2d0e4f88fa0e8cb06c6ec207748df70a Mon Sep 17 00:00:00 2001 From: Len Brown Date: Thu, 13 Dec 2007 17:38:03 -0500 Subject: PM: ACPI and APM must not be enabled at the same time ACPI and APM used "pm_active" to guarantee that they would not be simultaneously active. But pm_active was recently moved under CONFIG_PM_LEGACY, so that without CONFIG_PM_LEGACY, pm_active became a NOP -- allowing ACPI and APM to both be simultaneously enabled. This caused unpredictable results, including boot hangs. Further, the code under CONFIG_PM_LEGACY is scheduled for removal. So replace pm_active with pm_flags. pm_flags depends only on CONFIG_PM, which is present for both CONFIG_APM and CONFIG_ACPI. http://bugzilla.kernel.org/show_bug.cgi?id=9194 Signed-off-by: Len Brown Signed-off-by: Rafael J. Wysocki --- kernel/power/main.c | 3 +++ kernel/power/pm.c | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/power/main.c b/kernel/power/main.c index 3cdf95b1dc9..f71c9504a5c 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -28,6 +28,9 @@ BLOCKING_NOTIFIER_HEAD(pm_chain_head); DEFINE_MUTEX(pm_mutex); +unsigned int pm_flags; +EXPORT_SYMBOL(pm_flags); + #ifdef CONFIG_SUSPEND /* This is just an arbitrary number */ diff --git a/kernel/power/pm.c b/kernel/power/pm.c index c50d15266c1..60c73fa670d 100644 --- a/kernel/power/pm.c +++ b/kernel/power/pm.c @@ -27,8 +27,6 @@ #include #include -int pm_active; - /* * Locking notes: * pm_devs_lock can be a semaphore providing pm ops are not called @@ -204,6 +202,4 @@ int pm_send_all(pm_request_t rqst, void *data) EXPORT_SYMBOL(pm_register); EXPORT_SYMBOL(pm_send_all); -EXPORT_SYMBOL(pm_active); - -- cgit v1.2.3 From 84427eaef1fb91704c7112bdb598c810003b99f3 Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Thu, 10 Jan 2008 12:52:04 -0800 Subject: remove task_ppid_nr_ns task_ppid_nr_ns is called in three places. One of these should never have called it. In the other two, using it broke the existing semantics. This was presumably accidental. If the function had not been there, it would have been much more obvious to the eye that those patches were changing the behavior. We don't need this function. In task_state, the pid of the ptracer is not the ppid of the ptracer. In do_task_stat, ppid is the tgid of the real_parent, not its pid. I also moved the call outside of lock_task_sighand, since it doesn't need it. In sys_getppid, ppid is the tgid of the real_parent, not its pid. Signed-off-by: Roland McGrath Signed-off-by: Linus Torvalds --- kernel/timer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/timer.c b/kernel/timer.c index d4527dcef1a..26671f4db07 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -978,7 +978,7 @@ asmlinkage long sys_getppid(void) int pid; rcu_read_lock(); - pid = task_ppid_nr_ns(current, current->nsproxy->pid_ns); + pid = task_tgid_nr_ns(current->real_parent, current->nsproxy->pid_ns); rcu_read_unlock(); return pid; -- cgit v1.2.3 From cb2a52052cebe4716e83b9d2e53682ba00f67de6 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 14 Jan 2008 00:55:03 -0800 Subject: modules: de-mutex more symbol lookup paths in the module code Kyle McMartin reports sysrq_timer_list_show() can hit the module mutex from hard interrupt context. These paths don't need to though, since we long ago changed all the module list manipulation to occur via stop_machine(). Disabling preemption is enough. Signed-off-by: Rusty Russell Cc: Ingo Molnar Cc: Kyle McMartin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/module.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index 91fe6958b6e..c2e3e2e9880 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -2214,29 +2214,34 @@ static const char *get_ksymbol(struct module *mod, /* For kallsyms to ask for address resolution. NULL means not found. We don't lock, as this is used for oops resolution and races are a lesser concern. */ +/* FIXME: Risky: returns a pointer into a module w/o lock */ const char *module_address_lookup(unsigned long addr, unsigned long *size, unsigned long *offset, char **modname) { struct module *mod; + const char *ret = NULL; + preempt_disable(); list_for_each_entry(mod, &modules, list) { if (within(addr, mod->module_init, mod->init_size) || within(addr, mod->module_core, mod->core_size)) { if (modname) *modname = mod->name; - return get_ksymbol(mod, addr, size, offset); + ret = get_ksymbol(mod, addr, size, offset); + break; } } - return NULL; + preempt_enable(); + return ret; } int lookup_module_symbol_name(unsigned long addr, char *symname) { struct module *mod; - mutex_lock(&module_mutex); + preempt_disable(); list_for_each_entry(mod, &modules, list) { if (within(addr, mod->module_init, mod->init_size) || within(addr, mod->module_core, mod->core_size)) { @@ -2246,12 +2251,12 @@ int lookup_module_symbol_name(unsigned long addr, char *symname) if (!sym) goto out; strlcpy(symname, sym, KSYM_NAME_LEN); - mutex_unlock(&module_mutex); + preempt_enable(); return 0; } } out: - mutex_unlock(&module_mutex); + preempt_enable(); return -ERANGE; } @@ -2260,7 +2265,7 @@ int lookup_module_symbol_attrs(unsigned long addr, unsigned long *size, { struct module *mod; - mutex_lock(&module_mutex); + preempt_disable(); list_for_each_entry(mod, &modules, list) { if (within(addr, mod->module_init, mod->init_size) || within(addr, mod->module_core, mod->core_size)) { @@ -2273,12 +2278,12 @@ int lookup_module_symbol_attrs(unsigned long addr, unsigned long *size, strlcpy(modname, mod->name, MODULE_NAME_LEN); if (name) strlcpy(name, sym, KSYM_NAME_LEN); - mutex_unlock(&module_mutex); + preempt_enable(); return 0; } } out: - mutex_unlock(&module_mutex); + preempt_enable(); return -ERANGE; } @@ -2287,7 +2292,7 @@ int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type, { struct module *mod; - mutex_lock(&module_mutex); + preempt_disable(); list_for_each_entry(mod, &modules, list) { if (symnum < mod->num_symtab) { *value = mod->symtab[symnum].st_value; @@ -2296,12 +2301,12 @@ int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type, KSYM_NAME_LEN); strlcpy(module_name, mod->name, MODULE_NAME_LEN); *exported = is_exported(name, mod); - mutex_unlock(&module_mutex); + preempt_enable(); return 0; } symnum -= mod->num_symtab; } - mutex_unlock(&module_mutex); + preempt_enable(); return -ERANGE; } @@ -2324,6 +2329,7 @@ unsigned long module_kallsyms_lookup_name(const char *name) unsigned long ret = 0; /* Don't lock: we're in enough trouble already. */ + preempt_disable(); if ((colon = strchr(name, ':')) != NULL) { *colon = '\0'; if ((mod = find_module(name)) != NULL) @@ -2334,6 +2340,7 @@ unsigned long module_kallsyms_lookup_name(const char *name) if ((ret = mod_find_symname(mod, name)) != 0) break; } + preempt_enable(); return ret; } #endif /* CONFIG_KALLSYMS */ -- cgit v1.2.3 From 5a26db5bd25cf4bf32ae9fa9f6136b6b6d5b45c5 Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Wed, 16 Jan 2008 09:51:58 +0100 Subject: lockdep: fix internal double unlock during self-test Lockdep, during self-test (when it was simulating double unlocks) was sometimes unconditionally unlocking a spinlock when it had not been locked. This won't work for ticket locks. Signed-off-by: Nick Piggin Signed-off-by: Ingo Molnar Signed-off-by: Peter Zijlstra --- kernel/lockdep.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 723bd9f9255..4335f12a27c 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -2943,9 +2943,10 @@ void lockdep_free_key_range(void *start, unsigned long size) struct list_head *head; unsigned long flags; int i; + int locked; raw_local_irq_save(flags); - graph_lock(); + locked = graph_lock(); /* * Unhash all classes that were created by this module: @@ -2959,7 +2960,8 @@ void lockdep_free_key_range(void *start, unsigned long size) zap_class(class); } - graph_unlock(); + if (locked) + graph_unlock(); raw_local_irq_restore(flags); } @@ -2969,6 +2971,7 @@ void lockdep_reset_lock(struct lockdep_map *lock) struct list_head *head; unsigned long flags; int i, j; + int locked; raw_local_irq_save(flags); @@ -2987,7 +2990,7 @@ void lockdep_reset_lock(struct lockdep_map *lock) * Debug check: in the end all mapped classes should * be gone. */ - graph_lock(); + locked = graph_lock(); for (i = 0; i < CLASSHASH_SIZE; i++) { head = classhash_table + i; if (list_empty(head)) @@ -3000,7 +3003,8 @@ void lockdep_reset_lock(struct lockdep_map *lock) } } } - graph_unlock(); + if (locked) + graph_unlock(); out_restore: raw_local_irq_restore(flags); -- cgit v1.2.3 From eb13ba873881abd5e15af784756a61af635e665e Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Wed, 16 Jan 2008 09:51:58 +0100 Subject: lockdep: fix workqueue creation API lockdep interaction Dave Young reported warnings from lockdep that the workqueue API can sometimes try to register lockdep classes with the same key but different names. This is not permitted in lockdep. Unfortunately, I was unaware of that restriction when I wrote the code to debug workqueue problems with lockdep and used the workqueue name as the lockdep class name. This can obviously lead to the problem if the workqueue name is dynamic. This patch solves the problem by always using a constant name for the workqueue's lockdep class, namely either the constant name that was passed in or a string consisting of the variable name. Signed-off-by: Johannes Berg Signed-off-by: Ingo Molnar Signed-off-by: Peter Zijlstra --- kernel/workqueue.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 52d5e7c9a8e..8db0b597509 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -722,7 +722,8 @@ static void start_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) struct workqueue_struct *__create_workqueue_key(const char *name, int singlethread, int freezeable, - struct lock_class_key *key) + struct lock_class_key *key, + const char *lock_name) { struct workqueue_struct *wq; struct cpu_workqueue_struct *cwq; @@ -739,7 +740,7 @@ struct workqueue_struct *__create_workqueue_key(const char *name, } wq->name = name; - lockdep_init_map(&wq->lockdep_map, name, key, 0); + lockdep_init_map(&wq->lockdep_map, lock_name, key, 0); wq->singlethread = singlethread; wq->freezeable = freezeable; INIT_LIST_HEAD(&wq->list); -- cgit v1.2.3 From 784680336b616dcc4c17cbd25add3b49c555cdeb Mon Sep 17 00:00:00 2001 From: Nigel Cunningham Date: Thu, 17 Jan 2008 15:21:21 -0800 Subject: Fix unbalanced helper_lock in kernel/kmod.c call_usermodehelper_exec() has an exit path that can leave the helper_lock() call at the top of the routine unbalanced. The attached patch fixes this issue. Signed-off-by: Nigel Cunningham Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/kmod.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/kmod.c b/kernel/kmod.c index c6a4f8aebeb..bb7df2a28bd 100644 --- a/kernel/kmod.c +++ b/kernel/kmod.c @@ -451,13 +451,11 @@ int call_usermodehelper_exec(struct subprocess_info *sub_info, enum umh_wait wait) { DECLARE_COMPLETION_ONSTACK(done); - int retval; + int retval = 0; helper_lock(); - if (sub_info->path[0] == '\0') { - retval = 0; + if (sub_info->path[0] == '\0') goto out; - } if (!khelper_wq || usermodehelper_disabled) { retval = -EBUSY; @@ -468,13 +466,14 @@ int call_usermodehelper_exec(struct subprocess_info *sub_info, sub_info->wait = wait; queue_work(khelper_wq, &sub_info->work); - if (wait == UMH_NO_WAIT) /* task has freed sub_info */ - return 0; + if (wait == UMH_NO_WAIT) /* task has freed sub_info */ + goto unlock; wait_for_completion(&done); retval = sub_info->retval; - out: +out: call_usermodehelper_freeinfo(sub_info); +unlock: helper_unlock(); return retval; } -- cgit v1.2.3 From 0ec160dd48b666ddef39d639323d0da26d0b710d Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Mon, 21 Jan 2008 17:18:24 -0800 Subject: hrtimer: fix section mismatch Fix section mismatch in hrtimer.c: WARNING: vmlinux.o(.text+0x50c61): Section mismatch: reference to .init.text: (between 'hrtimer_cpu_notify' and 'down_read_trylock') Noticed by Johannes Berg and confirmed by Sam Ravnborg. Signed-off-by: Randy Dunlap Cc: Sam Ravnborg Cc: Ingo Molnar Cc: Thomas Gleixner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/hrtimer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index e65dd0b47cd..f994bb8065e 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -1378,7 +1378,7 @@ sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp) /* * Functions related to boot-time initialization: */ -static void __devinit init_hrtimers_cpu(int cpu) +static void __cpuinit init_hrtimers_cpu(int cpu) { struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); int i; -- cgit v1.2.3 From 48ccf3dac341118992b70ca89c47728e8b1d300b Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Mon, 21 Jan 2008 17:18:25 -0800 Subject: timer: fix section mismatch The caller is __cpuinit. Also, this code block and its caller are inside #ifdef CONFIG_HOTPLUG_CPU blocks, so this code should reflect that config symbol's usage. WARNING: vmlinux.o(.text+0x4252f): Section mismatch: reference to .init.text: (between 'timer_cpu_notify' and 'msleep') Signed-off-by: Randy Dunlap Cc: Sam Ravnborg Cc: Ingo Molnar Cc: Thomas Gleixner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/timer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/timer.c b/kernel/timer.c index 26671f4db07..2a00c22203f 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -1289,7 +1289,7 @@ static void migrate_timer_list(tvec_base_t *new_base, struct list_head *head) } } -static void __devinit migrate_timers(int cpu) +static void __cpuinit migrate_timers(int cpu) { tvec_base_t *old_base; tvec_base_t *new_base; -- cgit v1.2.3 From c61935fd0e7f087a643827b4bf5ef646963c10fa Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 22 Jan 2008 11:24:58 +0100 Subject: sched: group scheduler, set uid share fix setting cpu share to 1 causes hangs, as reported in: http://bugzilla.kernel.org/show_bug.cgi?id=9779 as the default share is 1024, the values of 0 and 1 can indeed cause problems. Limit it to 2 or higher values. These values can only be set by the root user - but still it makes sense to protect against nonsensical values. Signed-off-by: Ingo Molnar --- kernel/sched.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 37cf07aa416..e76b11ca6df 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -7153,6 +7153,14 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares) { int i; + /* + * A weight of 0 or 1 can cause arithmetics problems. + * (The default weight is 1024 - so there's no practical + * limitation from this.) + */ + if (shares < 2) + shares = 2; + spin_lock(&tg->lock); if (tg->shares == shares) goto done; -- cgit v1.2.3 From 00e10776ff908a767b3d36a53d330db8fdc53a56 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 22 Jan 2008 03:31:39 -0800 Subject: rcu: fix section mismatch rcu_online_cpu() should be __cpuinit instead of __devinit. WARNING: vmlinux.o(.text+0x4b6d5): Section mismatch: reference to .init.text: (between 'rcu_cpu_notify' and 'wakeme_after_rcu') Signed-off-by: Randy Dunlap Cc: Sam Ravnborg Acked-by: Ingo Molnar Cc: Thomas Gleixner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/rcupdate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c index a66d4d1615f..f2c1a04e9b1 100644 --- a/kernel/rcupdate.c +++ b/kernel/rcupdate.c @@ -549,7 +549,7 @@ static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp, rdp->blimit = blimit; } -static void __devinit rcu_online_cpu(int cpu) +static void __cpuinit rcu_online_cpu(int cpu) { struct rcu_data *rdp = &per_cpu(rcu_data, cpu); struct rcu_data *bh_rdp = &per_cpu(rcu_bh_data, cpu); -- cgit v1.2.3 From d4acd722b7bb5f48b9fc3848e8c2a845b100d84f Mon Sep 17 00:00:00 2001 From: James Bottomley Date: Wed, 31 Oct 2007 09:38:04 -0500 Subject: [SCSI] sysfs: add filter function to groups This patch allows the various users of attribute_groups to selectively allow the appearance of group attributes. The primary consumer of this will be the transport classes in which we currently have elaborate attribute selection algorithms to do this same thing. Acked-by: Greg KH Signed-off-by: James Bottomley --- kernel/params.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/params.c b/kernel/params.c index 7686417ee00..dfef46474e5 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -472,7 +472,7 @@ param_sysfs_setup(struct module_kobject *mk, sizeof(mp->grp.attrs[0])); size[1] = (valid_attrs + 1) * sizeof(mp->grp.attrs[0]); - mp = kmalloc(size[0] + size[1], GFP_KERNEL); + mp = kzalloc(size[0] + size[1], GFP_KERNEL); if (!mp) return ERR_PTR(-ENOMEM); -- cgit v1.2.3 From fa28237cfcc5827553044cbd6ee52e33692b0faa Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Thu, 24 Jan 2008 08:35:13 +1100 Subject: [POWERPC] Provide a way to protect 4k subpages when using 64k pages Using 64k pages on 64-bit PowerPC systems makes life difficult for emulators that are trying to emulate an ISA, such as x86, which use a smaller page size, since the emulator can no longer use the MMU and the normal system calls for controlling page protections. Of course, the emulator can emulate the MMU by checking and possibly remapping the address for each memory access in software, but that is pretty slow. This provides a facility for such programs to control the access permissions on individual 4k sub-pages of 64k pages. The idea is that the emulator supplies an array of protection masks to apply to a specified range of virtual addresses. These masks are applied at the level where hardware PTEs are inserted into the hardware page table based on the Linux PTEs, so the Linux PTEs are not affected. Note that this new mechanism does not allow any access that would otherwise be prohibited; it can only prohibit accesses that would otherwise be allowed. This new facility is only available on 64-bit PowerPC and only when the kernel is configured for 64k pages. The masks are supplied using a new subpage_prot system call, which takes a starting virtual address and length, and a pointer to an array of protection masks in memory. The array has a 32-bit word per 64k page to be protected; each 32-bit word consists of 16 2-bit fields, for which 0 allows any access (that is otherwise allowed), 1 prevents write accesses, and 2 or 3 prevent any access. Implicit in this is that the regions of the address space that are protected are switched to use 4k hardware pages rather than 64k hardware pages (on machines with hardware 64k page support). In fact the whole process is switched to use 4k hardware pages when the subpage_prot system call is used, but this could be improved in future to switch only the affected segments. The subpage protection bits are stored in a 3 level tree akin to the page table tree. The top level of this tree is stored in a structure that is appended to the top level of the page table tree, i.e., the pgd array. Since it will often only be 32-bit addresses (below 4GB) that are protected, the pointers to the first four bottom level pages are also stored in this structure (each bottom level page contains the protection bits for 1GB of address space), so the protection bits for addresses below 4GB can be accessed with one fewer loads than those for higher addresses. Signed-off-by: Paul Mackerras --- kernel/sys_ni.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index 56cb009a4b3..beee5b3b68a 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -131,6 +131,7 @@ cond_syscall(sys32_sysctl); cond_syscall(ppc_rtas); cond_syscall(sys_spu_run); cond_syscall(sys_spu_create); +cond_syscall(sys_subpage_prot); /* mmu depending weak syscall entries */ cond_syscall(sys_mprotect); -- cgit v1.2.3 From fabe874a48de45b137f99b4ed3641e0413f465ce Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Thu, 24 Jan 2008 07:00:45 +0100 Subject: lockdep: fix kernel crash on module unload Michael Wu noticed in his lkml post at http://marc.info/?l=linux-kernel&m=119396182726091&w=2 that certain wireless drivers ended up having their name in module memory, which would then crash the kernel on module unload. The patch he proposed was a bit clumsy in that it increased the size of a lockdep entry significantly; the patch below tries another approach, it checks, on module teardown, if the name of a class is in module space and then zaps the class. This is very similar to what we already do with keys that are in module space. Signed-off-by: Arjan van de Ven Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra Signed-off-by: Linus Torvalds --- kernel/lockdep.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 4335f12a27c..e2c07ece367 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -2932,7 +2932,7 @@ static void zap_class(struct lock_class *class) } -static inline int within(void *addr, void *start, unsigned long size) +static inline int within(const void *addr, void *start, unsigned long size) { return addr >= start && addr < start + size; } @@ -2955,9 +2955,12 @@ void lockdep_free_key_range(void *start, unsigned long size) head = classhash_table + i; if (list_empty(head)) continue; - list_for_each_entry_safe(class, next, head, hash_entry) + list_for_each_entry_safe(class, next, head, hash_entry) { if (within(class->key, start, size)) zap_class(class); + else if (within(class->name, start, size)) + zap_class(class); + } } if (locked) -- cgit v1.2.3 From 3514faca19a6fdc209734431c509631ea92b094e Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Tue, 16 Oct 2007 10:11:44 -0600 Subject: kobject: remove struct kobj_type from struct kset We don't need a "default" ktype for a kset. We should set this explicitly every time for each kset. This change is needed so that we can make ksets dynamic, and cleans up one of the odd, undocumented assumption that the kset/kobject/ktype model has. This patch is based on a lot of help from Kay Sievers. Nasty bug in the block code was found by Dave Young Cc: Kay Sievers Cc: Dave Young Signed-off-by: Greg Kroah-Hartman --- kernel/ksysfs.c | 2 +- kernel/module.c | 2 +- kernel/params.c | 9 +++++---- kernel/power/main.c | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index 65daa5373ca..094e2bc101a 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -94,7 +94,7 @@ static struct bin_attribute notes_attr = { .read = ¬es_read, }; -decl_subsys(kernel, NULL, NULL); +decl_subsys(kernel, NULL); EXPORT_SYMBOL_GPL(kernel_subsys); static struct attribute * kernel_attrs[] = { diff --git a/kernel/module.c b/kernel/module.c index c2e3e2e9880..68df79738b3 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1223,7 +1223,7 @@ int mod_sysfs_init(struct module *mod) err = kobject_set_name(&mod->mkobj.kobj, "%s", mod->name); if (err) goto out; - kobj_set_kset_s(&mod->mkobj, module_subsys); + mod->mkobj.kobj.kset = &module_subsys; mod->mkobj.mod = mod; kobject_init(&mod->mkobj.kobj); diff --git a/kernel/params.c b/kernel/params.c index 7686417ee00..9f051824097 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -30,6 +30,8 @@ #define DEBUGP(fmt, a...) #endif +static struct kobj_type module_ktype; + static inline char dash2underscore(char c) { if (c == '-') @@ -560,7 +562,8 @@ static void __init kernel_param_sysfs_setup(const char *name, BUG_ON(!mk); mk->mod = THIS_MODULE; - kobj_set_kset_s(mk, module_subsys); + mk->kobj.kset = &module_subsys; + mk->kobj.ktype = &module_ktype; kobject_set_name(&mk->kobj, name); kobject_init(&mk->kobj); ret = kobject_add(&mk->kobj); @@ -679,8 +682,6 @@ static struct sysfs_ops module_sysfs_ops = { .store = module_attr_store, }; -static struct kobj_type module_ktype; - static int uevent_filter(struct kset *kset, struct kobject *kobj) { struct kobj_type *ktype = get_ktype(kobj); @@ -694,7 +695,7 @@ static struct kset_uevent_ops module_uevent_ops = { .filter = uevent_filter, }; -decl_subsys(module, &module_ktype, &module_uevent_ops); +decl_subsys(module, &module_uevent_ops); int module_sysfs_initialized; static void module_release(struct kobject *kobj) diff --git a/kernel/power/main.c b/kernel/power/main.c index f71c9504a5c..1ef31c91ce0 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -276,7 +276,7 @@ EXPORT_SYMBOL(pm_suspend); #endif /* CONFIG_SUSPEND */ -decl_subsys(power,NULL,NULL); +decl_subsys(power, NULL); /** -- cgit v1.2.3 From 4ff6abff832fbc6cb1d769f6106c841bc2b09f63 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 5 Nov 2007 22:24:43 -0800 Subject: kobject: get rid of kobject_add_dir kobject_create_and_add is the same as kobject_add_dir, so drop kobject_add_dir. Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/module.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index 68df79738b3..55142775c58 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1122,7 +1122,7 @@ static void add_notes_attrs(struct module *mod, unsigned int nsect, ++loaded; } - notes_attrs->dir = kobject_add_dir(&mod->mkobj.kobj, "notes"); + notes_attrs->dir = kobject_create_and_add("notes", &mod->mkobj.kobj); if (!notes_attrs->dir) goto out; @@ -1243,7 +1243,7 @@ int mod_sysfs_setup(struct module *mod, if (err) goto out; - mod->holders_dir = kobject_add_dir(&mod->mkobj.kobj, "holders"); + mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj); if (!mod->holders_dir) { err = -ENOMEM; goto out_unreg; @@ -2521,7 +2521,7 @@ static void module_create_drivers_dir(struct module_kobject *mk) if (!mk || mk->drivers_dir) return; - mk->drivers_dir = kobject_add_dir(&mk->kobj, "drivers"); + mk->drivers_dir = kobject_create_and_add("drivers", &mk->kobj); } void module_add_driver(struct module *mod, struct device_driver *drv) -- cgit v1.2.3 From bd35b93d8049ab47b5bfaf6b10ba39badf21d1c3 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 29 Oct 2007 20:13:17 +0100 Subject: kset: convert kernel_subsys to use kset_create Dynamically create the kset instead of declaring it statically. We also rename kernel_subsys to kernel_kset to catch all users of this symbol with a build error instead of an easy-to-ignore build warning. Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/ksysfs.c | 42 ++++++++++++++++++++++++++++++------------ kernel/user.c | 4 ++-- 2 files changed, 32 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index 094e2bc101a..cf02d4ba9ad 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -94,8 +94,8 @@ static struct bin_attribute notes_attr = { .read = ¬es_read, }; -decl_subsys(kernel, NULL); -EXPORT_SYMBOL_GPL(kernel_subsys); +struct kset *kernel_kset; +EXPORT_SYMBOL_GPL(kernel_kset); static struct attribute * kernel_attrs[] = { #if defined(CONFIG_HOTPLUG) && defined(CONFIG_NET) @@ -116,24 +116,42 @@ static struct attribute_group kernel_attr_group = { static int __init ksysfs_init(void) { - int error = subsystem_register(&kernel_subsys); - if (!error) - error = sysfs_create_group(&kernel_subsys.kobj, - &kernel_attr_group); + int error; - if (!error && notes_size > 0) { + kernel_kset = kset_create_and_add("kernel", NULL, NULL); + if (!kernel_kset) { + error = -ENOMEM; + goto exit; + } + error = sysfs_create_group(&kernel_kset->kobj, &kernel_attr_group); + if (error) + goto kset_exit; + + if (notes_size > 0) { notes_attr.size = notes_size; - error = sysfs_create_bin_file(&kernel_subsys.kobj, - ¬es_attr); + error = sysfs_create_bin_file(&kernel_kset->kobj, ¬es_attr); + if (error) + goto group_exit; } /* * Create "/sys/kernel/uids" directory and corresponding root user's * directory under it. */ - if (!error) - error = uids_kobject_init(); - + error = uids_kobject_init(); + if (error) + goto notes_exit; + + return 0; + +notes_exit: + if (notes_size > 0) + sysfs_remove_bin_file(&kernel_kset->kobj, ¬es_attr); +group_exit: + sysfs_remove_group(&kernel_kset->kobj, &kernel_attr_group); +kset_exit: + kset_unregister(kernel_kset); +exit: return error; } diff --git a/kernel/user.c b/kernel/user.c index 8320a87f3e5..80f1116b8fc 100644 --- a/kernel/user.c +++ b/kernel/user.c @@ -198,8 +198,8 @@ int __init uids_kobject_init(void) int error; /* create under /sys/kernel dir */ - uids_kobject.parent = &kernel_subsys.kobj; - uids_kobject.kset = &kernel_subsys; + uids_kobject.parent = &kernel_kset->kobj; + uids_kobject.kset = kernel_kset; kobject_set_name(&uids_kobject, "uids"); kobject_init(&uids_kobject); -- cgit v1.2.3 From 7405c1e15edfe43b137bfbc5882f1af34d6d414d Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 1 Nov 2007 10:39:50 -0700 Subject: kset: convert /sys/module to use kset_create Dynamically create the kset instead of declaring it statically. We also rename module_subsys to module_kset to catch all users of the variable. Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/module.c | 7 +++---- kernel/params.c | 29 +++++++++-------------------- 2 files changed, 12 insertions(+), 24 deletions(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index 55142775c58..d03fcd9d652 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -47,8 +47,6 @@ #include #include -extern int module_sysfs_initialized; - #if 0 #define DEBUGP printk #else @@ -1223,7 +1221,8 @@ int mod_sysfs_init(struct module *mod) err = kobject_set_name(&mod->mkobj.kobj, "%s", mod->name); if (err) goto out; - mod->mkobj.kobj.kset = &module_subsys; + mod->mkobj.kobj.kset = module_kset; + mod->mkobj.kobj.ktype = &module_ktype; mod->mkobj.mod = mod; kobject_init(&mod->mkobj.kobj); @@ -2539,7 +2538,7 @@ void module_add_driver(struct module *mod, struct device_driver *drv) struct kobject *mkobj; /* Lookup built-in module entry in /sys/modules */ - mkobj = kset_find_obj(&module_subsys, drv->mod_name); + mkobj = kset_find_obj(module_kset, drv->mod_name); if (mkobj) { mk = container_of(mkobj, struct module_kobject, kobj); /* remember our module structure */ diff --git a/kernel/params.c b/kernel/params.c index 9f051824097..97e09231215 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -30,8 +30,6 @@ #define DEBUGP(fmt, a...) #endif -static struct kobj_type module_ktype; - static inline char dash2underscore(char c) { if (c == '-') @@ -562,7 +560,7 @@ static void __init kernel_param_sysfs_setup(const char *name, BUG_ON(!mk); mk->mod = THIS_MODULE; - mk->kobj.kset = &module_subsys; + mk->kobj.kset = module_kset; mk->kobj.ktype = &module_ktype; kobject_set_name(&mk->kobj, name); kobject_init(&mk->kobj); @@ -695,7 +693,7 @@ static struct kset_uevent_ops module_uevent_ops = { .filter = uevent_filter, }; -decl_subsys(module, &module_uevent_ops); +struct kset *module_kset; int module_sysfs_initialized; static void module_release(struct kobject *kobj) @@ -707,7 +705,7 @@ static void module_release(struct kobject *kobj) */ } -static struct kobj_type module_ktype = { +struct kobj_type module_ktype = { .sysfs_ops = &module_sysfs_ops, .release = module_release, }; @@ -717,13 +715,11 @@ static struct kobj_type module_ktype = { */ static int __init param_sysfs_init(void) { - int ret; - - ret = subsystem_register(&module_subsys); - if (ret < 0) { - printk(KERN_WARNING "%s (%d): subsystem_register error: %d\n", - __FILE__, __LINE__, ret); - return ret; + module_kset = kset_create_and_add("module", &module_uevent_ops, NULL); + if (!module_kset) { + printk(KERN_WARNING "%s (%d): error creating kset\n", + __FILE__, __LINE__); + return -ENOMEM; } module_sysfs_initialized = 1; @@ -733,14 +729,7 @@ static int __init param_sysfs_init(void) } subsys_initcall(param_sysfs_init); -#else -#if 0 -static struct sysfs_ops module_sysfs_ops = { - .show = NULL, - .store = NULL, -}; -#endif -#endif +#endif /* CONFIG_SYSFS */ EXPORT_SYMBOL(param_set_byte); EXPORT_SYMBOL(param_get_byte); -- cgit v1.2.3 From 039a5dcd2fc45188a2d522df630db4f7ef903a0f Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 1 Nov 2007 10:39:50 -0700 Subject: kset: convert /sys/power to use kset_create Dynamically create the kset instead of declaring it statically. We also rename power_subsys to power_kset to catch all users of the variable and we properly export it so that people don't have to guess that it really is present in the system. The pseries code is wierd, why is it createing /sys/power if CONFIG_PM is disabled? Oh well, stupid big boxes ignoring config options... Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/power/disk.c | 2 +- kernel/power/main.c | 11 +++++------ kernel/power/power.h | 2 -- 3 files changed, 6 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/power/disk.c b/kernel/power/disk.c index 05b64790fe8..c3f0e61365d 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c @@ -708,7 +708,7 @@ static struct attribute_group attr_group = { static int __init pm_disk_init(void) { - return sysfs_create_group(&power_subsys.kobj, &attr_group); + return sysfs_create_group(&power_kset->kobj, &attr_group); } core_initcall(pm_disk_init); diff --git a/kernel/power/main.c b/kernel/power/main.c index 1ef31c91ce0..dce2d76d66d 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -276,8 +276,7 @@ EXPORT_SYMBOL(pm_suspend); #endif /* CONFIG_SUSPEND */ -decl_subsys(power, NULL); - +struct kset *power_kset; /** * state - control system power state. @@ -386,10 +385,10 @@ static struct attribute_group attr_group = { static int __init pm_init(void) { - int error = subsystem_register(&power_subsys); - if (!error) - error = sysfs_create_group(&power_subsys.kobj,&attr_group); - return error; + power_kset = kset_create_and_add("power", NULL, NULL); + if (!power_kset) + return -ENOMEM; + return sysfs_create_group(&power_kset->kobj, &attr_group); } core_initcall(pm_init); diff --git a/kernel/power/power.h b/kernel/power/power.h index 195dc461176..1083e6b188a 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -63,8 +63,6 @@ static struct subsys_attribute _name##_attr = { \ .store = _name##_store, \ } -extern struct kset power_subsys; - /* Preferred image size in bytes (default 500 MB) */ extern unsigned long image_size; extern int in_suspend; -- cgit v1.2.3 From 386f275f5d097758f867bc99ddeaeb7a03b6b190 Mon Sep 17 00:00:00 2001 From: Kay Sievers Date: Fri, 2 Nov 2007 13:47:53 +0100 Subject: Driver Core: switch all dynamic ksets to kobj_sysfs_ops Switch all dynamically created ksets, that export simple attributes, to kobj_attribute from subsys_attribute. Struct subsys_attribute will be removed. Signed-off-by: Kay Sievers Cc: Mike Halcrow Cc: Phillip Hellewell Signed-off-by: Greg Kroah-Hartman --- kernel/ksysfs.c | 35 +++++++++++++++++++++-------------- kernel/power/disk.c | 18 ++++++++++++------ kernel/power/main.c | 12 ++++++++---- kernel/power/power.h | 2 +- 4 files changed, 42 insertions(+), 25 deletions(-) (limited to 'kernel') diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index cf02d4ba9ad..dd0f9e7f341 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -17,30 +17,34 @@ #include #define KERNEL_ATTR_RO(_name) \ -static struct subsys_attribute _name##_attr = __ATTR_RO(_name) +static struct kobj_attribute _name##_attr = __ATTR_RO(_name) #define KERNEL_ATTR_RW(_name) \ -static struct subsys_attribute _name##_attr = \ +static struct kobj_attribute _name##_attr = \ __ATTR(_name, 0644, _name##_show, _name##_store) #if defined(CONFIG_HOTPLUG) && defined(CONFIG_NET) /* current uevent sequence number */ -static ssize_t uevent_seqnum_show(struct kset *kset, char *page) +static ssize_t uevent_seqnum_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) { - return sprintf(page, "%llu\n", (unsigned long long)uevent_seqnum); + return sprintf(buf, "%llu\n", (unsigned long long)uevent_seqnum); } KERNEL_ATTR_RO(uevent_seqnum); /* uevent helper program, used during early boo */ -static ssize_t uevent_helper_show(struct kset *kset, char *page) +static ssize_t uevent_helper_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) { - return sprintf(page, "%s\n", uevent_helper); + return sprintf(buf, "%s\n", uevent_helper); } -static ssize_t uevent_helper_store(struct kset *kset, const char *page, size_t count) +static ssize_t uevent_helper_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) { if (count+1 > UEVENT_HELPER_PATH_LEN) return -ENOENT; - memcpy(uevent_helper, page, count); + memcpy(uevent_helper, buf, count); uevent_helper[count] = '\0'; if (count && uevent_helper[count-1] == '\n') uevent_helper[count-1] = '\0'; @@ -50,21 +54,24 @@ KERNEL_ATTR_RW(uevent_helper); #endif #ifdef CONFIG_KEXEC -static ssize_t kexec_loaded_show(struct kset *kset, char *page) +static ssize_t kexec_loaded_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) { - return sprintf(page, "%d\n", !!kexec_image); + return sprintf(buf, "%d\n", !!kexec_image); } KERNEL_ATTR_RO(kexec_loaded); -static ssize_t kexec_crash_loaded_show(struct kset *kset, char *page) +static ssize_t kexec_crash_loaded_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) { - return sprintf(page, "%d\n", !!kexec_crash_image); + return sprintf(buf, "%d\n", !!kexec_crash_image); } KERNEL_ATTR_RO(kexec_crash_loaded); -static ssize_t vmcoreinfo_show(struct kset *kset, char *page) +static ssize_t vmcoreinfo_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) { - return sprintf(page, "%lx %x\n", + return sprintf(buf, "%lx %x\n", paddr_vmcoreinfo_note(), (unsigned int)vmcoreinfo_max_size); } diff --git a/kernel/power/disk.c b/kernel/power/disk.c index c3f0e61365d..ef5aa2ca0ab 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c @@ -567,7 +567,8 @@ static const char * const hibernation_modes[] = { * supports it (as determined by having hibernation_ops). */ -static ssize_t disk_show(struct kset *kset, char *buf) +static ssize_t disk_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) { int i; char *start = buf; @@ -597,7 +598,8 @@ static ssize_t disk_show(struct kset *kset, char *buf) } -static ssize_t disk_store(struct kset *kset, const char *buf, size_t n) +static ssize_t disk_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t n) { int error = 0; int i; @@ -642,13 +644,15 @@ static ssize_t disk_store(struct kset *kset, const char *buf, size_t n) power_attr(disk); -static ssize_t resume_show(struct kset *kset, char *buf) +static ssize_t resume_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) { return sprintf(buf,"%d:%d\n", MAJOR(swsusp_resume_device), MINOR(swsusp_resume_device)); } -static ssize_t resume_store(struct kset *kset, const char *buf, size_t n) +static ssize_t resume_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t n) { unsigned int maj, min; dev_t res; @@ -674,12 +678,14 @@ static ssize_t resume_store(struct kset *kset, const char *buf, size_t n) power_attr(resume); -static ssize_t image_size_show(struct kset *kset, char *buf) +static ssize_t image_size_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) { return sprintf(buf, "%lu\n", image_size); } -static ssize_t image_size_store(struct kset *kset, const char *buf, size_t n) +static ssize_t image_size_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t n) { unsigned long size; diff --git a/kernel/power/main.c b/kernel/power/main.c index dce2d76d66d..b8139493b85 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -289,7 +289,8 @@ struct kset *power_kset; * proper enumerated value, and initiates a suspend transition. */ -static ssize_t state_show(struct kset *kset, char *buf) +static ssize_t state_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) { char *s = buf; #ifdef CONFIG_SUSPEND @@ -310,7 +311,8 @@ static ssize_t state_show(struct kset *kset, char *buf) return (s - buf); } -static ssize_t state_store(struct kset *kset, const char *buf, size_t n) +static ssize_t state_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t n) { #ifdef CONFIG_SUSPEND suspend_state_t state = PM_SUSPEND_STANDBY; @@ -347,13 +349,15 @@ power_attr(state); #ifdef CONFIG_PM_TRACE int pm_trace_enabled; -static ssize_t pm_trace_show(struct kset *kset, char *buf) +static ssize_t pm_trace_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) { return sprintf(buf, "%d\n", pm_trace_enabled); } static ssize_t -pm_trace_store(struct kset *kset, const char *buf, size_t n) +pm_trace_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t n) { int val; diff --git a/kernel/power/power.h b/kernel/power/power.h index 1083e6b188a..2093c3a9a99 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -54,7 +54,7 @@ extern int pfn_is_nosave(unsigned long); extern struct mutex pm_mutex; #define power_attr(_name) \ -static struct subsys_attribute _name##_attr = { \ +static struct kobj_attribute _name##_attr = { \ .attr = { \ .name = __stringify(_name), \ .mode = 0644, \ -- cgit v1.2.3 From eb41d9465cdafee45e0cb30f3b7338646221908e Mon Sep 17 00:00:00 2001 From: Kay Sievers Date: Fri, 2 Nov 2007 13:47:53 +0100 Subject: fix struct user_info export's sysfs interaction Clean up the use of ksets and kobjects. Kobjects are instances of objects (like struct user_info), ksets are collections of objects of a similar type (like the uids directory containing the user_info directories). So, use kobjects for the user_info directories, and a kset for the "uids" directory. On object cleanup, the final kobject_put() was missing. Cc: Dhaval Giani Cc: Srivatsa Vaddagiri Signed-off-by: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/ksysfs.c | 7 ++-- kernel/user.c | 104 +++++++++++++++++++++++++++----------------------------- 2 files changed, 53 insertions(+), 58 deletions(-) (limited to 'kernel') diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index dd0f9e7f341..45e64658560 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -141,11 +141,8 @@ static int __init ksysfs_init(void) goto group_exit; } - /* - * Create "/sys/kernel/uids" directory and corresponding root user's - * directory under it. - */ - error = uids_kobject_init(); + /* create the /sys/kernel/uids/ directory */ + error = uids_sysfs_init(); if (error) goto notes_exit; diff --git a/kernel/user.c b/kernel/user.c index 80f1116b8fc..5a106f3fdf0 100644 --- a/kernel/user.c +++ b/kernel/user.c @@ -115,7 +115,7 @@ static void sched_switch_user(struct task_struct *p) { } #if defined(CONFIG_FAIR_USER_SCHED) && defined(CONFIG_SYSFS) -static struct kobject uids_kobject; /* represents /sys/kernel/uids directory */ +static struct kset *uids_kset; /* represents the /sys/kernel/uids/ directory */ static DEFINE_MUTEX(uids_mutex); static inline void uids_mutex_lock(void) @@ -128,86 +128,84 @@ static inline void uids_mutex_unlock(void) mutex_unlock(&uids_mutex); } -/* return cpu shares held by the user */ -static ssize_t cpu_shares_show(struct kset *kset, char *buffer) +/* uid directory attributes */ +static ssize_t cpu_shares_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) { - struct user_struct *up = container_of(kset, struct user_struct, kset); + struct user_struct *up = container_of(kobj, struct user_struct, kobj); - return sprintf(buffer, "%lu\n", sched_group_shares(up->tg)); + return sprintf(buf, "%lu\n", sched_group_shares(up->tg)); } -/* modify cpu shares held by the user */ -static ssize_t cpu_shares_store(struct kset *kset, const char *buffer, - size_t size) +static ssize_t cpu_shares_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t size) { - struct user_struct *up = container_of(kset, struct user_struct, kset); + struct user_struct *up = container_of(kobj, struct user_struct, kobj); unsigned long shares; int rc; - sscanf(buffer, "%lu", &shares); + sscanf(buf, "%lu", &shares); rc = sched_group_set_shares(up->tg, shares); return (rc ? rc : size); } -static void user_attr_init(struct subsys_attribute *sa, char *name, int mode) +static struct kobj_attribute cpu_share_attr = + __ATTR(cpu_share, 0644, cpu_shares_show, cpu_shares_store); + +/* default attributes per uid directory */ +static struct attribute *uids_attributes[] = { + &cpu_share_attr.attr, + NULL +}; + +/* the lifetime of user_struct is not managed by the core (now) */ +static void uids_release(struct kobject *kobj) { - sa->attr.name = name; - sa->attr.mode = mode; - sa->show = cpu_shares_show; - sa->store = cpu_shares_store; + return; } -/* Create "/sys/kernel/uids/" directory and - * "/sys/kernel/uids//cpu_share" file for this user. - */ -static int user_kobject_create(struct user_struct *up) +static struct kobj_type uids_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .default_attrs = uids_attributes, + .release = uids_release, +}; + +/* create /sys/kernel/uids//cpu_share file for this user */ +static int uids_user_create(struct user_struct *up) { - struct kset *kset = &up->kset; - struct kobject *kobj = &kset->kobj; + struct kobject *kobj = &up->kobj; int error; - memset(kset, 0, sizeof(struct kset)); - kobj->parent = &uids_kobject; /* create under /sys/kernel/uids dir */ - kobject_set_name(kobj, "%d", up->uid); - kset_init(kset); - user_attr_init(&up->user_attr, "cpu_share", 0644); - + memset(kobj, 0, sizeof(struct kobject)); + kobj->ktype = &uids_ktype; + kobj->kset = uids_kset; + kobject_init(kobj); + kobject_set_name(&up->kobj, "%d", up->uid); error = kobject_add(kobj); if (error) goto done; - error = sysfs_create_file(kobj, &up->user_attr.attr); - if (error) - kobject_del(kobj); - kobject_uevent(kobj, KOBJ_ADD); - done: return error; } -/* create these in sysfs filesystem: +/* create these entries in sysfs: * "/sys/kernel/uids" directory * "/sys/kernel/uids/0" directory (for root user) * "/sys/kernel/uids/0/cpu_share" file (for root user) */ -int __init uids_kobject_init(void) +int __init uids_sysfs_init(void) { - int error; + uids_kset = kset_create_and_add("uids", NULL, &kernel_kset->kobj); + if (!uids_kset) + return -ENOMEM; - /* create under /sys/kernel dir */ - uids_kobject.parent = &kernel_kset->kobj; - uids_kobject.kset = kernel_kset; - kobject_set_name(&uids_kobject, "uids"); - kobject_init(&uids_kobject); - - error = kobject_add(&uids_kobject); - if (!error) - error = user_kobject_create(&root_user); - - return error; + return uids_user_create(&root_user); } /* work function to remove sysfs directory for a user and free up @@ -216,7 +214,6 @@ int __init uids_kobject_init(void) static void remove_user_sysfs_dir(struct work_struct *w) { struct user_struct *up = container_of(w, struct user_struct, work); - struct kobject *kobj = &up->kset.kobj; unsigned long flags; int remove_user = 0; @@ -238,9 +235,9 @@ static void remove_user_sysfs_dir(struct work_struct *w) if (!remove_user) goto done; - sysfs_remove_file(kobj, &up->user_attr.attr); - kobject_uevent(kobj, KOBJ_REMOVE); - kobject_del(kobj); + kobject_uevent(&up->kobj, KOBJ_REMOVE); + kobject_del(&up->kobj); + kobject_put(&up->kobj); sched_destroy_user(up); key_put(up->uid_keyring); @@ -267,7 +264,8 @@ static inline void free_user(struct user_struct *up, unsigned long flags) #else /* CONFIG_FAIR_USER_SCHED && CONFIG_SYSFS */ -static inline int user_kobject_create(struct user_struct *up) { return 0; } +int uids_sysfs_init(void) { return 0; } +static inline int uids_user_create(struct user_struct *up) { return 0; } static inline void uids_mutex_lock(void) { } static inline void uids_mutex_unlock(void) { } @@ -324,7 +322,7 @@ struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid) struct hlist_head *hashent = uidhashentry(ns, uid); struct user_struct *up; - /* Make uid_hash_find() + user_kobject_create() + uid_hash_insert() + /* Make uid_hash_find() + uids_user_create() + uid_hash_insert() * atomic. */ uids_mutex_lock(); @@ -370,7 +368,7 @@ struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid) return NULL; } - if (user_kobject_create(new)) { + if (uids_user_create(new)) { sched_destroy_user(new); key_put(new->uid_keyring); key_put(new->session_keyring); -- cgit v1.2.3 From 0ff21e46630abce11fdaaffabd72bbd4eed5ac2c Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Tue, 6 Nov 2007 10:36:58 -0800 Subject: kobject: convert kernel_kset to be a kobject kernel_kset does not need to be a kset, but a much simpler kobject now that we have kobj_attributes. We also rename kernel_kset to kernel_kobj to catch all users of this symbol with a build error instead of an easy-to-ignore build warning. Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/ksysfs.c | 18 +++++++++--------- kernel/user.c | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index 45e64658560..1081aff5fb9 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -101,8 +101,8 @@ static struct bin_attribute notes_attr = { .read = ¬es_read, }; -struct kset *kernel_kset; -EXPORT_SYMBOL_GPL(kernel_kset); +struct kobject *kernel_kobj; +EXPORT_SYMBOL_GPL(kernel_kobj); static struct attribute * kernel_attrs[] = { #if defined(CONFIG_HOTPLUG) && defined(CONFIG_NET) @@ -125,18 +125,18 @@ static int __init ksysfs_init(void) { int error; - kernel_kset = kset_create_and_add("kernel", NULL, NULL); - if (!kernel_kset) { + kernel_kobj = kobject_create_and_add("kernel", NULL); + if (!kernel_kobj) { error = -ENOMEM; goto exit; } - error = sysfs_create_group(&kernel_kset->kobj, &kernel_attr_group); + error = sysfs_create_group(kernel_kobj, &kernel_attr_group); if (error) goto kset_exit; if (notes_size > 0) { notes_attr.size = notes_size; - error = sysfs_create_bin_file(&kernel_kset->kobj, ¬es_attr); + error = sysfs_create_bin_file(kernel_kobj, ¬es_attr); if (error) goto group_exit; } @@ -150,11 +150,11 @@ static int __init ksysfs_init(void) notes_exit: if (notes_size > 0) - sysfs_remove_bin_file(&kernel_kset->kobj, ¬es_attr); + sysfs_remove_bin_file(kernel_kobj, ¬es_attr); group_exit: - sysfs_remove_group(&kernel_kset->kobj, &kernel_attr_group); + sysfs_remove_group(kernel_kobj, &kernel_attr_group); kset_exit: - kset_unregister(kernel_kset); + kobject_unregister(kernel_kobj); exit: return error; } diff --git a/kernel/user.c b/kernel/user.c index 5a106f3fdf0..7f17e6e8fd6 100644 --- a/kernel/user.c +++ b/kernel/user.c @@ -201,7 +201,7 @@ done: */ int __init uids_sysfs_init(void) { - uids_kset = kset_create_and_add("uids", NULL, &kernel_kset->kobj); + uids_kset = kset_create_and_add("uids", NULL, kernel_kobj); if (!uids_kset) return -ENOMEM; -- cgit v1.2.3 From d76e15fb20eeb7632ef38876a884fe3508b2c01d Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Tue, 27 Nov 2007 11:28:26 -0800 Subject: driver core: make /sys/power a kobject /sys/power should not be a kset, that's overkill. This patch renames it to power_kset and fixes up all usages of it in the tree. Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/power/disk.c | 2 +- kernel/power/main.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/power/disk.c b/kernel/power/disk.c index ef5aa2ca0ab..b138b431e27 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c @@ -714,7 +714,7 @@ static struct attribute_group attr_group = { static int __init pm_disk_init(void) { - return sysfs_create_group(&power_kset->kobj, &attr_group); + return sysfs_create_group(power_kobj, &attr_group); } core_initcall(pm_disk_init); diff --git a/kernel/power/main.c b/kernel/power/main.c index b8139493b85..efc08360e62 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -276,7 +276,7 @@ EXPORT_SYMBOL(pm_suspend); #endif /* CONFIG_SUSPEND */ -struct kset *power_kset; +struct kobject *power_kobj; /** * state - control system power state. @@ -389,10 +389,10 @@ static struct attribute_group attr_group = { static int __init pm_init(void) { - power_kset = kset_create_and_add("power", NULL, NULL); - if (!power_kset) + power_kobj = kobject_create_and_add("power", NULL); + if (!power_kobj) return -ENOMEM; - return sysfs_create_group(&power_kset->kobj, &attr_group); + return sysfs_create_group(power_kobj, &attr_group); } core_initcall(pm_init); -- cgit v1.2.3 From e43b9192c59402685bd1f809068dd13aa5931570 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 17 Dec 2007 23:05:35 -0700 Subject: Kobject: convert kernel/params.c to use kobject_init/add_ng() This converts the code to use the new kobject functions, cleaning up the logic in doing so. Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/params.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/params.c b/kernel/params.c index 97e09231215..1078b148ca8 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -561,11 +561,9 @@ static void __init kernel_param_sysfs_setup(const char *name, mk->mod = THIS_MODULE; mk->kobj.kset = module_kset; - mk->kobj.ktype = &module_ktype; - kobject_set_name(&mk->kobj, name); - kobject_init(&mk->kobj); - ret = kobject_add(&mk->kobj); + ret = kobject_init_and_add(&mk->kobj, &module_ktype, NULL, "%s", name); if (ret) { + kobject_put(&mk->kobj); printk(KERN_ERR "Module '%s' failed to be added to sysfs, " "error number %d\n", name, ret); printk(KERN_ERR "The system will be unstable now.\n"); -- cgit v1.2.3 From cf15126b3d4511e06e5299781ab74922590900be Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 17 Dec 2007 23:05:35 -0700 Subject: Kobject: convert kernel/user.c to use kobject_init/add_ng() This converts the code to use the new kobject functions, cleaning up the logic in doing so. Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/user.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/user.c b/kernel/user.c index 7f17e6e8fd6..ab4fd706993 100644 --- a/kernel/user.c +++ b/kernel/user.c @@ -181,13 +181,12 @@ static int uids_user_create(struct user_struct *up) int error; memset(kobj, 0, sizeof(struct kobject)); - kobj->ktype = &uids_ktype; kobj->kset = uids_kset; - kobject_init(kobj); - kobject_set_name(&up->kobj, "%d", up->uid); - error = kobject_add(kobj); - if (error) + error = kobject_init_and_add(kobj, &uids_ktype, NULL, "%d", up->uid); + if (error) { + kobject_put(kobj); goto done; + } kobject_uevent(kobj, KOBJ_ADD); done: -- cgit v1.2.3 From c63469a3985a9771c18a916b8d42845d044ea0b1 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Wed, 28 Nov 2007 12:23:18 -0800 Subject: Driver core: move the driver specific module code into the driver core The module driver specific code should belong in the driver core, not in the kernel/ directory. So move this code. This is done in preparation for some struct device_driver rework that should be confined to the driver core code only. This also lets us keep from exporting these functions, as no external code should ever be calling it. Thanks to Andrew Morton for the !CONFIG_MODULES fix. Signed-off-by: Greg Kroah-Hartman --- kernel/module.c | 87 --------------------------------------------------------- 1 file changed, 87 deletions(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index d03fcd9d652..dc4d3f5ce82 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -2501,93 +2501,6 @@ void print_modules(void) printk("\n"); } -#ifdef CONFIG_SYSFS -static char *make_driver_name(struct device_driver *drv) -{ - char *driver_name; - - driver_name = kmalloc(strlen(drv->name) + strlen(drv->bus->name) + 2, - GFP_KERNEL); - if (!driver_name) - return NULL; - - sprintf(driver_name, "%s:%s", drv->bus->name, drv->name); - return driver_name; -} - -static void module_create_drivers_dir(struct module_kobject *mk) -{ - if (!mk || mk->drivers_dir) - return; - - mk->drivers_dir = kobject_create_and_add("drivers", &mk->kobj); -} - -void module_add_driver(struct module *mod, struct device_driver *drv) -{ - char *driver_name; - int no_warn; - struct module_kobject *mk = NULL; - - if (!drv) - return; - - if (mod) - mk = &mod->mkobj; - else if (drv->mod_name) { - struct kobject *mkobj; - - /* Lookup built-in module entry in /sys/modules */ - mkobj = kset_find_obj(module_kset, drv->mod_name); - if (mkobj) { - mk = container_of(mkobj, struct module_kobject, kobj); - /* remember our module structure */ - drv->mkobj = mk; - /* kset_find_obj took a reference */ - kobject_put(mkobj); - } - } - - if (!mk) - return; - - /* Don't check return codes; these calls are idempotent */ - no_warn = sysfs_create_link(&drv->kobj, &mk->kobj, "module"); - driver_name = make_driver_name(drv); - if (driver_name) { - module_create_drivers_dir(mk); - no_warn = sysfs_create_link(mk->drivers_dir, &drv->kobj, - driver_name); - kfree(driver_name); - } -} -EXPORT_SYMBOL(module_add_driver); - -void module_remove_driver(struct device_driver *drv) -{ - struct module_kobject *mk = NULL; - char *driver_name; - - if (!drv) - return; - - sysfs_remove_link(&drv->kobj, "module"); - - if (drv->owner) - mk = &drv->owner->mkobj; - else if (drv->mkobj) - mk = drv->mkobj; - if (mk && mk->drivers_dir) { - driver_name = make_driver_name(drv); - if (driver_name) { - sysfs_remove_link(mk->drivers_dir, driver_name); - kfree(driver_name); - } - } -} -EXPORT_SYMBOL(module_remove_driver); -#endif - #ifdef CONFIG_MODVERSIONS /* Generate the signature for struct module here, too, for modversions. */ void struct_module(struct module *mod) { return; } -- cgit v1.2.3 From 97c146ef075dc40ae34407c03d3860fc3850b8e8 Mon Sep 17 00:00:00 2001 From: Kay Sievers Date: Thu, 29 Nov 2007 23:46:11 +0100 Subject: sysfs: fix /sys/module/*/holders after sysfs logic change Sysfs symlinks now require fully registered kobjects as a target, otherwise the call to create a symlink will fail. Here we register the kobject before we request the symlink in the holders directory. Signed-off-by: Kay Sievers Cc: Tejun Heo Signed-off-by: Greg Kroah-Hartman --- kernel/module.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index dc4d3f5ce82..0ae811785c5 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1227,6 +1227,8 @@ int mod_sysfs_init(struct module *mod) kobject_init(&mod->mkobj.kobj); + /* delay uevent until full sysfs population */ + err = kobject_add(&mod->mkobj.kobj); out: return err; } @@ -1237,11 +1239,6 @@ int mod_sysfs_setup(struct module *mod, { int err; - /* delay uevent until full sysfs population */ - err = kobject_add(&mod->mkobj.kobj); - if (err) - goto out; - mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj); if (!mod->holders_dir) { err = -ENOMEM; @@ -1266,7 +1263,6 @@ out_unreg_holders: out_unreg: kobject_del(&mod->mkobj.kobj); kobject_put(&mod->mkobj.kobj); -out: return err; } #endif @@ -1883,10 +1879,10 @@ static struct module *load_module(void __user *umod, /* Now we've moved module, initialize linked lists, etc. */ module_unload_init(mod); - /* Initialize kobject, so we can reference it. */ + /* add kobject, so we can reference it. */ err = mod_sysfs_init(mod); if (err) - goto cleanup; + goto free_unload; /* Set up license info based on the info section */ set_license(mod, get_modinfo(sechdrs, infoindex, "license")); @@ -2056,6 +2052,9 @@ static struct module *load_module(void __user *umod, arch_cleanup: module_arch_cleanup(mod); cleanup: + kobject_del(&mod->mkobj.kobj); + kobject_put(&mod->mkobj.kobj); + free_unload: module_unload_free(mod); module_free(mod, mod->module_init); free_core: -- cgit v1.2.3 From ac3c8141f62f357169980ec21b7be6d29964a394 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 17 Dec 2007 23:05:35 -0700 Subject: Kobject: convert kernel/module.c to use kobject_init/add_ng() This converts the code to use the new kobject functions, cleaning up the logic in doing so. Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/module.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index 0ae811785c5..89cd4c7361d 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1217,18 +1217,16 @@ int mod_sysfs_init(struct module *mod) err = -EINVAL; goto out; } - memset(&mod->mkobj.kobj, 0, sizeof(mod->mkobj.kobj)); - err = kobject_set_name(&mod->mkobj.kobj, "%s", mod->name); - if (err) - goto out; - mod->mkobj.kobj.kset = module_kset; - mod->mkobj.kobj.ktype = &module_ktype; mod->mkobj.mod = mod; - kobject_init(&mod->mkobj.kobj); + memset(&mod->mkobj.kobj, 0, sizeof(mod->mkobj.kobj)); + mod->mkobj.kobj.kset = module_kset; + err = kobject_init_and_add(&mod->mkobj.kobj, &module_ktype, NULL, + "%s", mod->name); + if (err) + kobject_put(&mod->mkobj.kobj); /* delay uevent until full sysfs population */ - err = kobject_add(&mod->mkobj.kobj); out: return err; } -- cgit v1.2.3 From 7a6a41615bfb2f03ce797bc24104c50b42c935e5 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Sat, 22 Dec 2007 21:18:25 -0800 Subject: Modules: remove unneeded release function Now that kobjects properly clean up their name structures, no matter if they have a release function or not, we can drop this empty module kobject release function too (it was needed prior to this because of the way we handled static kobject names, we based the fact that if a release function was present, then we could safely free the name string, now we are more smart about things and only free names we have previously set.) Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/params.c | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'kernel') diff --git a/kernel/params.c b/kernel/params.c index 1078b148ca8..b4da9505f4d 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -694,18 +694,8 @@ static struct kset_uevent_ops module_uevent_ops = { struct kset *module_kset; int module_sysfs_initialized; -static void module_release(struct kobject *kobj) -{ - /* - * Stupid empty release function to allow the memory for the kobject to - * be properly cleaned up. This will not need to be present for 2.6.25 - * with the upcoming kobject core rework. - */ -} - struct kobj_type module_ktype = { .sysfs_ops = &module_sysfs_ops, - .release = module_release, }; /* -- cgit v1.2.3 From 78a2d906b40fe530ea800c1e873bfe8f02326f1e Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 20 Dec 2007 08:13:05 -0800 Subject: Kobject: convert remaining kobject_unregister() to kobject_put() There is no need for kobject_unregister() anymore, thanks to Kay's kobject cleanup changes, so replace all instances of it with kobject_put(). Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/ksysfs.c | 2 +- kernel/module.c | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index 1081aff5fb9..e53bc30e9ba 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -154,7 +154,7 @@ notes_exit: group_exit: sysfs_remove_group(kernel_kobj, &kernel_attr_group); kset_exit: - kobject_unregister(kernel_kobj); + kobject_put(kernel_kobj); exit: return error; } diff --git a/kernel/module.c b/kernel/module.c index 89cd4c7361d..dcb8a2cbf75 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1257,9 +1257,8 @@ int mod_sysfs_setup(struct module *mod, out_unreg_param: module_param_sysfs_remove(mod); out_unreg_holders: - kobject_unregister(mod->holders_dir); + kobject_put(mod->holders_dir); out_unreg: - kobject_del(&mod->mkobj.kobj); kobject_put(&mod->mkobj.kobj); return err; } @@ -1269,9 +1268,9 @@ static void mod_kobject_remove(struct module *mod) { module_remove_modinfo_attrs(mod); module_param_sysfs_remove(mod); - kobject_unregister(mod->mkobj.drivers_dir); - kobject_unregister(mod->holders_dir); - kobject_unregister(&mod->mkobj.kobj); + kobject_put(mod->mkobj.drivers_dir); + kobject_put(mod->holders_dir); + kobject_put(&mod->mkobj.kobj); } /* -- cgit v1.2.3 From af5ca3f4ec5cc4432a42a73b050dd8898ce8fd00 Mon Sep 17 00:00:00 2001 From: Kay Sievers Date: Thu, 20 Dec 2007 02:09:39 +0100 Subject: Driver core: change sysdev classes to use dynamic kobject names All kobjects require a dynamically allocated name now. We no longer need to keep track if the name is statically assigned, we can just unconditionally free() all kobject names on cleanup. Signed-off-by: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- kernel/rtmutex-tester.c | 2 +- kernel/time/clocksource.c | 2 +- kernel/time/timekeeping.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/rtmutex-tester.c b/kernel/rtmutex-tester.c index e3055ba6915..092e4c620af 100644 --- a/kernel/rtmutex-tester.c +++ b/kernel/rtmutex-tester.c @@ -394,7 +394,7 @@ static SYSDEV_ATTR(status, 0600, sysfs_test_status, NULL); static SYSDEV_ATTR(command, 0600, NULL, sysfs_test_command); static struct sysdev_class rttest_sysclass = { - set_kset_name("rttest"), + .name = "rttest", }; static int init_test_thread(int id) diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index c8a9d13874d..8d6125ad2cf 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -441,7 +441,7 @@ static SYSDEV_ATTR(available_clocksource, 0600, sysfs_show_available_clocksources, NULL); static struct sysdev_class clocksource_sysclass = { - set_kset_name("clocksource"), + .name = "clocksource", }; static struct sys_device device_clocksource = { diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index e5e466b2759..ab46ae8c062 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -335,9 +335,9 @@ static int timekeeping_suspend(struct sys_device *dev, pm_message_t state) /* sysfs resume/suspend bits for timekeeping */ static struct sysdev_class timekeeping_sysclass = { + .name = "timekeeping", .resume = timekeeping_resume, .suspend = timekeeping_suspend, - set_kset_name("timekeeping"), }; static struct sys_device device_timer = { -- cgit v1.2.3 From 6ea6dd93c9454cc9521134f907bc970d09f460e4 Mon Sep 17 00:00:00 2001 From: Haavard Skinnemoen Date: Tue, 27 Nov 2007 13:02:40 +0100 Subject: ptrace: Call arch_ptrace_attach() when request=PTRACE_TRACEME arch_ptrace_attach() is a hook that allows the architecture to do book-keeping after a ptrace attach. This patch adds a call to this hook when handling a PTRACE_TRACEME request as well. Currently only one architecture, m32r, implements this hook. When called, it initializes a number of debug trap slots in the ptraced task's thread struct, and it looks to me like this is the right thing to do after a PTRACE_TRACEME request as well, not only after PTRACE_ATTACH. Please correct me if I'm wrong. I want to use this hook on AVR32 to turn the debugging hardware on when a process is actually being debugged and keep it off otherwise. To be able to do this, I need to intercept PTRACE_TRACEME and PTRACE_ATTACH, as well as PTRACE_DETACH and thread exit. The latter two can be handled by existing hooks. Signed-off-by: Haavard Skinnemoen --- kernel/ptrace.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/ptrace.c b/kernel/ptrace.c index c25db863081..c719bb9d79a 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -470,6 +470,8 @@ asmlinkage long sys_ptrace(long request, long pid, long addr, long data) lock_kernel(); if (request == PTRACE_TRACEME) { ret = ptrace_traceme(); + if (!ret) + arch_ptrace_attach(current); goto out; } -- cgit v1.2.3 From 32a76006683f7b28ae3cc491da37716e002f198e Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:07:58 +0100 Subject: printk: make printk more robust by not allowing recursion make printk more robust by allowing recursion only if there's a crash going on. Also add recursion detection. I've tested it with an artificially injected printk recursion - instead of a lockup or spontaneous reboot or other crash, the output was a well controlled: [ 41.057335] SysRq : <2>BUG: recent printk recursion! [ 41.057335] loglevel0-8 reBoot Crashdump show-all-locks(D) tErm Full kIll saK showMem Nice powerOff showPc show-all-timers(Q) unRaw Sync showTasks Unmount shoW-blocked-tasks also do all this printk-debug logic with irqs disabled. Signed-off-by: Ingo Molnar Reviewed-by: Nick Piggin --- kernel/printk.c | 48 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/printk.c b/kernel/printk.c index 89011bf8c10..e6c1f36d8c3 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -628,30 +628,57 @@ asmlinkage int printk(const char *fmt, ...) /* cpu currently holding logbuf_lock */ static volatile unsigned int printk_cpu = UINT_MAX; +const char printk_recursion_bug_msg [] = + KERN_CRIT "BUG: recent printk recursion!\n"; +static int printk_recursion_bug; + asmlinkage int vprintk(const char *fmt, va_list args) { + static int log_level_unknown = 1; + static char printk_buf[1024]; + unsigned long flags; - int printed_len; + int printed_len = 0; + int this_cpu; char *p; - static char printk_buf[1024]; - static int log_level_unknown = 1; boot_delay_msec(); preempt_disable(); - if (unlikely(oops_in_progress) && printk_cpu == smp_processor_id()) - /* If a crash is occurring during printk() on this CPU, - * make sure we can't deadlock */ - zap_locks(); - /* This stops the holder of console_sem just where we want him */ raw_local_irq_save(flags); + this_cpu = smp_processor_id(); + + /* + * Ouch, printk recursed into itself! + */ + if (unlikely(printk_cpu == this_cpu)) { + /* + * If a crash is occurring during printk() on this CPU, + * then try to get the crash message out but make sure + * we can't deadlock. Otherwise just return to avoid the + * recursion and return - but flag the recursion so that + * it can be printed at the next appropriate moment: + */ + if (!oops_in_progress) { + printk_recursion_bug = 1; + goto out_restore_irqs; + } + zap_locks(); + } + lockdep_off(); spin_lock(&logbuf_lock); - printk_cpu = smp_processor_id(); + printk_cpu = this_cpu; + if (printk_recursion_bug) { + printk_recursion_bug = 0; + strcpy(printk_buf, printk_recursion_bug_msg); + printed_len = sizeof(printk_recursion_bug_msg); + } /* Emit the output into the temporary buffer */ - printed_len = vscnprintf(printk_buf, sizeof(printk_buf), fmt, args); + printed_len += vscnprintf(printk_buf + printed_len, + sizeof(printk_buf), fmt, args); /* * Copy the output into log_buf. If the caller didn't provide @@ -744,6 +771,7 @@ asmlinkage int vprintk(const char *fmt, va_list args) printk_cpu = UINT_MAX; spin_unlock(&logbuf_lock); lockdep_on(); +out_restore_irqs: raw_local_irq_restore(flags); } -- cgit v1.2.3 From d713f519332e029d43eca8462629314eee1ded86 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:07:58 +0100 Subject: sched: fix CONFIG_PRINT_TIME's reliance on sched_clock() Stefano Brivio reported weird printk timestamp behavior during CPU frequency changes: http://bugzilla.kernel.org/show_bug.cgi?id=9475 fix CONFIG_PRINT_TIME's reliance on sched_clock() and use cpu_clock() instead. Reported-and-bisected-by: Stefano Brivio Signed-off-by: Ingo Molnar --- kernel/printk.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/printk.c b/kernel/printk.c index e6c1f36d8c3..5f9d053699f 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -707,7 +707,7 @@ asmlinkage int vprintk(const char *fmt, va_list args) loglev_char = default_message_loglevel + '0'; } - t = printk_clock(); + t = cpu_clock(printk_cpu); nanosec_rem = do_div(t, 1000000000); tlen = sprintf(tbuf, "<%c>[%5lu.%06lu] ", -- cgit v1.2.3 From b842271fbb9c8b5fd0e1c3e1895a3b67ba5bcc54 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:07:59 +0100 Subject: sched: remove printk_clock() printk_clock() is obsolete - it has been replaced with cpu_clock(). Signed-off-by: Ingo Molnar --- kernel/printk.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'kernel') diff --git a/kernel/printk.c b/kernel/printk.c index 5f9d053699f..3b7c968d0ef 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -573,11 +573,6 @@ static int __init printk_time_setup(char *str) __setup("time", printk_time_setup); -__attribute__((weak)) unsigned long long printk_clock(void) -{ - return sched_clock(); -} - /* Check if we have any console registered that can be called early in boot. */ static int have_callable_console(void) { -- cgit v1.2.3 From 93f992ccc008dd4030381caeebb252e85e66684b Mon Sep 17 00:00:00 2001 From: Srivatsa Vaddagiri Date: Fri, 25 Jan 2008 21:07:59 +0100 Subject: sched: group scheduling code cleanup Minor cleanups: - Fix coding style - remove obsolete comment Signed-off-by: Srivatsa Vaddagiri Signed-off-by: Ingo Molnar --- kernel/sched.c | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index e76b11ca6df..7f827b70ae0 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -191,12 +191,12 @@ struct task_group init_task_group = { }; #ifdef CONFIG_FAIR_USER_SCHED -# define INIT_TASK_GRP_LOAD 2*NICE_0_LOAD +# define INIT_TASK_GROUP_LOAD 2*NICE_0_LOAD #else -# define INIT_TASK_GRP_LOAD NICE_0_LOAD +# define INIT_TASK_GROUP_LOAD NICE_0_LOAD #endif -static int init_task_group_load = INIT_TASK_GRP_LOAD; +static int init_task_group_load = INIT_TASK_GROUP_LOAD; /* return group to which a task belongs */ static inline struct task_group *task_group(struct task_struct *p) @@ -881,21 +881,6 @@ static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {} #define sched_class_highest (&rt_sched_class) -/* - * Update delta_exec, delta_fair fields for rq. - * - * delta_fair clock advances at a rate inversely proportional to - * total load (rq->load.weight) on the runqueue, while - * delta_exec advances at the same rate as wall-clock (provided - * cpu is not idle). - * - * delta_exec / delta_fair is a measure of the (smoothened) load on this - * runqueue over any given interval. This (smoothened) load is used - * during load balance. - * - * This function is called /before/ updating rq->load - * and when switching tasks. - */ static inline void inc_load(struct rq *rq, const struct task_struct *p) { update_load_add(&rq->load, p->se.load.weight); -- cgit v1.2.3 From ec2c507fe8c8fa3c04fc6cb99a382a965c477379 Mon Sep 17 00:00:00 2001 From: Srivatsa Vaddagiri Date: Fri, 25 Jan 2008 21:07:59 +0100 Subject: sched: group scheduling, minor fixes Minor bug fixes for the group scheduler: - Use a mutex to serialize add/remove of task groups and also when changing shares of a task group. Use the same mutex when printing cfs_rq debugging stats for various task groups. - Use list_for_each_entry_rcu in for_each_leaf_cfs_rq macro (when walking task group list) Signed-off-by: Srivatsa Vaddagiri Signed-off-by: Ingo Molnar --- kernel/sched.c | 33 +++++++++++++++++++++++++-------- kernel/sched_fair.c | 4 +++- 2 files changed, 28 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 7f827b70ae0..cfa69581925 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -169,8 +169,6 @@ struct task_group { /* runqueue "owned" by this group on each cpu */ struct cfs_rq **cfs_rq; unsigned long shares; - /* spinlock to serialize modification to shares */ - spinlock_t lock; struct rcu_head rcu; }; @@ -182,6 +180,11 @@ static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; static struct sched_entity *init_sched_entity_p[NR_CPUS]; static struct cfs_rq *init_cfs_rq_p[NR_CPUS]; +/* task_group_mutex serializes add/remove of task groups and also changes to + * a task group's cpu shares. + */ +static DEFINE_MUTEX(task_group_mutex); + /* Default task group. * Every task in system belong to this group at bootup. */ @@ -221,9 +224,21 @@ static inline void set_task_cfs_rq(struct task_struct *p, unsigned int cpu) p->se.parent = task_group(p)->se[cpu]; } +static inline void lock_task_group_list(void) +{ + mutex_lock(&task_group_mutex); +} + +static inline void unlock_task_group_list(void) +{ + mutex_unlock(&task_group_mutex); +} + #else static inline void set_task_cfs_rq(struct task_struct *p, unsigned int cpu) { } +static inline void lock_task_group_list(void) { } +static inline void unlock_task_group_list(void) { } #endif /* CONFIG_FAIR_GROUP_SCHED */ @@ -6768,7 +6783,6 @@ void __init sched_init(void) se->parent = NULL; } init_task_group.shares = init_task_group_load; - spin_lock_init(&init_task_group.lock); #endif for (j = 0; j < CPU_LOAD_IDX_MAX; j++) @@ -7008,14 +7022,15 @@ struct task_group *sched_create_group(void) se->parent = NULL; } + tg->shares = NICE_0_LOAD; + + lock_task_group_list(); for_each_possible_cpu(i) { rq = cpu_rq(i); cfs_rq = tg->cfs_rq[i]; list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list); } - - tg->shares = NICE_0_LOAD; - spin_lock_init(&tg->lock); + unlock_task_group_list(); return tg; @@ -7061,10 +7076,12 @@ void sched_destroy_group(struct task_group *tg) struct cfs_rq *cfs_rq = NULL; int i; + lock_task_group_list(); for_each_possible_cpu(i) { cfs_rq = tg->cfs_rq[i]; list_del_rcu(&cfs_rq->leaf_cfs_rq_list); } + unlock_task_group_list(); BUG_ON(!cfs_rq); @@ -7146,7 +7163,7 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares) if (shares < 2) shares = 2; - spin_lock(&tg->lock); + lock_task_group_list(); if (tg->shares == shares) goto done; @@ -7155,7 +7172,7 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares) set_se_shares(tg->se[i], shares); done: - spin_unlock(&tg->lock); + unlock_task_group_list(); return 0; } diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index da7c061e720..0c5fdce6722 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -690,7 +690,7 @@ static inline struct cfs_rq *cpu_cfs_rq(struct cfs_rq *cfs_rq, int this_cpu) /* Iterate thr' all leaf cfs_rq's on a runqueue */ #define for_each_leaf_cfs_rq(rq, cfs_rq) \ - list_for_each_entry(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list) + list_for_each_entry_rcu(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list) /* Do the two (enqueued) entities belong to the same group ? */ static inline int @@ -1132,7 +1132,9 @@ static void print_cfs_stats(struct seq_file *m, int cpu) #ifdef CONFIG_FAIR_GROUP_SCHED print_cfs_rq(m, cpu, &cpu_rq(cpu)->cfs); #endif + lock_task_group_list(); for_each_leaf_cfs_rq(cpu_rq(cpu), cfs_rq) print_cfs_rq(m, cpu, cfs_rq); + unlock_task_group_list(); } #endif -- cgit v1.2.3 From 58e2d4ca581167c2a079f4ee02be2f0bc52e8729 Mon Sep 17 00:00:00 2001 From: Srivatsa Vaddagiri Date: Fri, 25 Jan 2008 21:08:00 +0100 Subject: sched: group scheduling, change how cpu load is calculated This patch changes how the cpu load exerted by fair_sched_class tasks is calculated. Load exerted by fair_sched_class tasks on a cpu is now a summation of the group weights, rather than summation of task weights. Weight exerted by a group on a cpu is dependent on the shares allocated to it. This version of patch has a minor impact on code size, but should have no runtime/functional impact for !CONFIG_FAIR_GROUP_SCHED. Signed-off-by: Srivatsa Vaddagiri Signed-off-by: Ingo Molnar --- kernel/sched.c | 27 +++++++++++---------------- kernel/sched_fair.c | 31 +++++++++++++++++++++++++++---- kernel/sched_rt.c | 2 ++ 3 files changed, 40 insertions(+), 20 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index cfa69581925..c915f3e6e59 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -886,6 +886,16 @@ static void cpuacct_charge(struct task_struct *tsk, u64 cputime); static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {} #endif +static inline void inc_cpu_load(struct rq *rq, unsigned long load) +{ + update_load_add(&rq->load, load); +} + +static inline void dec_cpu_load(struct rq *rq, unsigned long load) +{ + update_load_sub(&rq->load, load); +} + #include "sched_stats.h" #include "sched_idletask.c" #include "sched_fair.c" @@ -896,26 +906,14 @@ static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {} #define sched_class_highest (&rt_sched_class) -static inline void inc_load(struct rq *rq, const struct task_struct *p) -{ - update_load_add(&rq->load, p->se.load.weight); -} - -static inline void dec_load(struct rq *rq, const struct task_struct *p) -{ - update_load_sub(&rq->load, p->se.load.weight); -} - static void inc_nr_running(struct task_struct *p, struct rq *rq) { rq->nr_running++; - inc_load(rq, p); } static void dec_nr_running(struct task_struct *p, struct rq *rq) { rq->nr_running--; - dec_load(rq, p); } static void set_load_weight(struct task_struct *p) @@ -4087,10 +4085,8 @@ void set_user_nice(struct task_struct *p, long nice) goto out_unlock; } on_rq = p->se.on_rq; - if (on_rq) { + if (on_rq) dequeue_task(rq, p, 0); - dec_load(rq, p); - } p->static_prio = NICE_TO_PRIO(nice); set_load_weight(p); @@ -4100,7 +4096,6 @@ void set_user_nice(struct task_struct *p, long nice) if (on_rq) { enqueue_task(rq, p, 0); - inc_load(rq, p); /* * If the task increased its priority or is running and * lowered its priority, then reschedule its CPU: diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 0c5fdce6722..30ae9c2a286 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -760,15 +760,26 @@ static inline struct sched_entity *parent_entity(struct sched_entity *se) static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup) { struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; + struct sched_entity *se = &p->se, + *topse = NULL; /* Highest schedulable entity */ + int incload = 1; for_each_sched_entity(se) { - if (se->on_rq) + topse = se; + if (se->on_rq) { + incload = 0; break; + } cfs_rq = cfs_rq_of(se); enqueue_entity(cfs_rq, se, wakeup); wakeup = 1; } + /* Increment cpu load if we just enqueued the first task of a group on + * 'rq->cpu'. 'topse' represents the group to which task 'p' belongs + * at the highest grouping level. + */ + if (incload) + inc_cpu_load(rq, topse->load.weight); } /* @@ -779,16 +790,28 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup) static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep) { struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; + struct sched_entity *se = &p->se, + *topse = NULL; /* Highest schedulable entity */ + int decload = 1; for_each_sched_entity(se) { + topse = se; cfs_rq = cfs_rq_of(se); dequeue_entity(cfs_rq, se, sleep); /* Don't dequeue parent if it has other entities besides us */ - if (cfs_rq->load.weight) + if (cfs_rq->load.weight) { + if (parent_entity(se)) + decload = 0; break; + } sleep = 1; } + /* Decrement cpu load if we just dequeued the last task of a group on + * 'rq->cpu'. 'topse' represents the group to which task 'p' belongs + * at the highest grouping level. + */ + if (decload) + dec_cpu_load(rq, topse->load.weight); } /* diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 9ba3daa0347..cefcd510514 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -32,6 +32,7 @@ static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup) list_add_tail(&p->run_list, array->queue + p->prio); __set_bit(p->prio, array->bitmap); + inc_cpu_load(rq, p->se.load.weight); } /* @@ -46,6 +47,7 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep) list_del(&p->run_list); if (list_empty(array->queue + p->prio)) __clear_bit(p->prio, array->bitmap); + dec_cpu_load(rq, p->se.load.weight); } /* -- cgit v1.2.3 From a183561567b5446d3362b4839bd4f744f4b2af1e Mon Sep 17 00:00:00 2001 From: Srivatsa Vaddagiri Date: Fri, 25 Jan 2008 21:08:00 +0100 Subject: sched: introduce a mutex and corresponding API to serialize access to doms_curarray doms_cur[] array represents various scheduling domains which are mutually exclusive. Currently cpusets code can modify this array (by calling partition_sched_domains()) as a result of user modifying sched_load_balance flag for various cpusets. This patch introduces a mutex and corresponding API (only when CONFIG_FAIR_GROUP_SCHED is defined) which allows a reader to safely read the doms_cur[] array w/o worrying abt concurrent modifications to the array. The fair group scheduler code (introduced in next patch of this series) makes use of this mutex to walk thr' doms_cur[] array while rebalancing shares of task groups across cpus. Signed-off-by: Srivatsa Vaddagiri Signed-off-by: Ingo Molnar --- kernel/sched.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index c915f3e6e59..d9585f15043 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -185,6 +185,9 @@ static struct cfs_rq *init_cfs_rq_p[NR_CPUS]; */ static DEFINE_MUTEX(task_group_mutex); +/* doms_cur_mutex serializes access to doms_cur[] array */ +static DEFINE_MUTEX(doms_cur_mutex); + /* Default task group. * Every task in system belong to this group at bootup. */ @@ -234,11 +237,23 @@ static inline void unlock_task_group_list(void) mutex_unlock(&task_group_mutex); } +static inline void lock_doms_cur(void) +{ + mutex_lock(&doms_cur_mutex); +} + +static inline void unlock_doms_cur(void) +{ + mutex_unlock(&doms_cur_mutex); +} + #else static inline void set_task_cfs_rq(struct task_struct *p, unsigned int cpu) { } static inline void lock_task_group_list(void) { } static inline void unlock_task_group_list(void) { } +static inline void lock_doms_cur(void) { } +static inline void unlock_doms_cur(void) { } #endif /* CONFIG_FAIR_GROUP_SCHED */ @@ -6543,6 +6558,8 @@ void partition_sched_domains(int ndoms_new, cpumask_t *doms_new) { int i, j; + lock_doms_cur(); + /* always unregister in case we don't destroy any domains */ unregister_sched_domain_sysctl(); @@ -6583,6 +6600,8 @@ match2: ndoms_cur = ndoms_new; register_sched_domain_sysctl(); + + unlock_doms_cur(); } #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT) -- cgit v1.2.3 From 6b2d7700266b9402e12824e11e0099ae6a4a6a79 Mon Sep 17 00:00:00 2001 From: Srivatsa Vaddagiri Date: Fri, 25 Jan 2008 21:08:00 +0100 Subject: sched: group scheduler, fix fairness of cpu bandwidth allocation for task groups The current load balancing scheme isn't good enough for precise group fairness. For example: on a 8-cpu system, I created 3 groups as under: a = 8 tasks (cpu.shares = 1024) b = 4 tasks (cpu.shares = 1024) c = 3 tasks (cpu.shares = 1024) a, b and c are task groups that have equal weight. We would expect each of the groups to receive 33.33% of cpu bandwidth under a fair scheduler. This is what I get with the latest scheduler git tree: Signed-off-by: Ingo Molnar -------------------------------------------------------------------------------- Col1 | Col2 | Col3 | Col4 ------|---------|-------|------------------------------------------------------- a | 277.676 | 57.8% | 54.1% 54.1% 54.1% 54.2% 56.7% 62.2% 62.8% 64.5% b | 116.108 | 24.2% | 47.4% 48.1% 48.7% 49.3% c | 86.326 | 18.0% | 47.5% 47.9% 48.5% -------------------------------------------------------------------------------- Explanation of o/p: Col1 -> Group name Col2 -> Cumulative execution time (in seconds) received by all tasks of that group in a 60sec window across 8 cpus Col3 -> CPU bandwidth received by the group in the 60sec window, expressed in percentage. Col3 data is derived as: Col3 = 100 * Col2 / (NR_CPUS * 60) Col4 -> CPU bandwidth received by each individual task of the group. Col4 = 100 * cpu_time_recd_by_task / 60 [I can share the test case that produces a similar o/p if reqd] The deviation from desired group fairness is as below: a = +24.47% b = -9.13% c = -15.33% which is quite high. After the patch below is applied, here are the results: -------------------------------------------------------------------------------- Col1 | Col2 | Col3 | Col4 ------|---------|-------|------------------------------------------------------- a | 163.112 | 34.0% | 33.2% 33.4% 33.5% 33.5% 33.7% 34.4% 34.8% 35.3% b | 156.220 | 32.5% | 63.3% 64.5% 66.1% 66.5% c | 160.653 | 33.5% | 85.8% 90.6% 91.4% -------------------------------------------------------------------------------- Deviation from desired group fairness is as below: a = +0.67% b = -0.83% c = +0.17% which is far better IMO. Most of other runs have yielded a deviation within +-2% at the most, which is good. Why do we see bad (group) fairness with current scheuler? ========================================================= Currently cpu's weight is just the summation of individual task weights. This can yield incorrect results. For ex: consider three groups as below on a 2-cpu system: CPU0 CPU1 --------------------------- A (10) B(5) C(5) --------------------------- Group A has 10 tasks, all on CPU0, Group B and C have 5 tasks each all of which are on CPU1. Each task has the same weight (NICE_0_LOAD = 1024). The current scheme would yield a cpu weight of 10240 (10*1024) for each cpu and the load balancer will think both CPUs are perfectly balanced and won't move around any tasks. This, however, would yield this bandwidth: A = 50% B = 25% C = 25% which is not the desired result. What's changing in the patch? ============================= - How cpu weights are calculated when CONFIF_FAIR_GROUP_SCHED is defined (see below) - API Change - Two tunables introduced in sysfs (under SCHED_DEBUG) to control the frequency at which the load balance monitor thread runs. The basic change made in this patch is how cpu weight (rq->load.weight) is calculated. Its now calculated as the summation of group weights on a cpu, rather than summation of task weights. Weight exerted by a group on a cpu is dependent on the shares allocated to it and also the number of tasks the group has on that cpu compared to the total number of (runnable) tasks the group has in the system. Let, W(K,i) = Weight of group K on cpu i T(K,i) = Task load present in group K's cfs_rq on cpu i T(K) = Total task load of group K across various cpus S(K) = Shares allocated to group K NRCPUS = Number of online cpus in the scheduler domain to which group K is assigned. Then, W(K,i) = S(K) * NRCPUS * T(K,i) / T(K) A load balance monitor thread is created at bootup, which periodically runs and adjusts group's weight on each cpu. To avoid its overhead, two min/max tunables are introduced (under SCHED_DEBUG) to control the rate at which it runs. Fixes from: Peter Zijlstra - don't start the load_balance_monitor when there is only a single cpu. - rename the kthread because its currently longer than TASK_COMM_LEN Signed-off-by: Srivatsa Vaddagiri Signed-off-by: Ingo Molnar --- kernel/sched.c | 270 +++++++++++++++++++++++++++++++++++++++++++++++++--- kernel/sched_fair.c | 84 ++++++++++------ kernel/sysctl.c | 18 ++++ 3 files changed, 327 insertions(+), 45 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index d9585f15043..86e55a9c2de 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -168,7 +168,43 @@ struct task_group { struct sched_entity **se; /* runqueue "owned" by this group on each cpu */ struct cfs_rq **cfs_rq; + + /* + * shares assigned to a task group governs how much of cpu bandwidth + * is allocated to the group. The more shares a group has, the more is + * the cpu bandwidth allocated to it. + * + * For ex, lets say that there are three task groups, A, B and C which + * have been assigned shares 1000, 2000 and 3000 respectively. Then, + * cpu bandwidth allocated by the scheduler to task groups A, B and C + * should be: + * + * Bw(A) = 1000/(1000+2000+3000) * 100 = 16.66% + * Bw(B) = 2000/(1000+2000+3000) * 100 = 33.33% + * Bw(C) = 3000/(1000+2000+3000) * 100 = 50% + * + * The weight assigned to a task group's schedulable entities on every + * cpu (task_group.se[a_cpu]->load.weight) is derived from the task + * group's shares. For ex: lets say that task group A has been + * assigned shares of 1000 and there are two CPUs in a system. Then, + * + * tg_A->se[0]->load.weight = tg_A->se[1]->load.weight = 1000; + * + * Note: It's not necessary that each of a task's group schedulable + * entity have the same weight on all CPUs. If the group + * has 2 of its tasks on CPU0 and 1 task on CPU1, then a + * better distribution of weight could be: + * + * tg_A->se[0]->load.weight = 2/3 * 2000 = 1333 + * tg_A->se[1]->load.weight = 1/2 * 2000 = 667 + * + * rebalance_shares() is responsible for distributing the shares of a + * task groups like this among the group's schedulable entities across + * cpus. + * + */ unsigned long shares; + struct rcu_head rcu; }; @@ -188,6 +224,14 @@ static DEFINE_MUTEX(task_group_mutex); /* doms_cur_mutex serializes access to doms_cur[] array */ static DEFINE_MUTEX(doms_cur_mutex); +#ifdef CONFIG_SMP +/* kernel thread that runs rebalance_shares() periodically */ +static struct task_struct *lb_monitor_task; +static int load_balance_monitor(void *unused); +#endif + +static void set_se_shares(struct sched_entity *se, unsigned long shares); + /* Default task group. * Every task in system belong to this group at bootup. */ @@ -202,6 +246,8 @@ struct task_group init_task_group = { # define INIT_TASK_GROUP_LOAD NICE_0_LOAD #endif +#define MIN_GROUP_SHARES 2 + static int init_task_group_load = INIT_TASK_GROUP_LOAD; /* return group to which a task belongs */ @@ -6736,6 +6782,21 @@ void __init sched_init_smp(void) if (set_cpus_allowed(current, non_isolated_cpus) < 0) BUG(); sched_init_granularity(); + +#ifdef CONFIG_FAIR_GROUP_SCHED + if (nr_cpu_ids == 1) + return; + + lb_monitor_task = kthread_create(load_balance_monitor, NULL, + "group_balance"); + if (!IS_ERR(lb_monitor_task)) { + lb_monitor_task->flags |= PF_NOFREEZE; + wake_up_process(lb_monitor_task); + } else { + printk(KERN_ERR "Could not create load balance monitor thread" + "(error = %ld) \n", PTR_ERR(lb_monitor_task)); + } +#endif } #else void __init sched_init_smp(void) @@ -6988,6 +7049,157 @@ void set_curr_task(int cpu, struct task_struct *p) #ifdef CONFIG_FAIR_GROUP_SCHED +#ifdef CONFIG_SMP +/* + * distribute shares of all task groups among their schedulable entities, + * to reflect load distrbution across cpus. + */ +static int rebalance_shares(struct sched_domain *sd, int this_cpu) +{ + struct cfs_rq *cfs_rq; + struct rq *rq = cpu_rq(this_cpu); + cpumask_t sdspan = sd->span; + int balanced = 1; + + /* Walk thr' all the task groups that we have */ + for_each_leaf_cfs_rq(rq, cfs_rq) { + int i; + unsigned long total_load = 0, total_shares; + struct task_group *tg = cfs_rq->tg; + + /* Gather total task load of this group across cpus */ + for_each_cpu_mask(i, sdspan) + total_load += tg->cfs_rq[i]->load.weight; + + /* Nothing to do if this group has no load */ + if (!total_load) + continue; + + /* + * tg->shares represents the number of cpu shares the task group + * is eligible to hold on a single cpu. On N cpus, it is + * eligible to hold (N * tg->shares) number of cpu shares. + */ + total_shares = tg->shares * cpus_weight(sdspan); + + /* + * redistribute total_shares across cpus as per the task load + * distribution. + */ + for_each_cpu_mask(i, sdspan) { + unsigned long local_load, local_shares; + + local_load = tg->cfs_rq[i]->load.weight; + local_shares = (local_load * total_shares) / total_load; + if (!local_shares) + local_shares = MIN_GROUP_SHARES; + if (local_shares == tg->se[i]->load.weight) + continue; + + spin_lock_irq(&cpu_rq(i)->lock); + set_se_shares(tg->se[i], local_shares); + spin_unlock_irq(&cpu_rq(i)->lock); + balanced = 0; + } + } + + return balanced; +} + +/* + * How frequently should we rebalance_shares() across cpus? + * + * The more frequently we rebalance shares, the more accurate is the fairness + * of cpu bandwidth distribution between task groups. However higher frequency + * also implies increased scheduling overhead. + * + * sysctl_sched_min_bal_int_shares represents the minimum interval between + * consecutive calls to rebalance_shares() in the same sched domain. + * + * sysctl_sched_max_bal_int_shares represents the maximum interval between + * consecutive calls to rebalance_shares() in the same sched domain. + * + * These settings allows for the appropriate tradeoff between accuracy of + * fairness and the associated overhead. + * + */ + +/* default: 8ms, units: milliseconds */ +const_debug unsigned int sysctl_sched_min_bal_int_shares = 8; + +/* default: 128ms, units: milliseconds */ +const_debug unsigned int sysctl_sched_max_bal_int_shares = 128; + +/* kernel thread that runs rebalance_shares() periodically */ +static int load_balance_monitor(void *unused) +{ + unsigned int timeout = sysctl_sched_min_bal_int_shares; + struct sched_param schedparm; + int ret; + + /* + * We don't want this thread's execution to be limited by the shares + * assigned to default group (init_task_group). Hence make it run + * as a SCHED_RR RT task at the lowest priority. + */ + schedparm.sched_priority = 1; + ret = sched_setscheduler(current, SCHED_RR, &schedparm); + if (ret) + printk(KERN_ERR "Couldn't set SCHED_RR policy for load balance" + " monitor thread (error = %d) \n", ret); + + while (!kthread_should_stop()) { + int i, cpu, balanced = 1; + + /* Prevent cpus going down or coming up */ + lock_cpu_hotplug(); + /* lockout changes to doms_cur[] array */ + lock_doms_cur(); + /* + * Enter a rcu read-side critical section to safely walk rq->sd + * chain on various cpus and to walk task group list + * (rq->leaf_cfs_rq_list) in rebalance_shares(). + */ + rcu_read_lock(); + + for (i = 0; i < ndoms_cur; i++) { + cpumask_t cpumap = doms_cur[i]; + struct sched_domain *sd = NULL, *sd_prev = NULL; + + cpu = first_cpu(cpumap); + + /* Find the highest domain at which to balance shares */ + for_each_domain(cpu, sd) { + if (!(sd->flags & SD_LOAD_BALANCE)) + continue; + sd_prev = sd; + } + + sd = sd_prev; + /* sd == NULL? No load balance reqd in this domain */ + if (!sd) + continue; + + balanced &= rebalance_shares(sd, cpu); + } + + rcu_read_unlock(); + + unlock_doms_cur(); + unlock_cpu_hotplug(); + + if (!balanced) + timeout = sysctl_sched_min_bal_int_shares; + else if (timeout < sysctl_sched_max_bal_int_shares) + timeout *= 2; + + msleep_interruptible(timeout); + } + + return 0; +} +#endif /* CONFIG_SMP */ + /* allocate runqueue etc for a new task group */ struct task_group *sched_create_group(void) { @@ -7144,47 +7356,77 @@ done: task_rq_unlock(rq, &flags); } +/* rq->lock to be locked by caller */ static void set_se_shares(struct sched_entity *se, unsigned long shares) { struct cfs_rq *cfs_rq = se->cfs_rq; struct rq *rq = cfs_rq->rq; int on_rq; - spin_lock_irq(&rq->lock); + if (!shares) + shares = MIN_GROUP_SHARES; on_rq = se->on_rq; - if (on_rq) + if (on_rq) { dequeue_entity(cfs_rq, se, 0); + dec_cpu_load(rq, se->load.weight); + } se->load.weight = shares; se->load.inv_weight = div64_64((1ULL<<32), shares); - if (on_rq) + if (on_rq) { enqueue_entity(cfs_rq, se, 0); - - spin_unlock_irq(&rq->lock); + inc_cpu_load(rq, se->load.weight); + } } int sched_group_set_shares(struct task_group *tg, unsigned long shares) { int i; - - /* - * A weight of 0 or 1 can cause arithmetics problems. - * (The default weight is 1024 - so there's no practical - * limitation from this.) - */ - if (shares < 2) - shares = 2; + struct cfs_rq *cfs_rq; + struct rq *rq; lock_task_group_list(); if (tg->shares == shares) goto done; + if (shares < MIN_GROUP_SHARES) + shares = MIN_GROUP_SHARES; + + /* + * Prevent any load balance activity (rebalance_shares, + * load_balance_fair) from referring to this group first, + * by taking it off the rq->leaf_cfs_rq_list on each cpu. + */ + for_each_possible_cpu(i) { + cfs_rq = tg->cfs_rq[i]; + list_del_rcu(&cfs_rq->leaf_cfs_rq_list); + } + + /* wait for any ongoing reference to this group to finish */ + synchronize_sched(); + + /* + * Now we are free to modify the group's share on each cpu + * w/o tripping rebalance_share or load_balance_fair. + */ tg->shares = shares; - for_each_possible_cpu(i) + for_each_possible_cpu(i) { + spin_lock_irq(&cpu_rq(i)->lock); set_se_shares(tg->se[i], shares); + spin_unlock_irq(&cpu_rq(i)->lock); + } + /* + * Enable load balance activity on this group, by inserting it back on + * each cpu's rq->leaf_cfs_rq_list. + */ + for_each_possible_cpu(i) { + rq = cpu_rq(i); + cfs_rq = tg->cfs_rq[i]; + list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list); + } done: unlock_task_group_list(); return 0; diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 30ae9c2a286..5c208e090ae 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -707,6 +707,8 @@ static inline struct sched_entity *parent_entity(struct sched_entity *se) return se->parent; } +#define GROUP_IMBALANCE_PCT 20 + #else /* CONFIG_FAIR_GROUP_SCHED */ #define for_each_sched_entity(se) \ @@ -967,25 +969,6 @@ static struct task_struct *load_balance_next_fair(void *arg) return __load_balance_iterator(cfs_rq, cfs_rq->rb_load_balance_curr); } -#ifdef CONFIG_FAIR_GROUP_SCHED -static int cfs_rq_best_prio(struct cfs_rq *cfs_rq) -{ - struct sched_entity *curr; - struct task_struct *p; - - if (!cfs_rq->nr_running) - return MAX_PRIO; - - curr = cfs_rq->curr; - if (!curr) - curr = __pick_next_entity(cfs_rq); - - p = task_of(curr); - - return p->prio; -} -#endif - static unsigned long load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, unsigned long max_load_move, @@ -995,28 +978,45 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, struct cfs_rq *busy_cfs_rq; long rem_load_move = max_load_move; struct rq_iterator cfs_rq_iterator; + unsigned long load_moved; cfs_rq_iterator.start = load_balance_start_fair; cfs_rq_iterator.next = load_balance_next_fair; for_each_leaf_cfs_rq(busiest, busy_cfs_rq) { #ifdef CONFIG_FAIR_GROUP_SCHED - struct cfs_rq *this_cfs_rq; - long imbalance; - unsigned long maxload; + struct cfs_rq *this_cfs_rq = busy_cfs_rq->tg->cfs_rq[this_cpu]; + unsigned long maxload, task_load, group_weight; + unsigned long thisload, per_task_load; + struct sched_entity *se = busy_cfs_rq->tg->se[busiest->cpu]; - this_cfs_rq = cpu_cfs_rq(busy_cfs_rq, this_cpu); + task_load = busy_cfs_rq->load.weight; + group_weight = se->load.weight; - imbalance = busy_cfs_rq->load.weight - this_cfs_rq->load.weight; - /* Don't pull if this_cfs_rq has more load than busy_cfs_rq */ - if (imbalance <= 0) + /* + * 'group_weight' is contributed by tasks of total weight + * 'task_load'. To move 'rem_load_move' worth of weight only, + * we need to move a maximum task load of: + * + * maxload = (remload / group_weight) * task_load; + */ + maxload = (rem_load_move * task_load) / group_weight; + + if (!maxload || !task_load) continue; - /* Don't pull more than imbalance/2 */ - imbalance /= 2; - maxload = min(rem_load_move, imbalance); + per_task_load = task_load / busy_cfs_rq->nr_running; + /* + * balance_tasks will try to forcibly move atleast one task if + * possible (because of SCHED_LOAD_SCALE_FUZZ). Avoid that if + * maxload is less than GROUP_IMBALANCE_FUZZ% the per_task_load. + */ + if (100 * maxload < GROUP_IMBALANCE_PCT * per_task_load) + continue; - *this_best_prio = cfs_rq_best_prio(this_cfs_rq); + /* Disable priority-based load balance */ + *this_best_prio = 0; + thisload = this_cfs_rq->load.weight; #else # define maxload rem_load_move #endif @@ -1025,11 +1025,33 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, * load_balance_[start|next]_fair iterators */ cfs_rq_iterator.arg = busy_cfs_rq; - rem_load_move -= balance_tasks(this_rq, this_cpu, busiest, + load_moved = balance_tasks(this_rq, this_cpu, busiest, maxload, sd, idle, all_pinned, this_best_prio, &cfs_rq_iterator); +#ifdef CONFIG_FAIR_GROUP_SCHED + /* + * load_moved holds the task load that was moved. The + * effective (group) weight moved would be: + * load_moved_eff = load_moved/task_load * group_weight; + */ + load_moved = (group_weight * load_moved) / task_load; + + /* Adjust shares on both cpus to reflect load_moved */ + group_weight -= load_moved; + set_se_shares(se, group_weight); + + se = busy_cfs_rq->tg->se[this_cpu]; + if (!thisload) + group_weight = load_moved; + else + group_weight = se->load.weight + load_moved; + set_se_shares(se, group_weight); +#endif + + rem_load_move -= load_moved; + if (rem_load_move <= 0) break; } diff --git a/kernel/sysctl.c b/kernel/sysctl.c index c68f68dcc60..c95f3ed3447 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -309,6 +309,24 @@ static struct ctl_table kern_table[] = { .mode = 644, .proc_handler = &proc_dointvec, }, +#if defined(CONFIG_FAIR_GROUP_SCHED) && defined(CONFIG_SMP) + { + .ctl_name = CTL_UNNUMBERED, + .procname = "sched_min_bal_int_shares", + .data = &sysctl_sched_min_bal_int_shares, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = &proc_dointvec, + }, + { + .ctl_name = CTL_UNNUMBERED, + .procname = "sched_max_bal_int_shares", + .data = &sysctl_sched_max_bal_int_shares, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = &proc_dointvec, + }, +#endif #endif { .ctl_name = CTL_UNNUMBERED, -- cgit v1.2.3 From d221938c049f4845da13c8593132595a6b9222a8 Mon Sep 17 00:00:00 2001 From: Gautham R Shenoy Date: Fri, 25 Jan 2008 21:08:01 +0100 Subject: cpu-hotplug: refcount based cpu hotplug This patch implements a Refcount + Waitqueue based model for cpu-hotplug. Now, a thread which wants to prevent cpu-hotplug, will bump up a global refcount and the thread which wants to perform a cpu-hotplug operation will block till the global refcount goes to zero. The readers, if any, during an ongoing cpu-hotplug operation are blocked until the cpu-hotplug operation is over. Signed-off-by: Gautham R Shenoy Signed-off-by: Paul Jackson [For !CONFIG_HOTPLUG_CPU ] Signed-off-by: Ingo Molnar --- kernel/cpu.c | 152 +++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 111 insertions(+), 41 deletions(-) (limited to 'kernel') diff --git a/kernel/cpu.c b/kernel/cpu.c index 6b3a0c15144..656dc3fcbba 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -15,9 +15,8 @@ #include #include -/* This protects CPUs going up and down... */ +/* Serializes the updates to cpu_online_map, cpu_present_map */ static DEFINE_MUTEX(cpu_add_remove_lock); -static DEFINE_MUTEX(cpu_bitmask_lock); static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain); @@ -26,52 +25,123 @@ static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain); */ static int cpu_hotplug_disabled; -#ifdef CONFIG_HOTPLUG_CPU +static struct { + struct task_struct *active_writer; + struct mutex lock; /* Synchronizes accesses to refcount, */ + /* + * Also blocks the new readers during + * an ongoing cpu hotplug operation. + */ + int refcount; + wait_queue_head_t writer_queue; +} cpu_hotplug; -/* Crappy recursive lock-takers in cpufreq! Complain loudly about idiots */ -static struct task_struct *recursive; -static int recursive_depth; +#define writer_exists() (cpu_hotplug.active_writer != NULL) + +void __init cpu_hotplug_init(void) +{ + cpu_hotplug.active_writer = NULL; + mutex_init(&cpu_hotplug.lock); + cpu_hotplug.refcount = 0; + init_waitqueue_head(&cpu_hotplug.writer_queue); +} + +#ifdef CONFIG_HOTPLUG_CPU void lock_cpu_hotplug(void) { - struct task_struct *tsk = current; - - if (tsk == recursive) { - static int warnings = 10; - if (warnings) { - printk(KERN_ERR "Lukewarm IQ detected in hotplug locking\n"); - WARN_ON(1); - warnings--; - } - recursive_depth++; + might_sleep(); + if (cpu_hotplug.active_writer == current) return; - } - mutex_lock(&cpu_bitmask_lock); - recursive = tsk; + mutex_lock(&cpu_hotplug.lock); + cpu_hotplug.refcount++; + mutex_unlock(&cpu_hotplug.lock); + } EXPORT_SYMBOL_GPL(lock_cpu_hotplug); void unlock_cpu_hotplug(void) { - WARN_ON(recursive != current); - if (recursive_depth) { - recursive_depth--; + if (cpu_hotplug.active_writer == current) return; - } - recursive = NULL; - mutex_unlock(&cpu_bitmask_lock); + mutex_lock(&cpu_hotplug.lock); + cpu_hotplug.refcount--; + + if (unlikely(writer_exists()) && !cpu_hotplug.refcount) + wake_up(&cpu_hotplug.writer_queue); + + mutex_unlock(&cpu_hotplug.lock); + } EXPORT_SYMBOL_GPL(unlock_cpu_hotplug); #endif /* CONFIG_HOTPLUG_CPU */ +/* + * The following two API's must be used when attempting + * to serialize the updates to cpu_online_map, cpu_present_map. + */ +void cpu_maps_update_begin(void) +{ + mutex_lock(&cpu_add_remove_lock); +} + +void cpu_maps_update_done(void) +{ + mutex_unlock(&cpu_add_remove_lock); +} + +/* + * This ensures that the hotplug operation can begin only when the + * refcount goes to zero. + * + * Note that during a cpu-hotplug operation, the new readers, if any, + * will be blocked by the cpu_hotplug.lock + * + * Since cpu_maps_update_begin is always called after invoking + * cpu_maps_update_begin, we can be sure that only one writer is active. + * + * Note that theoretically, there is a possibility of a livelock: + * - Refcount goes to zero, last reader wakes up the sleeping + * writer. + * - Last reader unlocks the cpu_hotplug.lock. + * - A new reader arrives at this moment, bumps up the refcount. + * - The writer acquires the cpu_hotplug.lock finds the refcount + * non zero and goes to sleep again. + * + * However, this is very difficult to achieve in practice since + * lock_cpu_hotplug() not an api which is called all that often. + * + */ +static void cpu_hotplug_begin(void) +{ + DECLARE_WAITQUEUE(wait, current); + + mutex_lock(&cpu_hotplug.lock); + + cpu_hotplug.active_writer = current; + add_wait_queue_exclusive(&cpu_hotplug.writer_queue, &wait); + while (cpu_hotplug.refcount) { + set_current_state(TASK_UNINTERRUPTIBLE); + mutex_unlock(&cpu_hotplug.lock); + schedule(); + mutex_lock(&cpu_hotplug.lock); + } + remove_wait_queue_locked(&cpu_hotplug.writer_queue, &wait); +} + +static void cpu_hotplug_done(void) +{ + cpu_hotplug.active_writer = NULL; + mutex_unlock(&cpu_hotplug.lock); +} /* Need to know about CPUs going up/down? */ int __cpuinit register_cpu_notifier(struct notifier_block *nb) { int ret; - mutex_lock(&cpu_add_remove_lock); + cpu_maps_update_begin(); ret = raw_notifier_chain_register(&cpu_chain, nb); - mutex_unlock(&cpu_add_remove_lock); + cpu_maps_update_done(); return ret; } @@ -81,9 +151,9 @@ EXPORT_SYMBOL(register_cpu_notifier); void unregister_cpu_notifier(struct notifier_block *nb) { - mutex_lock(&cpu_add_remove_lock); + cpu_maps_update_begin(); raw_notifier_chain_unregister(&cpu_chain, nb); - mutex_unlock(&cpu_add_remove_lock); + cpu_maps_update_done(); } EXPORT_SYMBOL(unregister_cpu_notifier); @@ -147,6 +217,7 @@ static int _cpu_down(unsigned int cpu, int tasks_frozen) if (!cpu_online(cpu)) return -EINVAL; + cpu_hotplug_begin(); raw_notifier_call_chain(&cpu_chain, CPU_LOCK_ACQUIRE, hcpu); err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); @@ -166,9 +237,7 @@ static int _cpu_down(unsigned int cpu, int tasks_frozen) cpu_clear(cpu, tmp); set_cpus_allowed(current, tmp); - mutex_lock(&cpu_bitmask_lock); p = __stop_machine_run(take_cpu_down, &tcd_param, cpu); - mutex_unlock(&cpu_bitmask_lock); if (IS_ERR(p) || cpu_online(cpu)) { /* CPU didn't die: tell everyone. Can't complain. */ @@ -203,6 +272,7 @@ out_allowed: set_cpus_allowed(current, old_allowed); out_release: raw_notifier_call_chain(&cpu_chain, CPU_LOCK_RELEASE, hcpu); + cpu_hotplug_done(); return err; } @@ -210,13 +280,13 @@ int cpu_down(unsigned int cpu) { int err = 0; - mutex_lock(&cpu_add_remove_lock); + cpu_maps_update_begin(); if (cpu_hotplug_disabled) err = -EBUSY; else err = _cpu_down(cpu, 0); - mutex_unlock(&cpu_add_remove_lock); + cpu_maps_update_done(); return err; } #endif /*CONFIG_HOTPLUG_CPU*/ @@ -231,6 +301,7 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) if (cpu_online(cpu) || !cpu_present(cpu)) return -EINVAL; + cpu_hotplug_begin(); raw_notifier_call_chain(&cpu_chain, CPU_LOCK_ACQUIRE, hcpu); ret = __raw_notifier_call_chain(&cpu_chain, CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls); @@ -243,9 +314,7 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) } /* Arch-specific enabling code. */ - mutex_lock(&cpu_bitmask_lock); ret = __cpu_up(cpu); - mutex_unlock(&cpu_bitmask_lock); if (ret != 0) goto out_notify; BUG_ON(!cpu_online(cpu)); @@ -258,6 +327,7 @@ out_notify: __raw_notifier_call_chain(&cpu_chain, CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL); raw_notifier_call_chain(&cpu_chain, CPU_LOCK_RELEASE, hcpu); + cpu_hotplug_done(); return ret; } @@ -275,13 +345,13 @@ int __cpuinit cpu_up(unsigned int cpu) return -EINVAL; } - mutex_lock(&cpu_add_remove_lock); + cpu_maps_update_begin(); if (cpu_hotplug_disabled) err = -EBUSY; else err = _cpu_up(cpu, 0); - mutex_unlock(&cpu_add_remove_lock); + cpu_maps_update_done(); return err; } @@ -292,7 +362,7 @@ int disable_nonboot_cpus(void) { int cpu, first_cpu, error = 0; - mutex_lock(&cpu_add_remove_lock); + cpu_maps_update_begin(); first_cpu = first_cpu(cpu_online_map); /* We take down all of the non-boot CPUs in one shot to avoid races * with the userspace trying to use the CPU hotplug at the same time @@ -319,7 +389,7 @@ int disable_nonboot_cpus(void) } else { printk(KERN_ERR "Non-boot CPUs are not disabled\n"); } - mutex_unlock(&cpu_add_remove_lock); + cpu_maps_update_done(); return error; } @@ -328,7 +398,7 @@ void enable_nonboot_cpus(void) int cpu, error; /* Allow everyone to use the CPU hotplug again */ - mutex_lock(&cpu_add_remove_lock); + cpu_maps_update_begin(); cpu_hotplug_disabled = 0; if (cpus_empty(frozen_cpus)) goto out; @@ -344,6 +414,6 @@ void enable_nonboot_cpus(void) } cpus_clear(frozen_cpus); out: - mutex_unlock(&cpu_add_remove_lock); + cpu_maps_update_done(); } #endif /* CONFIG_PM_SLEEP_SMP */ -- cgit v1.2.3 From 86ef5c9a8edd78e6bf92879f32329d89b2d55b5a Mon Sep 17 00:00:00 2001 From: Gautham R Shenoy Date: Fri, 25 Jan 2008 21:08:02 +0100 Subject: cpu-hotplug: replace lock_cpu_hotplug() with get_online_cpus() Replace all lock_cpu_hotplug/unlock_cpu_hotplug from the kernel and use get_online_cpus and put_online_cpus instead as it highlights the refcount semantics in these operations. The new API guarantees protection against the cpu-hotplug operation, but it doesn't guarantee serialized access to any of the local data structures. Hence the changes needs to be reviewed. In case of pseries_add_processor/pseries_remove_processor, use cpu_maps_update_begin()/cpu_maps_update_done() as we're modifying the cpu_present_map there. Signed-off-by: Gautham R Shenoy Signed-off-by: Ingo Molnar --- kernel/cpu.c | 10 +++++----- kernel/cpuset.c | 14 +++++++------- kernel/rcutorture.c | 6 +++--- kernel/sched.c | 4 ++-- kernel/stop_machine.c | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) (limited to 'kernel') diff --git a/kernel/cpu.c b/kernel/cpu.c index 656dc3fcbba..b0c4152995f 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -48,7 +48,7 @@ void __init cpu_hotplug_init(void) #ifdef CONFIG_HOTPLUG_CPU -void lock_cpu_hotplug(void) +void get_online_cpus(void) { might_sleep(); if (cpu_hotplug.active_writer == current) @@ -58,9 +58,9 @@ void lock_cpu_hotplug(void) mutex_unlock(&cpu_hotplug.lock); } -EXPORT_SYMBOL_GPL(lock_cpu_hotplug); +EXPORT_SYMBOL_GPL(get_online_cpus); -void unlock_cpu_hotplug(void) +void put_online_cpus(void) { if (cpu_hotplug.active_writer == current) return; @@ -73,7 +73,7 @@ void unlock_cpu_hotplug(void) mutex_unlock(&cpu_hotplug.lock); } -EXPORT_SYMBOL_GPL(unlock_cpu_hotplug); +EXPORT_SYMBOL_GPL(put_online_cpus); #endif /* CONFIG_HOTPLUG_CPU */ @@ -110,7 +110,7 @@ void cpu_maps_update_done(void) * non zero and goes to sleep again. * * However, this is very difficult to achieve in practice since - * lock_cpu_hotplug() not an api which is called all that often. + * get_online_cpus() not an api which is called all that often. * */ static void cpu_hotplug_begin(void) diff --git a/kernel/cpuset.c b/kernel/cpuset.c index 50f5dc46368..cfaf6419d81 100644 --- a/kernel/cpuset.c +++ b/kernel/cpuset.c @@ -537,10 +537,10 @@ static int cpusets_overlap(struct cpuset *a, struct cpuset *b) * * Call with cgroup_mutex held. May take callback_mutex during * call due to the kfifo_alloc() and kmalloc() calls. May nest - * a call to the lock_cpu_hotplug()/unlock_cpu_hotplug() pair. + * a call to the get_online_cpus()/put_online_cpus() pair. * Must not be called holding callback_mutex, because we must not - * call lock_cpu_hotplug() while holding callback_mutex. Elsewhere - * the kernel nests callback_mutex inside lock_cpu_hotplug() calls. + * call get_online_cpus() while holding callback_mutex. Elsewhere + * the kernel nests callback_mutex inside get_online_cpus() calls. * So the reverse nesting would risk an ABBA deadlock. * * The three key local variables below are: @@ -691,9 +691,9 @@ restart: rebuild: /* Have scheduler rebuild sched domains */ - lock_cpu_hotplug(); + get_online_cpus(); partition_sched_domains(ndoms, doms); - unlock_cpu_hotplug(); + put_online_cpus(); done: if (q && !IS_ERR(q)) @@ -1617,10 +1617,10 @@ static struct cgroup_subsys_state *cpuset_create( * * If the cpuset being removed has its flag 'sched_load_balance' * enabled, then simulate turning sched_load_balance off, which - * will call rebuild_sched_domains(). The lock_cpu_hotplug() + * will call rebuild_sched_domains(). The get_online_cpus() * call in rebuild_sched_domains() must not be made while holding * callback_mutex. Elsewhere the kernel nests callback_mutex inside - * lock_cpu_hotplug() calls. So the reverse nesting would risk an + * get_online_cpus() calls. So the reverse nesting would risk an * ABBA deadlock. */ diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c index c3e165c2318..fd599829e72 100644 --- a/kernel/rcutorture.c +++ b/kernel/rcutorture.c @@ -726,11 +726,11 @@ static void rcu_torture_shuffle_tasks(void) cpumask_t tmp_mask = CPU_MASK_ALL; int i; - lock_cpu_hotplug(); + get_online_cpus(); /* No point in shuffling if there is only one online CPU (ex: UP) */ if (num_online_cpus() == 1) { - unlock_cpu_hotplug(); + put_online_cpus(); return; } @@ -762,7 +762,7 @@ static void rcu_torture_shuffle_tasks(void) else rcu_idle_cpu--; - unlock_cpu_hotplug(); + put_online_cpus(); } /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the diff --git a/kernel/sched.c b/kernel/sched.c index 86e55a9c2de..672aa68bfea 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -7152,7 +7152,7 @@ static int load_balance_monitor(void *unused) int i, cpu, balanced = 1; /* Prevent cpus going down or coming up */ - lock_cpu_hotplug(); + get_online_cpus(); /* lockout changes to doms_cur[] array */ lock_doms_cur(); /* @@ -7186,7 +7186,7 @@ static int load_balance_monitor(void *unused) rcu_read_unlock(); unlock_doms_cur(); - unlock_cpu_hotplug(); + put_online_cpus(); if (!balanced) timeout = sysctl_sched_min_bal_int_shares; diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index 319821ef78a..51b5ee53571 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -203,13 +203,13 @@ int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu) int ret; /* No CPUs can come up or down during this. */ - lock_cpu_hotplug(); + get_online_cpus(); p = __stop_machine_run(fn, data, cpu); if (!IS_ERR(p)) ret = kthread_stop(p); else ret = PTR_ERR(p); - unlock_cpu_hotplug(); + put_online_cpus(); return ret; } -- cgit v1.2.3 From 95402b3829010fe1e208f44e4a158ccade88969a Mon Sep 17 00:00:00 2001 From: Gautham R Shenoy Date: Fri, 25 Jan 2008 21:08:02 +0100 Subject: cpu-hotplug: replace per-subsystem mutexes with get_online_cpus() This patch converts the known per-subsystem mutexes to get_online_cpus put_online_cpus. It also eliminates the CPU_LOCK_ACQUIRE and CPU_LOCK_RELEASE hotplug notification events. Signed-off-by: Gautham R Shenoy Signed-off-by: Ingo Molnar --- kernel/cpu.c | 4 ---- kernel/sched.c | 25 +++++++++---------------- kernel/workqueue.c | 35 +++++++++++++++-------------------- 3 files changed, 24 insertions(+), 40 deletions(-) (limited to 'kernel') diff --git a/kernel/cpu.c b/kernel/cpu.c index b0c4152995f..e0d3a4f56ec 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -218,7 +218,6 @@ static int _cpu_down(unsigned int cpu, int tasks_frozen) return -EINVAL; cpu_hotplug_begin(); - raw_notifier_call_chain(&cpu_chain, CPU_LOCK_ACQUIRE, hcpu); err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); if (err == NOTIFY_BAD) { @@ -271,7 +270,6 @@ out_thread: out_allowed: set_cpus_allowed(current, old_allowed); out_release: - raw_notifier_call_chain(&cpu_chain, CPU_LOCK_RELEASE, hcpu); cpu_hotplug_done(); return err; } @@ -302,7 +300,6 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) return -EINVAL; cpu_hotplug_begin(); - raw_notifier_call_chain(&cpu_chain, CPU_LOCK_ACQUIRE, hcpu); ret = __raw_notifier_call_chain(&cpu_chain, CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls); if (ret == NOTIFY_BAD) { @@ -326,7 +323,6 @@ out_notify: if (ret != 0) __raw_notifier_call_chain(&cpu_chain, CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL); - raw_notifier_call_chain(&cpu_chain, CPU_LOCK_RELEASE, hcpu); cpu_hotplug_done(); return ret; diff --git a/kernel/sched.c b/kernel/sched.c index 672aa68bfea..c0e2db683e2 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -439,7 +439,6 @@ struct rq { }; static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); -static DEFINE_MUTEX(sched_hotcpu_mutex); static inline void check_preempt_curr(struct rq *rq, struct task_struct *p) { @@ -4546,13 +4545,13 @@ long sched_setaffinity(pid_t pid, cpumask_t new_mask) struct task_struct *p; int retval; - mutex_lock(&sched_hotcpu_mutex); + get_online_cpus(); read_lock(&tasklist_lock); p = find_process_by_pid(pid); if (!p) { read_unlock(&tasklist_lock); - mutex_unlock(&sched_hotcpu_mutex); + put_online_cpus(); return -ESRCH; } @@ -4592,7 +4591,7 @@ long sched_setaffinity(pid_t pid, cpumask_t new_mask) } out_unlock: put_task_struct(p); - mutex_unlock(&sched_hotcpu_mutex); + put_online_cpus(); return retval; } @@ -4649,7 +4648,7 @@ long sched_getaffinity(pid_t pid, cpumask_t *mask) struct task_struct *p; int retval; - mutex_lock(&sched_hotcpu_mutex); + get_online_cpus(); read_lock(&tasklist_lock); retval = -ESRCH; @@ -4665,7 +4664,7 @@ long sched_getaffinity(pid_t pid, cpumask_t *mask) out_unlock: read_unlock(&tasklist_lock); - mutex_unlock(&sched_hotcpu_mutex); + put_online_cpus(); return retval; } @@ -5625,9 +5624,6 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) struct rq *rq; switch (action) { - case CPU_LOCK_ACQUIRE: - mutex_lock(&sched_hotcpu_mutex); - break; case CPU_UP_PREPARE: case CPU_UP_PREPARE_FROZEN: @@ -5697,9 +5693,6 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) spin_unlock_irq(&rq->lock); break; #endif - case CPU_LOCK_RELEASE: - mutex_unlock(&sched_hotcpu_mutex); - break; } return NOTIFY_OK; } @@ -6655,10 +6648,10 @@ static int arch_reinit_sched_domains(void) { int err; - mutex_lock(&sched_hotcpu_mutex); + get_online_cpus(); detach_destroy_domains(&cpu_online_map); err = arch_init_sched_domains(&cpu_online_map); - mutex_unlock(&sched_hotcpu_mutex); + put_online_cpus(); return err; } @@ -6769,12 +6762,12 @@ void __init sched_init_smp(void) { cpumask_t non_isolated_cpus; - mutex_lock(&sched_hotcpu_mutex); + get_online_cpus(); arch_init_sched_domains(&cpu_online_map); cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map); if (cpus_empty(non_isolated_cpus)) cpu_set(smp_processor_id(), non_isolated_cpus); - mutex_unlock(&sched_hotcpu_mutex); + put_online_cpus(); /* XXX: Theoretical race here - CPU may be hotplugged now */ hotcpu_notifier(update_sched_domains, 0); diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 8db0b597509..52db48e7f6e 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -67,9 +67,8 @@ struct workqueue_struct { #endif }; -/* All the per-cpu workqueues on the system, for hotplug cpu to add/remove - threads to each one as cpus come/go. */ -static DEFINE_MUTEX(workqueue_mutex); +/* Serializes the accesses to the list of workqueues. */ +static DEFINE_SPINLOCK(workqueue_lock); static LIST_HEAD(workqueues); static int singlethread_cpu __read_mostly; @@ -592,8 +591,6 @@ EXPORT_SYMBOL(schedule_delayed_work_on); * Returns zero on success. * Returns -ve errno on failure. * - * Appears to be racy against CPU hotplug. - * * schedule_on_each_cpu() is very slow. */ int schedule_on_each_cpu(work_func_t func) @@ -605,7 +602,7 @@ int schedule_on_each_cpu(work_func_t func) if (!works) return -ENOMEM; - preempt_disable(); /* CPU hotplug */ + get_online_cpus(); for_each_online_cpu(cpu) { struct work_struct *work = per_cpu_ptr(works, cpu); @@ -613,8 +610,8 @@ int schedule_on_each_cpu(work_func_t func) set_bit(WORK_STRUCT_PENDING, work_data_bits(work)); __queue_work(per_cpu_ptr(keventd_wq->cpu_wq, cpu), work); } - preempt_enable(); flush_workqueue(keventd_wq); + put_online_cpus(); free_percpu(works); return 0; } @@ -750,8 +747,10 @@ struct workqueue_struct *__create_workqueue_key(const char *name, err = create_workqueue_thread(cwq, singlethread_cpu); start_workqueue_thread(cwq, -1); } else { - mutex_lock(&workqueue_mutex); + get_online_cpus(); + spin_lock(&workqueue_lock); list_add(&wq->list, &workqueues); + spin_unlock(&workqueue_lock); for_each_possible_cpu(cpu) { cwq = init_cpu_workqueue(wq, cpu); @@ -760,7 +759,7 @@ struct workqueue_struct *__create_workqueue_key(const char *name, err = create_workqueue_thread(cwq, cpu); start_workqueue_thread(cwq, cpu); } - mutex_unlock(&workqueue_mutex); + put_online_cpus(); } if (err) { @@ -775,7 +774,7 @@ static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) { /* * Our caller is either destroy_workqueue() or CPU_DEAD, - * workqueue_mutex protects cwq->thread + * get_online_cpus() protects cwq->thread. */ if (cwq->thread == NULL) return; @@ -810,9 +809,11 @@ void destroy_workqueue(struct workqueue_struct *wq) struct cpu_workqueue_struct *cwq; int cpu; - mutex_lock(&workqueue_mutex); + get_online_cpus(); + spin_lock(&workqueue_lock); list_del(&wq->list); - mutex_unlock(&workqueue_mutex); + spin_unlock(&workqueue_lock); + put_online_cpus(); for_each_cpu_mask(cpu, *cpu_map) { cwq = per_cpu_ptr(wq->cpu_wq, cpu); @@ -835,13 +836,6 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, action &= ~CPU_TASKS_FROZEN; switch (action) { - case CPU_LOCK_ACQUIRE: - mutex_lock(&workqueue_mutex); - return NOTIFY_OK; - - case CPU_LOCK_RELEASE: - mutex_unlock(&workqueue_mutex); - return NOTIFY_OK; case CPU_UP_PREPARE: cpu_set(cpu, cpu_populated_map); @@ -854,7 +848,8 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, case CPU_UP_PREPARE: if (!create_workqueue_thread(cwq, cpu)) break; - printk(KERN_ERR "workqueue for %i failed\n", cpu); + printk(KERN_ERR "workqueue [%s] for %i failed\n", + wq->name, cpu); return NOTIFY_BAD; case CPU_ONLINE: -- cgit v1.2.3 From 82a1fcb90287052aabfa235e7ffc693ea003fe69 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:02 +0100 Subject: softlockup: automatically detect hung TASK_UNINTERRUPTIBLE tasks this patch extends the soft-lockup detector to automatically detect hung TASK_UNINTERRUPTIBLE tasks. Such hung tasks are printed the following way: ------------------> INFO: task prctl:3042 blocked for more than 120 seconds. "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message prctl D fd5e3793 0 3042 2997 f6050f38 00000046 00000001 fd5e3793 00000009 c06d8264 c06dae80 00000286 f6050f40 f6050f00 f7d34d90 f7d34fc8 c1e1be80 00000001 f6050000 00000000 f7e92d00 00000286 f6050f18 c0489d1a f6050f40 00006605 00000000 c0133a5b Call Trace: [] schedule_timeout+0x6d/0x8b [] schedule_timeout_uninterruptible+0x15/0x17 [] msleep+0x10/0x16 [] sys_prctl+0x30/0x1e2 [] sysenter_past_esp+0x5f/0xa5 ======================= 2 locks held by prctl/3042: #0: (&sb->s_type->i_mutex_key#5){--..}, at: [] do_fsync+0x38/0x7a #1: (jbd_handle){--..}, at: [] journal_start+0xc7/0xe9 <------------------ the current default timeout is 120 seconds. Such messages are printed up to 10 times per bootup. If the system has crashed already then the messages are not printed. if lockdep is enabled then all held locks are printed as well. this feature is a natural extension to the softlockup-detector (kernel locked up without scheduling) and to the NMI watchdog (kernel locked up with IRQs disabled). [ Gautham R Shenoy : CPU hotplug fixes. ] [ Andrew Morton : build warning fix. ] Signed-off-by: Ingo Molnar Signed-off-by: Arjan van de Ven --- kernel/fork.c | 5 +++ kernel/lockdep.c | 12 +++++- kernel/sched.c | 4 +- kernel/softlockup.c | 114 +++++++++++++++++++++++++++++++++++++++++++++++----- kernel/sysctl.c | 27 +++++++++++++ 5 files changed, 149 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 8dd8ff28100..09c0b90a69c 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1059,6 +1059,11 @@ static struct task_struct *copy_process(unsigned long clone_flags, p->prev_utime = cputime_zero; p->prev_stime = cputime_zero; +#ifdef CONFIG_DETECT_SOFTLOCKUP + p->last_switch_count = 0; + p->last_switch_timestamp = 0; +#endif + #ifdef CONFIG_TASK_XACCT p->rchar = 0; /* I/O counter: bytes read */ p->wchar = 0; /* I/O counter: bytes written */ diff --git a/kernel/lockdep.c b/kernel/lockdep.c index e2c07ece367..3574379f4d6 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -3206,7 +3206,11 @@ retry: EXPORT_SYMBOL_GPL(debug_show_all_locks); -void debug_show_held_locks(struct task_struct *task) +/* + * Careful: only use this function if you are sure that + * the task cannot run in parallel! + */ +void __debug_show_held_locks(struct task_struct *task) { if (unlikely(!debug_locks)) { printk("INFO: lockdep is turned off.\n"); @@ -3214,6 +3218,12 @@ void debug_show_held_locks(struct task_struct *task) } lockdep_print_held_locks(task); } +EXPORT_SYMBOL_GPL(__debug_show_held_locks); + +void debug_show_held_locks(struct task_struct *task) +{ + __debug_show_held_locks(task); +} EXPORT_SYMBOL_GPL(debug_show_held_locks); diff --git a/kernel/sched.c b/kernel/sched.c index c0e2db683e2..5b3d46574ee 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -4945,7 +4945,7 @@ out_unlock: static const char stat_nam[] = "RSDTtZX"; -static void show_task(struct task_struct *p) +void sched_show_task(struct task_struct *p) { unsigned long free = 0; unsigned state; @@ -4998,7 +4998,7 @@ void show_state_filter(unsigned long state_filter) */ touch_nmi_watchdog(); if (!state_filter || (p->state & state_filter)) - show_task(p); + sched_show_task(p); } while_each_thread(g, p); touch_all_softlockup_watchdogs(); diff --git a/kernel/softlockup.c b/kernel/softlockup.c index 11df812263c..02f0ad53444 100644 --- a/kernel/softlockup.c +++ b/kernel/softlockup.c @@ -8,6 +8,7 @@ */ #include #include +#include #include #include #include @@ -24,7 +25,7 @@ static DEFINE_PER_CPU(unsigned long, print_timestamp); static DEFINE_PER_CPU(struct task_struct *, watchdog_task); static int did_panic; -int softlockup_thresh = 10; +int softlockup_thresh = 60; static int softlock_panic(struct notifier_block *this, unsigned long event, void *ptr) @@ -45,7 +46,7 @@ static struct notifier_block panic_block = { */ static unsigned long get_timestamp(int this_cpu) { - return cpu_clock(this_cpu) >> 30; /* 2^30 ~= 10^9 */ + return cpu_clock(this_cpu) >> 30LL; /* 2^30 ~= 10^9 */ } void touch_softlockup_watchdog(void) @@ -100,11 +101,7 @@ void softlockup_tick(void) now = get_timestamp(this_cpu); - /* Wake up the high-prio watchdog task every second: */ - if (now > (touch_timestamp + 1)) - wake_up_process(per_cpu(watchdog_task, this_cpu)); - - /* Warn about unreasonable 10+ seconds delays: */ + /* Warn about unreasonable delays: */ if (now <= (touch_timestamp + softlockup_thresh)) return; @@ -121,12 +118,94 @@ void softlockup_tick(void) spin_unlock(&print_lock); } +/* + * Have a reasonable limit on the number of tasks checked: + */ +unsigned long sysctl_hung_task_check_count = 1024; + +/* + * Zero means infinite timeout - no checking done: + */ +unsigned long sysctl_hung_task_timeout_secs = 120; + +long sysctl_hung_task_warnings = 10; + +/* + * Only do the hung-tasks check on one CPU: + */ +static int check_cpu __read_mostly = -1; + +static void check_hung_task(struct task_struct *t, unsigned long now) +{ + unsigned long switch_count = t->nvcsw + t->nivcsw; + + if (t->flags & PF_FROZEN) + return; + + if (switch_count != t->last_switch_count || !t->last_switch_timestamp) { + t->last_switch_count = switch_count; + t->last_switch_timestamp = now; + return; + } + if ((long)(now - t->last_switch_timestamp) < + sysctl_hung_task_timeout_secs) + return; + if (sysctl_hung_task_warnings < 0) + return; + sysctl_hung_task_warnings--; + + /* + * Ok, the task did not get scheduled for more than 2 minutes, + * complain: + */ + printk(KERN_ERR "INFO: task %s:%d blocked for more than " + "%ld seconds.\n", t->comm, t->pid, + sysctl_hung_task_timeout_secs); + printk(KERN_ERR "\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\"" + " disables this message.\n"); + sched_show_task(t); + __debug_show_held_locks(t); + + t->last_switch_timestamp = now; + touch_nmi_watchdog(); +} + +/* + * Check whether a TASK_UNINTERRUPTIBLE does not get woken up for + * a really long time (120 seconds). If that happens, print out + * a warning. + */ +static void check_hung_uninterruptible_tasks(int this_cpu) +{ + int max_count = sysctl_hung_task_check_count; + unsigned long now = get_timestamp(this_cpu); + struct task_struct *g, *t; + + /* + * If the system crashed already then all bets are off, + * do not report extra hung tasks: + */ + if ((tainted & TAINT_DIE) || did_panic) + return; + + read_lock(&tasklist_lock); + do_each_thread(g, t) { + if (!--max_count) + break; + if (t->state & TASK_UNINTERRUPTIBLE) + check_hung_task(t, now); + } while_each_thread(g, t); + + read_unlock(&tasklist_lock); +} + /* * The watchdog thread - runs every second and touches the timestamp. */ static int watchdog(void *__bind_cpu) { struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; + int this_cpu = (long)__bind_cpu; sched_setscheduler(current, SCHED_FIFO, ¶m); @@ -135,13 +214,18 @@ static int watchdog(void *__bind_cpu) /* * Run briefly once per second to reset the softlockup timestamp. - * If this gets delayed for more than 10 seconds then the + * If this gets delayed for more than 60 seconds then the * debug-printout triggers in softlockup_tick(). */ while (!kthread_should_stop()) { - set_current_state(TASK_INTERRUPTIBLE); touch_softlockup_watchdog(); - schedule(); + msleep_interruptible(10000); + + if (this_cpu != check_cpu) + continue; + + if (sysctl_hung_task_timeout_secs) + check_hung_uninterruptible_tasks(this_cpu); } return 0; @@ -171,6 +255,7 @@ cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) break; case CPU_ONLINE: case CPU_ONLINE_FROZEN: + check_cpu = any_online_cpu(cpu_online_map); wake_up_process(per_cpu(watchdog_task, hotcpu)); break; #ifdef CONFIG_HOTPLUG_CPU @@ -181,6 +266,15 @@ cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) /* Unbind so it can run. Fall thru. */ kthread_bind(per_cpu(watchdog_task, hotcpu), any_online_cpu(cpu_online_map)); + case CPU_DOWN_PREPARE: + case CPU_DOWN_PREPARE_FROZEN: + if (hotcpu == check_cpu) { + cpumask_t temp_cpu_online_map = cpu_online_map; + + cpu_clear(hotcpu, temp_cpu_online_map); + check_cpu = any_online_cpu(temp_cpu_online_map); + } + break; case CPU_DEAD: case CPU_DEAD_FROZEN: p = per_cpu(watchdog_task, hotcpu); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index c95f3ed3447..96f31c1bc4f 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -753,6 +753,33 @@ static struct ctl_table kern_table[] = { .extra1 = &one, .extra2 = &sixty, }, + { + .ctl_name = CTL_UNNUMBERED, + .procname = "hung_task_check_count", + .data = &sysctl_hung_task_check_count, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .strategy = &sysctl_intvec, + }, + { + .ctl_name = CTL_UNNUMBERED, + .procname = "hung_task_timeout_secs", + .data = &sysctl_hung_task_timeout_secs, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .strategy = &sysctl_intvec, + }, + { + .ctl_name = CTL_UNNUMBERED, + .procname = "hung_task_warnings", + .data = &sysctl_hung_task_warnings, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .strategy = &sysctl_intvec, + }, #endif #ifdef CONFIG_COMPAT { -- cgit v1.2.3 From 63489e45e265f64c368882be1f01c42dec5d984c Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:03 +0100 Subject: sched: count # of queued RT tasks This patch adds accounting to keep track of the number of RT tasks running on a runqueue. This information will be used in later patches. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched.c | 1 + kernel/sched_rt.c | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 5b3d46574ee..9dd8d121eea 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -342,6 +342,7 @@ struct rt_rq { struct rt_prio_array active; int rt_load_balance_idx; struct list_head *rt_load_balance_head, *rt_load_balance_curr; + unsigned long rt_nr_running; }; /* diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index cefcd510514..e3d2ca8832a 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -26,6 +26,19 @@ static void update_curr_rt(struct rq *rq) cpuacct_charge(curr, delta_exec); } +static inline void inc_rt_tasks(struct task_struct *p, struct rq *rq) +{ + WARN_ON(!rt_task(p)); + rq->rt.rt_nr_running++; +} + +static inline void dec_rt_tasks(struct task_struct *p, struct rq *rq) +{ + WARN_ON(!rt_task(p)); + WARN_ON(!rq->rt.rt_nr_running); + rq->rt.rt_nr_running--; +} + static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup) { struct rt_prio_array *array = &rq->rt.active; @@ -33,6 +46,8 @@ static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup) list_add_tail(&p->run_list, array->queue + p->prio); __set_bit(p->prio, array->bitmap); inc_cpu_load(rq, p->se.load.weight); + + inc_rt_tasks(p, rq); } /* @@ -48,6 +63,8 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep) if (list_empty(array->queue + p->prio)) __clear_bit(p->prio, array->bitmap); dec_cpu_load(rq, p->se.load.weight); + + dec_rt_tasks(p, rq); } /* -- cgit v1.2.3 From 764a9d6fe4b52995c8aba277e3634385699354f4 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:04 +0100 Subject: sched: track highest prio task queued This patch adds accounting to each runqueue to keep track of the highest prio task queued on the run queue. We only care about RT tasks, so if the run queue does not contain any active RT tasks its priority will be considered MAX_RT_PRIO. This information will be used for later patches. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched.c | 3 +++ kernel/sched_rt.c | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 9dd8d121eea..6185fa080ec 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -343,6 +343,8 @@ struct rt_rq { int rt_load_balance_idx; struct list_head *rt_load_balance_head, *rt_load_balance_curr; unsigned long rt_nr_running; + /* highest queued rt task prio */ + int highest_prio; }; /* @@ -6864,6 +6866,7 @@ void __init sched_init(void) rq->cpu = i; rq->migration_thread = NULL; INIT_LIST_HEAD(&rq->migration_queue); + rq->rt.highest_prio = MAX_RT_PRIO; #endif atomic_set(&rq->nr_iowait, 0); diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index e3d2ca8832a..136c2857a04 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -30,6 +30,10 @@ static inline void inc_rt_tasks(struct task_struct *p, struct rq *rq) { WARN_ON(!rt_task(p)); rq->rt.rt_nr_running++; +#ifdef CONFIG_SMP + if (p->prio < rq->rt.highest_prio) + rq->rt.highest_prio = p->prio; +#endif /* CONFIG_SMP */ } static inline void dec_rt_tasks(struct task_struct *p, struct rq *rq) @@ -37,6 +41,20 @@ static inline void dec_rt_tasks(struct task_struct *p, struct rq *rq) WARN_ON(!rt_task(p)); WARN_ON(!rq->rt.rt_nr_running); rq->rt.rt_nr_running--; +#ifdef CONFIG_SMP + if (rq->rt.rt_nr_running) { + struct rt_prio_array *array; + + WARN_ON(p->prio < rq->rt.highest_prio); + if (p->prio == rq->rt.highest_prio) { + /* recalculate */ + array = &rq->rt.active; + rq->rt.highest_prio = + sched_find_first_bit(array->bitmap); + } /* otherwise leave rq->highest prio alone */ + } else + rq->rt.highest_prio = MAX_RT_PRIO; +#endif /* CONFIG_SMP */ } static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup) -- cgit v1.2.3 From e8fa136262e1121288bb93befe2295928ffd240d Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:05 +0100 Subject: sched: add RT task pushing This patch adds an algorithm to push extra RT tasks off a run queue to other CPU runqueues. When more than one RT task is added to a run queue, this algorithm takes an assertive approach to push the RT tasks that are not running onto other run queues that have lower priority. The way this works is that the highest RT task that is not running is looked at and we examine the runqueues on the CPUS for that tasks affinity mask. We find the runqueue with the lowest prio in the CPU affinity of the picked task, and if it is lower in prio than the picked task, we push the task onto that CPU runqueue. We continue pushing RT tasks off the current runqueue until we don't push any more. The algorithm stops when the next highest RT task can't preempt any other processes on other CPUS. TODO: The algorithm may stop when there are still RT tasks that can be migrated. Specifically, if the highest non running RT task CPU affinity is restricted to CPUs that are running higher priority tasks, there may be a lower priority task queued that has an affinity with a CPU that is running a lower priority task that it could be migrated to. This patch set does not address this issue. Note: checkpatch reveals two over 80 character instances. I'm not sure that breaking them up will help visually, so I left them as is. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched.c | 8 +- kernel/sched_rt.c | 225 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 231 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 6185fa080ec..97cab609fc3 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -1952,6 +1952,8 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev) prev_state = prev->state; finish_arch_switch(prev); finish_lock_switch(rq, prev); + schedule_tail_balance_rt(rq); + fire_sched_in_preempt_notifiers(current); if (mm) mmdrop(mm); @@ -2185,11 +2187,13 @@ static void double_rq_unlock(struct rq *rq1, struct rq *rq2) /* * double_lock_balance - lock the busiest runqueue, this_rq is locked already. */ -static void double_lock_balance(struct rq *this_rq, struct rq *busiest) +static int double_lock_balance(struct rq *this_rq, struct rq *busiest) __releases(this_rq->lock) __acquires(busiest->lock) __acquires(this_rq->lock) { + int ret = 0; + if (unlikely(!irqs_disabled())) { /* printk() doesn't work good under rq->lock */ spin_unlock(&this_rq->lock); @@ -2200,9 +2204,11 @@ static void double_lock_balance(struct rq *this_rq, struct rq *busiest) spin_unlock(&this_rq->lock); spin_lock(&busiest->lock); spin_lock(&this_rq->lock); + ret = 1; } else spin_lock(&busiest->lock); } + return ret; } /* diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 136c2857a04..7815e90b114 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -137,6 +137,227 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p) } #ifdef CONFIG_SMP +/* Only try algorithms three times */ +#define RT_MAX_TRIES 3 + +static int double_lock_balance(struct rq *this_rq, struct rq *busiest); +static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep); + +/* Return the second highest RT task, NULL otherwise */ +static struct task_struct *pick_next_highest_task_rt(struct rq *rq) +{ + struct rt_prio_array *array = &rq->rt.active; + struct task_struct *next; + struct list_head *queue; + int idx; + + assert_spin_locked(&rq->lock); + + if (likely(rq->rt.rt_nr_running < 2)) + return NULL; + + idx = sched_find_first_bit(array->bitmap); + if (unlikely(idx >= MAX_RT_PRIO)) { + WARN_ON(1); /* rt_nr_running is bad */ + return NULL; + } + + queue = array->queue + idx; + next = list_entry(queue->next, struct task_struct, run_list); + if (unlikely(next != rq->curr)) + return next; + + if (queue->next->next != queue) { + /* same prio task */ + next = list_entry(queue->next->next, struct task_struct, run_list); + return next; + } + + /* slower, but more flexible */ + idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1); + if (unlikely(idx >= MAX_RT_PRIO)) { + WARN_ON(1); /* rt_nr_running was 2 and above! */ + return NULL; + } + + queue = array->queue + idx; + next = list_entry(queue->next, struct task_struct, run_list); + + return next; +} + +static DEFINE_PER_CPU(cpumask_t, local_cpu_mask); + +/* Will lock the rq it finds */ +static struct rq *find_lock_lowest_rq(struct task_struct *task, + struct rq *this_rq) +{ + struct rq *lowest_rq = NULL; + int cpu; + int tries; + cpumask_t *cpu_mask = &__get_cpu_var(local_cpu_mask); + + cpus_and(*cpu_mask, cpu_online_map, task->cpus_allowed); + + for (tries = 0; tries < RT_MAX_TRIES; tries++) { + /* + * Scan each rq for the lowest prio. + */ + for_each_cpu_mask(cpu, *cpu_mask) { + struct rq *rq = &per_cpu(runqueues, cpu); + + if (cpu == this_rq->cpu) + continue; + + /* We look for lowest RT prio or non-rt CPU */ + if (rq->rt.highest_prio >= MAX_RT_PRIO) { + lowest_rq = rq; + break; + } + + /* no locking for now */ + if (rq->rt.highest_prio > task->prio && + (!lowest_rq || rq->rt.highest_prio > lowest_rq->rt.highest_prio)) { + lowest_rq = rq; + } + } + + if (!lowest_rq) + break; + + /* if the prio of this runqueue changed, try again */ + if (double_lock_balance(this_rq, lowest_rq)) { + /* + * We had to unlock the run queue. In + * the mean time, task could have + * migrated already or had its affinity changed. + * Also make sure that it wasn't scheduled on its rq. + */ + if (unlikely(task_rq(task) != this_rq || + !cpu_isset(lowest_rq->cpu, task->cpus_allowed) || + task_running(this_rq, task) || + !task->se.on_rq)) { + spin_unlock(&lowest_rq->lock); + lowest_rq = NULL; + break; + } + } + + /* If this rq is still suitable use it. */ + if (lowest_rq->rt.highest_prio > task->prio) + break; + + /* try again */ + spin_unlock(&lowest_rq->lock); + lowest_rq = NULL; + } + + return lowest_rq; +} + +/* + * If the current CPU has more than one RT task, see if the non + * running task can migrate over to a CPU that is running a task + * of lesser priority. + */ +static int push_rt_task(struct rq *this_rq) +{ + struct task_struct *next_task; + struct rq *lowest_rq; + int ret = 0; + int paranoid = RT_MAX_TRIES; + + assert_spin_locked(&this_rq->lock); + + next_task = pick_next_highest_task_rt(this_rq); + if (!next_task) + return 0; + + retry: + if (unlikely(next_task == this_rq->curr)) + return 0; + + /* + * It's possible that the next_task slipped in of + * higher priority than current. If that's the case + * just reschedule current. + */ + if (unlikely(next_task->prio < this_rq->curr->prio)) { + resched_task(this_rq->curr); + return 0; + } + + /* We might release this_rq lock */ + get_task_struct(next_task); + + /* find_lock_lowest_rq locks the rq if found */ + lowest_rq = find_lock_lowest_rq(next_task, this_rq); + if (!lowest_rq) { + struct task_struct *task; + /* + * find lock_lowest_rq releases this_rq->lock + * so it is possible that next_task has changed. + * If it has, then try again. + */ + task = pick_next_highest_task_rt(this_rq); + if (unlikely(task != next_task) && task && paranoid--) { + put_task_struct(next_task); + next_task = task; + goto retry; + } + goto out; + } + + assert_spin_locked(&lowest_rq->lock); + + deactivate_task(this_rq, next_task, 0); + set_task_cpu(next_task, lowest_rq->cpu); + activate_task(lowest_rq, next_task, 0); + + resched_task(lowest_rq->curr); + + spin_unlock(&lowest_rq->lock); + + ret = 1; +out: + put_task_struct(next_task); + + return ret; +} + +/* + * TODO: Currently we just use the second highest prio task on + * the queue, and stop when it can't migrate (or there's + * no more RT tasks). There may be a case where a lower + * priority RT task has a different affinity than the + * higher RT task. In this case the lower RT task could + * possibly be able to migrate where as the higher priority + * RT task could not. We currently ignore this issue. + * Enhancements are welcome! + */ +static void push_rt_tasks(struct rq *rq) +{ + /* push_rt_task will return true if it moved an RT */ + while (push_rt_task(rq)) + ; +} + +static void schedule_tail_balance_rt(struct rq *rq) +{ + /* + * If we have more than one rt_task queued, then + * see if we can push the other rt_tasks off to other CPUS. + * Note we may release the rq lock, and since + * the lock was owned by prev, we need to release it + * first via finish_lock_switch and then reaquire it here. + */ + if (unlikely(rq->rt.rt_nr_running > 1)) { + spin_lock_irq(&rq->lock); + push_rt_tasks(rq); + spin_unlock_irq(&rq->lock); + } +} + /* * Load-balancing iterator. Note: while the runqueue stays locked * during the whole iteration, the current task might be @@ -241,7 +462,9 @@ move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, return iter_move_one_task(this_rq, this_cpu, busiest, sd, idle, &rt_rq_iterator); } -#endif +#else /* CONFIG_SMP */ +# define schedule_tail_balance_rt(rq) do { } while (0) +#endif /* CONFIG_SMP */ static void task_tick_rt(struct rq *rq, struct task_struct *p) { -- cgit v1.2.3 From 4fd29176b7cd24909f8ceba2105cb3ae2857b90c Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:06 +0100 Subject: sched: add rt-overload tracking This patch adds an RT overload accounting system. When a runqueue has more than one RT task queued, it is marked as overloaded. That is that it is a candidate to have RT tasks pulled from it. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 7815e90b114..547f858b075 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -3,6 +3,38 @@ * policies) */ +#ifdef CONFIG_SMP +static cpumask_t rt_overload_mask; +static atomic_t rto_count; +static inline int rt_overloaded(void) +{ + return atomic_read(&rto_count); +} +static inline cpumask_t *rt_overload(void) +{ + return &rt_overload_mask; +} +static inline void rt_set_overload(struct rq *rq) +{ + cpu_set(rq->cpu, rt_overload_mask); + /* + * Make sure the mask is visible before we set + * the overload count. That is checked to determine + * if we should look at the mask. It would be a shame + * if we looked at the mask, but the mask was not + * updated yet. + */ + wmb(); + atomic_inc(&rto_count); +} +static inline void rt_clear_overload(struct rq *rq) +{ + /* the order here really doesn't matter */ + atomic_dec(&rto_count); + cpu_clear(rq->cpu, rt_overload_mask); +} +#endif /* CONFIG_SMP */ + /* * Update the current task's runtime statistics. Skip current tasks that * are not in our scheduling class. @@ -33,6 +65,8 @@ static inline void inc_rt_tasks(struct task_struct *p, struct rq *rq) #ifdef CONFIG_SMP if (p->prio < rq->rt.highest_prio) rq->rt.highest_prio = p->prio; + if (rq->rt.rt_nr_running > 1) + rt_set_overload(rq); #endif /* CONFIG_SMP */ } @@ -54,6 +88,8 @@ static inline void dec_rt_tasks(struct task_struct *p, struct rq *rq) } /* otherwise leave rq->highest prio alone */ } else rq->rt.highest_prio = MAX_RT_PRIO; + if (rq->rt.rt_nr_running < 2) + rt_clear_overload(rq); #endif /* CONFIG_SMP */ } -- cgit v1.2.3 From f65eda4f789168ba5ff3fa75546c29efeed19f58 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:07 +0100 Subject: sched: pull RT tasks from overloaded runqueues This patch adds the algorithm to pull tasks from RT overloaded runqueues. When a pull RT is initiated, all overloaded runqueues are examined for a RT task that is higher in prio than the highest prio task queued on the target runqueue. If another runqueue holds a RT task that is of higher prio than the highest prio task on the target runqueue is found it is pulled to the target runqueue. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched.c | 2 + kernel/sched_rt.c | 187 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 178 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 97cab609fc3..c9179710791 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -3721,6 +3721,8 @@ need_resched_nonpreemptible: switch_count = &prev->nvcsw; } + schedule_balance_rt(rq, prev); + if (unlikely(!rq->nr_running)) idle_balance(cpu, rq); diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 547f858b075..bacb32039e9 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -179,8 +179,17 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p) static int double_lock_balance(struct rq *this_rq, struct rq *busiest); static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep); +static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) +{ + if (!task_running(rq, p) && + (cpu < 0 || cpu_isset(cpu, p->cpus_allowed))) + return 1; + return 0; +} + /* Return the second highest RT task, NULL otherwise */ -static struct task_struct *pick_next_highest_task_rt(struct rq *rq) +static struct task_struct *pick_next_highest_task_rt(struct rq *rq, + int cpu) { struct rt_prio_array *array = &rq->rt.active; struct task_struct *next; @@ -199,26 +208,36 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq) } queue = array->queue + idx; + BUG_ON(list_empty(queue)); + next = list_entry(queue->next, struct task_struct, run_list); - if (unlikely(next != rq->curr)) - return next; + if (unlikely(pick_rt_task(rq, next, cpu))) + goto out; if (queue->next->next != queue) { /* same prio task */ next = list_entry(queue->next->next, struct task_struct, run_list); - return next; + if (pick_rt_task(rq, next, cpu)) + goto out; } + retry: /* slower, but more flexible */ idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1); - if (unlikely(idx >= MAX_RT_PRIO)) { - WARN_ON(1); /* rt_nr_running was 2 and above! */ + if (unlikely(idx >= MAX_RT_PRIO)) return NULL; - } queue = array->queue + idx; - next = list_entry(queue->next, struct task_struct, run_list); + BUG_ON(list_empty(queue)); + + list_for_each_entry(next, queue, run_list) { + if (pick_rt_task(rq, next, cpu)) + goto out; + } + + goto retry; + out: return next; } @@ -305,13 +324,15 @@ static int push_rt_task(struct rq *this_rq) assert_spin_locked(&this_rq->lock); - next_task = pick_next_highest_task_rt(this_rq); + next_task = pick_next_highest_task_rt(this_rq, -1); if (!next_task) return 0; retry: - if (unlikely(next_task == this_rq->curr)) + if (unlikely(next_task == this_rq->curr)) { + WARN_ON(1); return 0; + } /* * It's possible that the next_task slipped in of @@ -335,7 +356,7 @@ static int push_rt_task(struct rq *this_rq) * so it is possible that next_task has changed. * If it has, then try again. */ - task = pick_next_highest_task_rt(this_rq); + task = pick_next_highest_task_rt(this_rq, -1); if (unlikely(task != next_task) && task && paranoid--) { put_task_struct(next_task); next_task = task; @@ -378,6 +399,149 @@ static void push_rt_tasks(struct rq *rq) ; } +static int pull_rt_task(struct rq *this_rq) +{ + struct task_struct *next; + struct task_struct *p; + struct rq *src_rq; + cpumask_t *rto_cpumask; + int this_cpu = this_rq->cpu; + int cpu; + int ret = 0; + + assert_spin_locked(&this_rq->lock); + + /* + * If cpusets are used, and we have overlapping + * run queue cpusets, then this algorithm may not catch all. + * This is just the price you pay on trying to keep + * dirtying caches down on large SMP machines. + */ + if (likely(!rt_overloaded())) + return 0; + + next = pick_next_task_rt(this_rq); + + rto_cpumask = rt_overload(); + + for_each_cpu_mask(cpu, *rto_cpumask) { + if (this_cpu == cpu) + continue; + + src_rq = cpu_rq(cpu); + if (unlikely(src_rq->rt.rt_nr_running <= 1)) { + /* + * It is possible that overlapping cpusets + * will miss clearing a non overloaded runqueue. + * Clear it now. + */ + if (double_lock_balance(this_rq, src_rq)) { + /* unlocked our runqueue lock */ + struct task_struct *old_next = next; + next = pick_next_task_rt(this_rq); + if (next != old_next) + ret = 1; + } + if (likely(src_rq->rt.rt_nr_running <= 1)) + /* + * Small chance that this_rq->curr changed + * but it's really harmless here. + */ + rt_clear_overload(this_rq); + else + /* + * Heh, the src_rq is now overloaded, since + * we already have the src_rq lock, go straight + * to pulling tasks from it. + */ + goto try_pulling; + spin_unlock(&src_rq->lock); + continue; + } + + /* + * We can potentially drop this_rq's lock in + * double_lock_balance, and another CPU could + * steal our next task - hence we must cause + * the caller to recalculate the next task + * in that case: + */ + if (double_lock_balance(this_rq, src_rq)) { + struct task_struct *old_next = next; + next = pick_next_task_rt(this_rq); + if (next != old_next) + ret = 1; + } + + /* + * Are there still pullable RT tasks? + */ + if (src_rq->rt.rt_nr_running <= 1) { + spin_unlock(&src_rq->lock); + continue; + } + + try_pulling: + p = pick_next_highest_task_rt(src_rq, this_cpu); + + /* + * Do we have an RT task that preempts + * the to-be-scheduled task? + */ + if (p && (!next || (p->prio < next->prio))) { + WARN_ON(p == src_rq->curr); + WARN_ON(!p->se.on_rq); + + /* + * There's a chance that p is higher in priority + * than what's currently running on its cpu. + * This is just that p is wakeing up and hasn't + * had a chance to schedule. We only pull + * p if it is lower in priority than the + * current task on the run queue or + * this_rq next task is lower in prio than + * the current task on that rq. + */ + if (p->prio < src_rq->curr->prio || + (next && next->prio < src_rq->curr->prio)) + goto bail; + + ret = 1; + + deactivate_task(src_rq, p, 0); + set_task_cpu(p, this_cpu); + activate_task(this_rq, p, 0); + /* + * We continue with the search, just in + * case there's an even higher prio task + * in another runqueue. (low likelyhood + * but possible) + */ + + /* + * Update next so that we won't pick a task + * on another cpu with a priority lower (or equal) + * than the one we just picked. + */ + next = p; + + } + bail: + spin_unlock(&src_rq->lock); + } + + return ret; +} + +static void schedule_balance_rt(struct rq *rq, + struct task_struct *prev) +{ + /* Try to pull RT tasks here if we lower this rq's prio */ + if (unlikely(rt_task(prev)) && + rq->rt.highest_prio > prev->prio) + pull_rt_task(rq); +} + static void schedule_tail_balance_rt(struct rq *rq) { /* @@ -500,6 +664,7 @@ move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, } #else /* CONFIG_SMP */ # define schedule_tail_balance_rt(rq) do { } while (0) +# define schedule_balance_rt(rq, prev) do { } while (0) #endif /* CONFIG_SMP */ static void task_tick_rt(struct rq *rq, struct task_struct *p) -- cgit v1.2.3 From 4642dafdf93dc7d66ee33437b93a5e6b8cea20d2 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:07 +0100 Subject: sched: push RT tasks from overloaded CPUs This patch adds pushing of overloaded RT tasks from a runqueue that is having tasks (most likely RT tasks) added to the run queue. TODO: We don't cover the case of waking of new RT tasks (yet). Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched.c | 1 + kernel/sched_rt.c | 10 ++++++++++ 2 files changed, 11 insertions(+) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index c9179710791..357d3a084de 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -1710,6 +1710,7 @@ out_activate: out_running: p->state = TASK_RUNNING; + wakeup_balance_rt(rq, p); out: task_rq_unlock(rq, &flags); diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index bacb32039e9..d38a8a559aa 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -558,6 +558,15 @@ static void schedule_tail_balance_rt(struct rq *rq) } } + +static void wakeup_balance_rt(struct rq *rq, struct task_struct *p) +{ + if (unlikely(rt_task(p)) && + !task_running(rq, p) && + (p->prio >= rq->curr->prio)) + push_rt_tasks(rq); +} + /* * Load-balancing iterator. Note: while the runqueue stays locked * during the whole iteration, the current task might be @@ -665,6 +674,7 @@ move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, #else /* CONFIG_SMP */ # define schedule_tail_balance_rt(rq) do { } while (0) # define schedule_balance_rt(rq, prev) do { } while (0) +# define wakeup_balance_rt(rq, p) do { } while (0) #endif /* CONFIG_SMP */ static void task_tick_rt(struct rq *rq, struct task_struct *p) -- cgit v1.2.3 From c7a1e46aa9782a947cf2ed506245d43396dbf991 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:07 +0100 Subject: sched: disable standard balancer for RT tasks Since we now take an active approach to load balancing, we don't need to balance RT tasks via the normal task balancer. In fact, this code was found to pull RT tasks away from CPUS that the active movement performed, resulting in large latencies. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 95 +++---------------------------------------------------- 1 file changed, 4 insertions(+), 91 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index d38a8a559aa..c492fd2b2ee 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -567,109 +567,22 @@ static void wakeup_balance_rt(struct rq *rq, struct task_struct *p) push_rt_tasks(rq); } -/* - * Load-balancing iterator. Note: while the runqueue stays locked - * during the whole iteration, the current task might be - * dequeued so the iterator has to be dequeue-safe. Here we - * achieve that by always pre-iterating before returning - * the current task: - */ -static struct task_struct *load_balance_start_rt(void *arg) -{ - struct rq *rq = arg; - struct rt_prio_array *array = &rq->rt.active; - struct list_head *head, *curr; - struct task_struct *p; - int idx; - - idx = sched_find_first_bit(array->bitmap); - if (idx >= MAX_RT_PRIO) - return NULL; - - head = array->queue + idx; - curr = head->prev; - - p = list_entry(curr, struct task_struct, run_list); - - curr = curr->prev; - - rq->rt.rt_load_balance_idx = idx; - rq->rt.rt_load_balance_head = head; - rq->rt.rt_load_balance_curr = curr; - - return p; -} - -static struct task_struct *load_balance_next_rt(void *arg) -{ - struct rq *rq = arg; - struct rt_prio_array *array = &rq->rt.active; - struct list_head *head, *curr; - struct task_struct *p; - int idx; - - idx = rq->rt.rt_load_balance_idx; - head = rq->rt.rt_load_balance_head; - curr = rq->rt.rt_load_balance_curr; - - /* - * If we arrived back to the head again then - * iterate to the next queue (if any): - */ - if (unlikely(head == curr)) { - int next_idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1); - - if (next_idx >= MAX_RT_PRIO) - return NULL; - - idx = next_idx; - head = array->queue + idx; - curr = head->prev; - - rq->rt.rt_load_balance_idx = idx; - rq->rt.rt_load_balance_head = head; - } - - p = list_entry(curr, struct task_struct, run_list); - - curr = curr->prev; - - rq->rt.rt_load_balance_curr = curr; - - return p; -} - static unsigned long load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, unsigned long max_load_move, struct sched_domain *sd, enum cpu_idle_type idle, int *all_pinned, int *this_best_prio) { - struct rq_iterator rt_rq_iterator; - - rt_rq_iterator.start = load_balance_start_rt; - rt_rq_iterator.next = load_balance_next_rt; - /* pass 'busiest' rq argument into - * load_balance_[start|next]_rt iterators - */ - rt_rq_iterator.arg = busiest; - - return balance_tasks(this_rq, this_cpu, busiest, max_load_move, sd, - idle, all_pinned, this_best_prio, &rt_rq_iterator); + /* don't touch RT tasks */ + return 0; } static int move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, struct sched_domain *sd, enum cpu_idle_type idle) { - struct rq_iterator rt_rq_iterator; - - rt_rq_iterator.start = load_balance_start_rt; - rt_rq_iterator.next = load_balance_next_rt; - rt_rq_iterator.arg = busiest; - - return iter_move_one_task(this_rq, this_cpu, busiest, sd, idle, - &rt_rq_iterator); + /* don't touch RT tasks */ + return 0; } #else /* CONFIG_SMP */ # define schedule_tail_balance_rt(rq) do { } while (0) -- cgit v1.2.3 From 73fe6aae84400e2b475e2a1dc4e8592cd3ed6e69 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:07 +0100 Subject: sched: add RT-balance cpu-weight Some RT tasks (particularly kthreads) are bound to one specific CPU. It is fairly common for two or more bound tasks to get queued up at the same time. Consider, for instance, softirq_timer and softirq_sched. A timer goes off in an ISR which schedules softirq_thread to run at RT50. Then the timer handler determines that it's time to smp-rebalance the system so it schedules softirq_sched to run. So we are in a situation where we have two RT50 tasks queued, and the system will go into rt-overload condition to request other CPUs for help. This causes two problems in the current code: 1) If a high-priority bound task and a low-priority unbounded task queue up behind the running task, we will fail to ever relocate the unbounded task because we terminate the search on the first unmovable task. 2) We spend precious futile cycles in the fast-path trying to pull overloaded tasks over. It is therefore optimial to strive to avoid the overhead all together if we can cheaply detect the condition before overload even occurs. This patch tries to achieve this optimization by utilizing the hamming weight of the task->cpus_allowed mask. A weight of 1 indicates that the task cannot be migrated. We will then utilize this information to skip non-migratable tasks and to eliminate uncessary rebalance attempts. We introduce a per-rq variable to count the number of migratable tasks that are currently running. We only go into overload if we have more than one rt task, AND at least one of them is migratable. In addition, we introduce a per-task variable to cache the cpus_allowed weight, since the hamming calculation is probably relatively expensive. We only update the cached value when the mask is updated which should be relatively infrequent, especially compared to scheduling frequency in the fast path. Signed-off-by: Gregory Haskins Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/fork.c | 1 + kernel/sched.c | 9 ++++++++- kernel/sched_rt.c | 50 +++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 54 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 09c0b90a69c..930c51865ab 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1242,6 +1242,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, * parent's CPU). This avoids alot of nasty races. */ p->cpus_allowed = current->cpus_allowed; + p->nr_cpus_allowed = current->nr_cpus_allowed; if (unlikely(!cpu_isset(task_cpu(p), p->cpus_allowed) || !cpu_online(task_cpu(p)))) set_task_cpu(p, smp_processor_id()); diff --git a/kernel/sched.c b/kernel/sched.c index 357d3a084de..66e99b419b3 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -343,6 +343,7 @@ struct rt_rq { int rt_load_balance_idx; struct list_head *rt_load_balance_head, *rt_load_balance_curr; unsigned long rt_nr_running; + unsigned long rt_nr_migratory; /* highest queued rt task prio */ int highest_prio; }; @@ -5144,7 +5145,13 @@ int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask) goto out; } - p->cpus_allowed = new_mask; + if (p->sched_class->set_cpus_allowed) + p->sched_class->set_cpus_allowed(p, &new_mask); + else { + p->cpus_allowed = new_mask; + p->nr_cpus_allowed = cpus_weight(new_mask); + } + /* Can the task run on the task's current CPU? If so, we're done */ if (cpu_isset(task_cpu(p), new_mask)) goto out; diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index c492fd2b2ee..ae4995c09aa 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -33,6 +33,14 @@ static inline void rt_clear_overload(struct rq *rq) atomic_dec(&rto_count); cpu_clear(rq->cpu, rt_overload_mask); } + +static void update_rt_migration(struct rq *rq) +{ + if (rq->rt.rt_nr_migratory && (rq->rt.rt_nr_running > 1)) + rt_set_overload(rq); + else + rt_clear_overload(rq); +} #endif /* CONFIG_SMP */ /* @@ -65,8 +73,10 @@ static inline void inc_rt_tasks(struct task_struct *p, struct rq *rq) #ifdef CONFIG_SMP if (p->prio < rq->rt.highest_prio) rq->rt.highest_prio = p->prio; - if (rq->rt.rt_nr_running > 1) - rt_set_overload(rq); + if (p->nr_cpus_allowed > 1) + rq->rt.rt_nr_migratory++; + + update_rt_migration(rq); #endif /* CONFIG_SMP */ } @@ -88,8 +98,10 @@ static inline void dec_rt_tasks(struct task_struct *p, struct rq *rq) } /* otherwise leave rq->highest prio alone */ } else rq->rt.highest_prio = MAX_RT_PRIO; - if (rq->rt.rt_nr_running < 2) - rt_clear_overload(rq); + if (p->nr_cpus_allowed > 1) + rq->rt.rt_nr_migratory--; + + update_rt_migration(rq); #endif /* CONFIG_SMP */ } @@ -182,7 +194,8 @@ static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep); static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) { if (!task_running(rq, p) && - (cpu < 0 || cpu_isset(cpu, p->cpus_allowed))) + (cpu < 0 || cpu_isset(cpu, p->cpus_allowed)) && + (p->nr_cpus_allowed > 1)) return 1; return 0; } @@ -584,6 +597,32 @@ move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, /* don't touch RT tasks */ return 0; } +static void set_cpus_allowed_rt(struct task_struct *p, cpumask_t *new_mask) +{ + int weight = cpus_weight(*new_mask); + + BUG_ON(!rt_task(p)); + + /* + * Update the migration status of the RQ if we have an RT task + * which is running AND changing its weight value. + */ + if (p->se.on_rq && (weight != p->nr_cpus_allowed)) { + struct rq *rq = task_rq(p); + + if ((p->nr_cpus_allowed <= 1) && (weight > 1)) + rq->rt.rt_nr_migratory++; + else if((p->nr_cpus_allowed > 1) && (weight <= 1)) { + BUG_ON(!rq->rt.rt_nr_migratory); + rq->rt.rt_nr_migratory--; + } + + update_rt_migration(rq); + } + + p->cpus_allowed = *new_mask; + p->nr_cpus_allowed = weight; +} #else /* CONFIG_SMP */ # define schedule_tail_balance_rt(rq) do { } while (0) # define schedule_balance_rt(rq, prev) do { } while (0) @@ -637,6 +676,7 @@ const struct sched_class rt_sched_class = { #ifdef CONFIG_SMP .load_balance = load_balance_rt, .move_one_task = move_one_task_rt, + .set_cpus_allowed = set_cpus_allowed_rt, #endif .set_curr_task = set_curr_task_rt, -- cgit v1.2.3 From 697f0a487f294e634a342764472b79375bb3158a Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:09 +0100 Subject: sched: clean up this_rq use in kernel/sched_rt.c "this_rq" is normally used to denote the RQ on the current cpu (i.e. "cpu_rq(this_cpu)"). So clean up the usage of this_rq to be more consistent with the rest of the code. Signed-off-by: Gregory Haskins Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index ae4995c09aa..b788e35ffd3 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -328,21 +328,21 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, * running task can migrate over to a CPU that is running a task * of lesser priority. */ -static int push_rt_task(struct rq *this_rq) +static int push_rt_task(struct rq *rq) { struct task_struct *next_task; struct rq *lowest_rq; int ret = 0; int paranoid = RT_MAX_TRIES; - assert_spin_locked(&this_rq->lock); + assert_spin_locked(&rq->lock); - next_task = pick_next_highest_task_rt(this_rq, -1); + next_task = pick_next_highest_task_rt(rq, -1); if (!next_task) return 0; retry: - if (unlikely(next_task == this_rq->curr)) { + if (unlikely(next_task == rq->curr)) { WARN_ON(1); return 0; } @@ -352,24 +352,24 @@ static int push_rt_task(struct rq *this_rq) * higher priority than current. If that's the case * just reschedule current. */ - if (unlikely(next_task->prio < this_rq->curr->prio)) { - resched_task(this_rq->curr); + if (unlikely(next_task->prio < rq->curr->prio)) { + resched_task(rq->curr); return 0; } - /* We might release this_rq lock */ + /* We might release rq lock */ get_task_struct(next_task); /* find_lock_lowest_rq locks the rq if found */ - lowest_rq = find_lock_lowest_rq(next_task, this_rq); + lowest_rq = find_lock_lowest_rq(next_task, rq); if (!lowest_rq) { struct task_struct *task; /* - * find lock_lowest_rq releases this_rq->lock + * find lock_lowest_rq releases rq->lock * so it is possible that next_task has changed. * If it has, then try again. */ - task = pick_next_highest_task_rt(this_rq, -1); + task = pick_next_highest_task_rt(rq, -1); if (unlikely(task != next_task) && task && paranoid--) { put_task_struct(next_task); next_task = task; @@ -380,7 +380,7 @@ static int push_rt_task(struct rq *this_rq) assert_spin_locked(&lowest_rq->lock); - deactivate_task(this_rq, next_task, 0); + deactivate_task(rq, next_task, 0); set_task_cpu(next_task, lowest_rq->cpu); activate_task(lowest_rq, next_task, 0); -- cgit v1.2.3 From e7693a362ec84bb5b6fd441d8a8b4b9d568a7a0c Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:09 +0100 Subject: sched: de-SCHED_OTHER-ize the RT path The current wake-up code path tries to determine if it can optimize the wake-up to "this_cpu" by computing load calculations. The problem is that these calculations are only relevant to SCHED_OTHER tasks where load is king. For RT tasks, priority is king. So the load calculation is completely wasted bandwidth. Therefore, we create a new sched_class interface to help with pre-wakeup routing decisions and move the load calculation as a function of CFS task's class. Signed-off-by: Gregory Haskins Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched.c | 167 ++++++++---------------------------------------- kernel/sched_fair.c | 148 ++++++++++++++++++++++++++++++++++++++++++ kernel/sched_idletask.c | 9 +++ kernel/sched_rt.c | 10 +++ 4 files changed, 194 insertions(+), 140 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 66e99b419b3..3344ba776b9 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -960,6 +960,13 @@ static inline void dec_cpu_load(struct rq *rq, unsigned long load) update_load_sub(&rq->load, load); } +#ifdef CONFIG_SMP +static unsigned long source_load(int cpu, int type); +static unsigned long target_load(int cpu, int type); +static unsigned long cpu_avg_load_per_task(int cpu); +static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd); +#endif /* CONFIG_SMP */ + #include "sched_stats.h" #include "sched_idletask.c" #include "sched_fair.c" @@ -1118,7 +1125,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) /* * Is this task likely cache-hot: */ -static inline int +static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) { s64 delta; @@ -1343,7 +1350,7 @@ static unsigned long target_load(int cpu, int type) /* * Return the average load per task on the cpu's run queue */ -static inline unsigned long cpu_avg_load_per_task(int cpu) +static unsigned long cpu_avg_load_per_task(int cpu) { struct rq *rq = cpu_rq(cpu); unsigned long total = weighted_cpuload(cpu); @@ -1500,58 +1507,6 @@ static int sched_balance_self(int cpu, int flag) #endif /* CONFIG_SMP */ -/* - * wake_idle() will wake a task on an idle cpu if task->cpu is - * not idle and an idle cpu is available. The span of cpus to - * search starts with cpus closest then further out as needed, - * so we always favor a closer, idle cpu. - * - * Returns the CPU we should wake onto. - */ -#if defined(ARCH_HAS_SCHED_WAKE_IDLE) -static int wake_idle(int cpu, struct task_struct *p) -{ - cpumask_t tmp; - struct sched_domain *sd; - int i; - - /* - * If it is idle, then it is the best cpu to run this task. - * - * This cpu is also the best, if it has more than one task already. - * Siblings must be also busy(in most cases) as they didn't already - * pickup the extra load from this cpu and hence we need not check - * sibling runqueue info. This will avoid the checks and cache miss - * penalities associated with that. - */ - if (idle_cpu(cpu) || cpu_rq(cpu)->nr_running > 1) - return cpu; - - for_each_domain(cpu, sd) { - if (sd->flags & SD_WAKE_IDLE) { - cpus_and(tmp, sd->span, p->cpus_allowed); - for_each_cpu_mask(i, tmp) { - if (idle_cpu(i)) { - if (i != task_cpu(p)) { - schedstat_inc(p, - se.nr_wakeups_idle); - } - return i; - } - } - } else { - break; - } - } - return cpu; -} -#else -static inline int wake_idle(int cpu, struct task_struct *p) -{ - return cpu; -} -#endif - /*** * try_to_wake_up - wake up a thread * @p: the to-be-woken-up thread @@ -1573,8 +1528,6 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync) long old_state; struct rq *rq; #ifdef CONFIG_SMP - struct sched_domain *sd, *this_sd = NULL; - unsigned long load, this_load; int new_cpu; #endif @@ -1594,90 +1547,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync) if (unlikely(task_running(rq, p))) goto out_activate; - new_cpu = cpu; - - schedstat_inc(rq, ttwu_count); - if (cpu == this_cpu) { - schedstat_inc(rq, ttwu_local); - goto out_set_cpu; - } - - for_each_domain(this_cpu, sd) { - if (cpu_isset(cpu, sd->span)) { - schedstat_inc(sd, ttwu_wake_remote); - this_sd = sd; - break; - } - } - - if (unlikely(!cpu_isset(this_cpu, p->cpus_allowed))) - goto out_set_cpu; - - /* - * Check for affine wakeup and passive balancing possibilities. - */ - if (this_sd) { - int idx = this_sd->wake_idx; - unsigned int imbalance; - - imbalance = 100 + (this_sd->imbalance_pct - 100) / 2; - - load = source_load(cpu, idx); - this_load = target_load(this_cpu, idx); - - new_cpu = this_cpu; /* Wake to this CPU if we can */ - - if (this_sd->flags & SD_WAKE_AFFINE) { - unsigned long tl = this_load; - unsigned long tl_per_task; - - /* - * Attract cache-cold tasks on sync wakeups: - */ - if (sync && !task_hot(p, rq->clock, this_sd)) - goto out_set_cpu; - - schedstat_inc(p, se.nr_wakeups_affine_attempts); - tl_per_task = cpu_avg_load_per_task(this_cpu); - - /* - * If sync wakeup then subtract the (maximum possible) - * effect of the currently running task from the load - * of the current CPU: - */ - if (sync) - tl -= current->se.load.weight; - - if ((tl <= load && - tl + target_load(cpu, idx) <= tl_per_task) || - 100*(tl + p->se.load.weight) <= imbalance*load) { - /* - * This domain has SD_WAKE_AFFINE and - * p is cache cold in this domain, and - * there is no bad imbalance. - */ - schedstat_inc(this_sd, ttwu_move_affine); - schedstat_inc(p, se.nr_wakeups_affine); - goto out_set_cpu; - } - } - - /* - * Start passive balancing when half the imbalance_pct - * limit is reached. - */ - if (this_sd->flags & SD_WAKE_BALANCE) { - if (imbalance*this_load <= 100*load) { - schedstat_inc(this_sd, ttwu_move_balance); - schedstat_inc(p, se.nr_wakeups_passive); - goto out_set_cpu; - } - } - } - - new_cpu = cpu; /* Could not wake to this_cpu. Wake to cpu instead */ -out_set_cpu: - new_cpu = wake_idle(new_cpu, p); + new_cpu = p->sched_class->select_task_rq(p, sync); if (new_cpu != cpu) { set_task_cpu(p, new_cpu); task_rq_unlock(rq, &flags); @@ -1693,6 +1563,23 @@ out_set_cpu: cpu = task_cpu(p); } +#ifdef CONFIG_SCHEDSTATS + schedstat_inc(rq, ttwu_count); + if (cpu == this_cpu) + schedstat_inc(rq, ttwu_local); + else { + struct sched_domain *sd; + for_each_domain(this_cpu, sd) { + if (cpu_isset(cpu, sd->span)) { + schedstat_inc(sd, ttwu_wake_remote); + break; + } + } + } + +#endif + + out_activate: #endif /* CONFIG_SMP */ schedstat_inc(p, se.nr_wakeups); diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 5c208e090ae..f881fc5e035 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -860,6 +860,151 @@ static void yield_task_fair(struct rq *rq) se->vruntime = rightmost->vruntime + 1; } +/* + * wake_idle() will wake a task on an idle cpu if task->cpu is + * not idle and an idle cpu is available. The span of cpus to + * search starts with cpus closest then further out as needed, + * so we always favor a closer, idle cpu. + * + * Returns the CPU we should wake onto. + */ +#if defined(ARCH_HAS_SCHED_WAKE_IDLE) +static int wake_idle(int cpu, struct task_struct *p) +{ + cpumask_t tmp; + struct sched_domain *sd; + int i; + + /* + * If it is idle, then it is the best cpu to run this task. + * + * This cpu is also the best, if it has more than one task already. + * Siblings must be also busy(in most cases) as they didn't already + * pickup the extra load from this cpu and hence we need not check + * sibling runqueue info. This will avoid the checks and cache miss + * penalities associated with that. + */ + if (idle_cpu(cpu) || cpu_rq(cpu)->nr_running > 1) + return cpu; + + for_each_domain(cpu, sd) { + if (sd->flags & SD_WAKE_IDLE) { + cpus_and(tmp, sd->span, p->cpus_allowed); + for_each_cpu_mask(i, tmp) { + if (idle_cpu(i)) { + if (i != task_cpu(p)) { + schedstat_inc(p, + se.nr_wakeups_idle); + } + return i; + } + } + } else { + break; + } + } + return cpu; +} +#else +static inline int wake_idle(int cpu, struct task_struct *p) +{ + return cpu; +} +#endif + +#ifdef CONFIG_SMP +static int select_task_rq_fair(struct task_struct *p, int sync) +{ + int cpu, this_cpu; + struct rq *rq; + struct sched_domain *sd, *this_sd = NULL; + int new_cpu; + + cpu = task_cpu(p); + rq = task_rq(p); + this_cpu = smp_processor_id(); + new_cpu = cpu; + + for_each_domain(this_cpu, sd) { + if (cpu_isset(cpu, sd->span)) { + this_sd = sd; + break; + } + } + + if (unlikely(!cpu_isset(this_cpu, p->cpus_allowed))) + goto out_set_cpu; + + /* + * Check for affine wakeup and passive balancing possibilities. + */ + if (this_sd) { + int idx = this_sd->wake_idx; + unsigned int imbalance; + unsigned long load, this_load; + + imbalance = 100 + (this_sd->imbalance_pct - 100) / 2; + + load = source_load(cpu, idx); + this_load = target_load(this_cpu, idx); + + new_cpu = this_cpu; /* Wake to this CPU if we can */ + + if (this_sd->flags & SD_WAKE_AFFINE) { + unsigned long tl = this_load; + unsigned long tl_per_task; + + /* + * Attract cache-cold tasks on sync wakeups: + */ + if (sync && !task_hot(p, rq->clock, this_sd)) + goto out_set_cpu; + + schedstat_inc(p, se.nr_wakeups_affine_attempts); + tl_per_task = cpu_avg_load_per_task(this_cpu); + + /* + * If sync wakeup then subtract the (maximum possible) + * effect of the currently running task from the load + * of the current CPU: + */ + if (sync) + tl -= current->se.load.weight; + + if ((tl <= load && + tl + target_load(cpu, idx) <= tl_per_task) || + 100*(tl + p->se.load.weight) <= imbalance*load) { + /* + * This domain has SD_WAKE_AFFINE and + * p is cache cold in this domain, and + * there is no bad imbalance. + */ + schedstat_inc(this_sd, ttwu_move_affine); + schedstat_inc(p, se.nr_wakeups_affine); + goto out_set_cpu; + } + } + + /* + * Start passive balancing when half the imbalance_pct + * limit is reached. + */ + if (this_sd->flags & SD_WAKE_BALANCE) { + if (imbalance*this_load <= 100*load) { + schedstat_inc(this_sd, ttwu_move_balance); + schedstat_inc(p, se.nr_wakeups_passive); + goto out_set_cpu; + } + } + } + + new_cpu = cpu; /* Could not wake to this_cpu. Wake to cpu instead */ +out_set_cpu: + return wake_idle(new_cpu, p); +} +#endif /* CONFIG_SMP */ + + /* * Preempt the current task with a newly woken task if needed: */ @@ -1153,6 +1298,9 @@ static const struct sched_class fair_sched_class = { .enqueue_task = enqueue_task_fair, .dequeue_task = dequeue_task_fair, .yield_task = yield_task_fair, +#ifdef CONFIG_SMP + .select_task_rq = select_task_rq_fair, +#endif /* CONFIG_SMP */ .check_preempt_curr = check_preempt_wakeup, diff --git a/kernel/sched_idletask.c b/kernel/sched_idletask.c index bf9c25c15b8..ca5374860ae 100644 --- a/kernel/sched_idletask.c +++ b/kernel/sched_idletask.c @@ -5,6 +5,12 @@ * handled in sched_fair.c) */ +#ifdef CONFIG_SMP +static int select_task_rq_idle(struct task_struct *p, int sync) +{ + return task_cpu(p); /* IDLE tasks as never migrated */ +} +#endif /* CONFIG_SMP */ /* * Idle tasks are unconditionally rescheduled: */ @@ -72,6 +78,9 @@ const struct sched_class idle_sched_class = { /* dequeue is not valid, we print a debug message there: */ .dequeue_task = dequeue_task_idle, +#ifdef CONFIG_SMP + .select_task_rq = select_task_rq_idle, +#endif /* CONFIG_SMP */ .check_preempt_curr = check_preempt_curr_idle, diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index b788e35ffd3..5de1aebdbd1 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -150,6 +150,13 @@ yield_task_rt(struct rq *rq) requeue_task_rt(rq, rq->curr); } +#ifdef CONFIG_SMP +static int select_task_rq_rt(struct task_struct *p, int sync) +{ + return task_cpu(p); +} +#endif /* CONFIG_SMP */ + /* * Preempt the current task with a newly woken task if needed: */ @@ -667,6 +674,9 @@ const struct sched_class rt_sched_class = { .enqueue_task = enqueue_task_rt, .dequeue_task = dequeue_task_rt, .yield_task = yield_task_rt, +#ifdef CONFIG_SMP + .select_task_rq = select_task_rq_rt, +#endif /* CONFIG_SMP */ .check_preempt_curr = check_preempt_curr_rt, -- cgit v1.2.3 From 07b4032c9e505e2a1fbe7703aff64a153c3249be Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:10 +0100 Subject: sched: break out search for RT tasks Isolate the search logic into a function so that it can be used later in places other than find_locked_lowest_rq(). Signed-off-by: Gregory Haskins Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 66 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 5de1aebdbd1..ffd02720b58 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -263,54 +263,66 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, static DEFINE_PER_CPU(cpumask_t, local_cpu_mask); -/* Will lock the rq it finds */ -static struct rq *find_lock_lowest_rq(struct task_struct *task, - struct rq *this_rq) +static int find_lowest_rq(struct task_struct *task) { - struct rq *lowest_rq = NULL; int cpu; - int tries; cpumask_t *cpu_mask = &__get_cpu_var(local_cpu_mask); + struct rq *lowest_rq = NULL; cpus_and(*cpu_mask, cpu_online_map, task->cpus_allowed); - for (tries = 0; tries < RT_MAX_TRIES; tries++) { - /* - * Scan each rq for the lowest prio. - */ - for_each_cpu_mask(cpu, *cpu_mask) { - struct rq *rq = &per_cpu(runqueues, cpu); + /* + * Scan each rq for the lowest prio. + */ + for_each_cpu_mask(cpu, *cpu_mask) { + struct rq *rq = cpu_rq(cpu); - if (cpu == this_rq->cpu) - continue; + if (cpu == rq->cpu) + continue; - /* We look for lowest RT prio or non-rt CPU */ - if (rq->rt.highest_prio >= MAX_RT_PRIO) { - lowest_rq = rq; - break; - } + /* We look for lowest RT prio or non-rt CPU */ + if (rq->rt.highest_prio >= MAX_RT_PRIO) { + lowest_rq = rq; + break; + } - /* no locking for now */ - if (rq->rt.highest_prio > task->prio && - (!lowest_rq || rq->rt.highest_prio > lowest_rq->rt.highest_prio)) { - lowest_rq = rq; - } + /* no locking for now */ + if (rq->rt.highest_prio > task->prio && + (!lowest_rq || rq->rt.highest_prio > lowest_rq->rt.highest_prio)) { + lowest_rq = rq; } + } + + return lowest_rq ? lowest_rq->cpu : -1; +} + +/* Will lock the rq it finds */ +static struct rq *find_lock_lowest_rq(struct task_struct *task, + struct rq *rq) +{ + struct rq *lowest_rq = NULL; + int cpu; + int tries; - if (!lowest_rq) + for (tries = 0; tries < RT_MAX_TRIES; tries++) { + cpu = find_lowest_rq(task); + + if (cpu == -1) break; + lowest_rq = cpu_rq(cpu); + /* if the prio of this runqueue changed, try again */ - if (double_lock_balance(this_rq, lowest_rq)) { + if (double_lock_balance(rq, lowest_rq)) { /* * We had to unlock the run queue. In * the mean time, task could have * migrated already or had its affinity changed. * Also make sure that it wasn't scheduled on its rq. */ - if (unlikely(task_rq(task) != this_rq || + if (unlikely(task_rq(task) != rq || !cpu_isset(lowest_rq->cpu, task->cpus_allowed) || - task_running(this_rq, task) || + task_running(rq, task) || !task->se.on_rq)) { spin_unlock(&lowest_rq->lock); lowest_rq = NULL; -- cgit v1.2.3 From 2de0b4639f4b1b6bfe31f795e5855f041f177170 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:10 +0100 Subject: sched: RT balancing: include current CPU It doesn't hurt if we allow the current CPU to be included in the search. We will just simply skip it later if the current CPU turns out to be the lowest. We will use this later in the series Signed-off-by: Gregory Haskins Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index ffd02720b58..95f36f61ae1 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -277,9 +277,6 @@ static int find_lowest_rq(struct task_struct *task) for_each_cpu_mask(cpu, *cpu_mask) { struct rq *rq = cpu_rq(cpu); - if (cpu == rq->cpu) - continue; - /* We look for lowest RT prio or non-rt CPU */ if (rq->rt.highest_prio >= MAX_RT_PRIO) { lowest_rq = rq; @@ -307,7 +304,7 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, for (tries = 0; tries < RT_MAX_TRIES; tries++) { cpu = find_lowest_rq(task); - if (cpu == -1) + if ((cpu == -1) || (cpu == rq->cpu)) break; lowest_rq = cpu_rq(cpu); -- cgit v1.2.3 From 318e0893ce3f524ca045f9fd9dfd567c0a6f9446 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:10 +0100 Subject: sched: pre-route RT tasks on wakeup In the original patch series that Steven Rostedt and I worked on together, we both took different approaches to low-priority wakeup path. I utilized "pre-routing" (push the task away to a less important RQ before activating) approach, while Steve utilized a "post-routing" approach. The advantage of my approach is that you avoid the overhead of a wasted activate/deactivate cycle and peripherally related burdens. The advantage of Steve's method is that it neatly solves an issue preventing a "pull" optimization from being deployed. In the end, we ended up deploying Steve's idea. But it later dawned on me that we could get the best of both worlds by deploying both ideas together, albeit slightly modified. The idea is simple: Use a "light-weight" lookup for pre-routing, since we only need to approximate a good home for the task. And we also retain the post-routing push logic to clean up any inaccuracies caused by a condition of "priority mistargeting" caused by the lightweight lookup. Most of the time, the pre-routing should work and yield lower overhead. In the cases where it doesnt, the post-router will bat cleanup. Signed-off-by: Gregory Haskins Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 95f36f61ae1..ac7d0678645 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -151,8 +151,27 @@ yield_task_rt(struct rq *rq) } #ifdef CONFIG_SMP +static int find_lowest_rq(struct task_struct *task); + static int select_task_rq_rt(struct task_struct *p, int sync) { + struct rq *rq = task_rq(p); + + /* + * If the task will not preempt the RQ, try to find a better RQ + * before we even activate the task + */ + if ((p->prio >= rq->rt.highest_prio) + && (p->nr_cpus_allowed > 1)) { + int cpu = find_lowest_rq(p); + + return (cpu == -1) ? task_cpu(p) : cpu; + } + + /* + * Otherwise, just let it ride on the affined RQ and the + * post-schedule router will push the preempted task away + */ return task_cpu(p); } #endif /* CONFIG_SMP */ -- cgit v1.2.3 From 6e1254d2c41215da27025add8900ed187bca121d Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:11 +0100 Subject: sched: optimize RT affinity The current code base assumes a relatively flat CPU/core topology and will route RT tasks to any CPU fairly equally. In the real world, there are various toplogies and affinities that govern where a task is best suited to run with the smallest amount of overhead. NUMA and multi-core CPUs are prime examples of topologies that can impact cache performance. Fortunately, linux is already structured to represent these topologies via the sched_domains interface. So we change our RT router to consult a combination of topology and affinity policy to best place tasks during migration. Signed-off-by: Gregory Haskins Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 100 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index ac7d0678645..a9d7d440816 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -281,35 +281,111 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, } static DEFINE_PER_CPU(cpumask_t, local_cpu_mask); +static DEFINE_PER_CPU(cpumask_t, valid_cpu_mask); -static int find_lowest_rq(struct task_struct *task) +static int find_lowest_cpus(struct task_struct *task, cpumask_t *lowest_mask) { - int cpu; - cpumask_t *cpu_mask = &__get_cpu_var(local_cpu_mask); - struct rq *lowest_rq = NULL; + int cpu; + cpumask_t *valid_mask = &__get_cpu_var(valid_cpu_mask); + int lowest_prio = -1; + int ret = 0; - cpus_and(*cpu_mask, cpu_online_map, task->cpus_allowed); + cpus_clear(*lowest_mask); + cpus_and(*valid_mask, cpu_online_map, task->cpus_allowed); /* * Scan each rq for the lowest prio. */ - for_each_cpu_mask(cpu, *cpu_mask) { + for_each_cpu_mask(cpu, *valid_mask) { struct rq *rq = cpu_rq(cpu); /* We look for lowest RT prio or non-rt CPU */ if (rq->rt.highest_prio >= MAX_RT_PRIO) { - lowest_rq = rq; - break; + if (ret) + cpus_clear(*lowest_mask); + cpu_set(rq->cpu, *lowest_mask); + return 1; } /* no locking for now */ - if (rq->rt.highest_prio > task->prio && - (!lowest_rq || rq->rt.highest_prio > lowest_rq->rt.highest_prio)) { - lowest_rq = rq; + if ((rq->rt.highest_prio > task->prio) + && (rq->rt.highest_prio >= lowest_prio)) { + if (rq->rt.highest_prio > lowest_prio) { + /* new low - clear old data */ + lowest_prio = rq->rt.highest_prio; + cpus_clear(*lowest_mask); + } + cpu_set(rq->cpu, *lowest_mask); + ret = 1; + } + } + + return ret; +} + +static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask) +{ + int first; + + /* "this_cpu" is cheaper to preempt than a remote processor */ + if ((this_cpu != -1) && cpu_isset(this_cpu, *mask)) + return this_cpu; + + first = first_cpu(*mask); + if (first != NR_CPUS) + return first; + + return -1; +} + +static int find_lowest_rq(struct task_struct *task) +{ + struct sched_domain *sd; + cpumask_t *lowest_mask = &__get_cpu_var(local_cpu_mask); + int this_cpu = smp_processor_id(); + int cpu = task_cpu(task); + + if (!find_lowest_cpus(task, lowest_mask)) + return -1; + + /* + * At this point we have built a mask of cpus representing the + * lowest priority tasks in the system. Now we want to elect + * the best one based on our affinity and topology. + * + * We prioritize the last cpu that the task executed on since + * it is most likely cache-hot in that location. + */ + if (cpu_isset(cpu, *lowest_mask)) + return cpu; + + /* + * Otherwise, we consult the sched_domains span maps to figure + * out which cpu is logically closest to our hot cache data. + */ + if (this_cpu == cpu) + this_cpu = -1; /* Skip this_cpu opt if the same */ + + for_each_domain(cpu, sd) { + if (sd->flags & SD_WAKE_AFFINE) { + cpumask_t domain_mask; + int best_cpu; + + cpus_and(domain_mask, sd->span, *lowest_mask); + + best_cpu = pick_optimal_cpu(this_cpu, + &domain_mask); + if (best_cpu != -1) + return best_cpu; } } - return lowest_rq ? lowest_rq->cpu : -1; + /* + * And finally, if there were no matches within the domains + * just give the caller *something* to work with from the compatible + * locations. + */ + return pick_optimal_cpu(this_cpu, lowest_mask); } /* Will lock the rq it finds */ -- cgit v1.2.3 From a22d7fc187ed996b66d8439db27b2303f79a8e7b Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:12 +0100 Subject: sched: wake-balance fixes We have logic to detect whether the system has migratable tasks, but we are not using it when deciding whether to push tasks away. So we add support for considering this new information. Signed-off-by: Gregory Haskins Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched.c | 2 ++ kernel/sched_rt.c | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 3344ba776b9..c591abd9ca3 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -346,6 +346,7 @@ struct rt_rq { unsigned long rt_nr_migratory; /* highest queued rt task prio */ int highest_prio; + int overloaded; }; /* @@ -6770,6 +6771,7 @@ void __init sched_init(void) rq->migration_thread = NULL; INIT_LIST_HEAD(&rq->migration_queue); rq->rt.highest_prio = MAX_RT_PRIO; + rq->rt.overloaded = 0; #endif atomic_set(&rq->nr_iowait, 0); diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index a9d7d440816..87d7b3ff386 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -16,6 +16,7 @@ static inline cpumask_t *rt_overload(void) } static inline void rt_set_overload(struct rq *rq) { + rq->rt.overloaded = 1; cpu_set(rq->cpu, rt_overload_mask); /* * Make sure the mask is visible before we set @@ -32,6 +33,7 @@ static inline void rt_clear_overload(struct rq *rq) /* the order here really doesn't matter */ atomic_dec(&rto_count); cpu_clear(rq->cpu, rt_overload_mask); + rq->rt.overloaded = 0; } static void update_rt_migration(struct rq *rq) @@ -448,6 +450,9 @@ static int push_rt_task(struct rq *rq) assert_spin_locked(&rq->lock); + if (!rq->rt.overloaded) + return 0; + next_task = pick_next_highest_task_rt(rq, -1); if (!next_task) return 0; @@ -675,7 +680,7 @@ static void schedule_tail_balance_rt(struct rq *rq) * the lock was owned by prev, we need to release it * first via finish_lock_switch and then reaquire it here. */ - if (unlikely(rq->rt.rt_nr_running > 1)) { + if (unlikely(rq->rt.overloaded)) { spin_lock_irq(&rq->lock); push_rt_tasks(rq); spin_unlock_irq(&rq->lock); @@ -687,7 +692,8 @@ static void wakeup_balance_rt(struct rq *rq, struct task_struct *p) { if (unlikely(rt_task(p)) && !task_running(rq, p) && - (p->prio >= rq->curr->prio)) + (p->prio >= rq->rt.highest_prio) && + rq->rt.overloaded) push_rt_tasks(rq); } -- cgit v1.2.3 From e1f47d891c0f00769d6d40ac5740f943e998d089 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:12 +0100 Subject: sched: RT-balance, avoid overloading This patch changes the searching for a run queue by a waking RT task to try to pick another runqueue if the currently running task is an RT task. The reason is that RT tasks behave different than normal tasks. Preempting a normal task to run a RT task to keep its cache hot is fine, because the preempted non-RT task may wait on that same runqueue to run again unless the migration thread comes along and pulls it off. RT tasks behave differently. If one is preempted, it makes an active effort to continue to run. So by having a high priority task preempt a lower priority RT task, that lower RT task will then quickly try to run on another runqueue. This will cause that lower RT task to replace its nice hot cache (and TLB) with a completely cold one. This is for the hope that the new high priority RT task will keep its cache hot. Remeber that this high priority RT task was just woken up. So it may likely have been sleeping for several milliseconds, and will end up with a cold cache anyway. RT tasks run till they voluntarily stop, or are preempted by a higher priority task. This means that it is unlikely that the woken RT task will have a hot cache to wake up to. So pushing off a lower RT task is just killing its cache for no good reason. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 87d7b3ff386..9becc3710b6 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -160,11 +160,23 @@ static int select_task_rq_rt(struct task_struct *p, int sync) struct rq *rq = task_rq(p); /* - * If the task will not preempt the RQ, try to find a better RQ - * before we even activate the task + * If the current task is an RT task, then + * try to see if we can wake this RT task up on another + * runqueue. Otherwise simply start this RT task + * on its current runqueue. + * + * We want to avoid overloading runqueues. Even if + * the RT task is of higher priority than the current RT task. + * RT tasks behave differently than other tasks. If + * one gets preempted, we try to push it off to another queue. + * So trying to keep a preempting RT task on the same + * cache hot CPU will force the running RT task to + * a cold CPU. So we waste all the cache for the lower + * RT task in hopes of saving some of a RT task + * that is just being woken and probably will have + * cold cache anyway. */ - if ((p->prio >= rq->rt.highest_prio) - && (p->nr_cpus_allowed > 1)) { + if (unlikely(rt_task(rq->curr))) { int cpu = find_lowest_rq(p); return (cpu == -1) ? task_cpu(p) : cpu; -- cgit v1.2.3 From 17b3279b48835eb522d842eae16f541da3729c8a Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:13 +0100 Subject: sched: break out early if RT task cannot be migrated We don't need to bother searching if the task cannot be migrated Signed-off-by: Gregory Haskins Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 9becc3710b6..72c81322fb9 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -176,7 +176,8 @@ static int select_task_rq_rt(struct task_struct *p, int sync) * that is just being woken and probably will have * cold cache anyway. */ - if (unlikely(rt_task(rq->curr))) { + if (unlikely(rt_task(rq->curr)) && + (p->nr_cpus_allowed > 1)) { int cpu = find_lowest_rq(p); return (cpu == -1) ? task_cpu(p) : cpu; -- cgit v1.2.3 From 06f90dbd7610d51549004ea9c2ada337831eb292 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:13 +0100 Subject: sched: RT-balance, optimize We can cheaply track the number of bits set in the cpumask for the lowest priority CPUs. Therefore, compute the mask's weight and use it to skip the optimal domain search logic when there is only one CPU available. Signed-off-by: Gregory Haskins Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 72c81322fb9..52d88f193af 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -303,7 +303,7 @@ static int find_lowest_cpus(struct task_struct *task, cpumask_t *lowest_mask) int cpu; cpumask_t *valid_mask = &__get_cpu_var(valid_cpu_mask); int lowest_prio = -1; - int ret = 0; + int count = 0; cpus_clear(*lowest_mask); cpus_and(*valid_mask, cpu_online_map, task->cpus_allowed); @@ -316,7 +316,7 @@ static int find_lowest_cpus(struct task_struct *task, cpumask_t *lowest_mask) /* We look for lowest RT prio or non-rt CPU */ if (rq->rt.highest_prio >= MAX_RT_PRIO) { - if (ret) + if (count) cpus_clear(*lowest_mask); cpu_set(rq->cpu, *lowest_mask); return 1; @@ -328,14 +328,17 @@ static int find_lowest_cpus(struct task_struct *task, cpumask_t *lowest_mask) if (rq->rt.highest_prio > lowest_prio) { /* new low - clear old data */ lowest_prio = rq->rt.highest_prio; - cpus_clear(*lowest_mask); + if (count) { + cpus_clear(*lowest_mask); + count = 0; + } } cpu_set(rq->cpu, *lowest_mask); - ret = 1; + count++; } } - return ret; + return count; } static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask) @@ -359,9 +362,17 @@ static int find_lowest_rq(struct task_struct *task) cpumask_t *lowest_mask = &__get_cpu_var(local_cpu_mask); int this_cpu = smp_processor_id(); int cpu = task_cpu(task); + int count = find_lowest_cpus(task, lowest_mask); - if (!find_lowest_cpus(task, lowest_mask)) - return -1; + if (!count) + return -1; /* No targets found */ + + /* + * There is no sense in performing an optimal search if only one + * target is found. + */ + if (count == 1) + return first_cpu(*lowest_mask); /* * At this point we have built a mask of cpus representing the -- cgit v1.2.3 From 610bf05645a7ac6ea104a474e328eeaaea148870 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:13 +0100 Subject: sched: RT-balance, optimize cpu search This patch removes several cpumask operations by keeping track of the first of the CPUS that is of the lowest priority. When the search for the lowest priority runqueue is completed, all the bits up to the first CPU with the lowest priority runqueue is cleared. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 49 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 52d88f193af..61d198845f0 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -296,29 +296,36 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, } static DEFINE_PER_CPU(cpumask_t, local_cpu_mask); -static DEFINE_PER_CPU(cpumask_t, valid_cpu_mask); static int find_lowest_cpus(struct task_struct *task, cpumask_t *lowest_mask) { - int cpu; - cpumask_t *valid_mask = &__get_cpu_var(valid_cpu_mask); int lowest_prio = -1; + int lowest_cpu = -1; int count = 0; + int cpu; - cpus_clear(*lowest_mask); - cpus_and(*valid_mask, cpu_online_map, task->cpus_allowed); + cpus_and(*lowest_mask, cpu_online_map, task->cpus_allowed); /* * Scan each rq for the lowest prio. */ - for_each_cpu_mask(cpu, *valid_mask) { + for_each_cpu_mask(cpu, *lowest_mask) { struct rq *rq = cpu_rq(cpu); /* We look for lowest RT prio or non-rt CPU */ if (rq->rt.highest_prio >= MAX_RT_PRIO) { - if (count) + /* + * if we already found a low RT queue + * and now we found this non-rt queue + * clear the mask and set our bit. + * Otherwise just return the queue as is + * and the count==1 will cause the algorithm + * to use the first bit found. + */ + if (lowest_cpu != -1) { cpus_clear(*lowest_mask); - cpu_set(rq->cpu, *lowest_mask); + cpu_set(rq->cpu, *lowest_mask); + } return 1; } @@ -328,13 +335,29 @@ static int find_lowest_cpus(struct task_struct *task, cpumask_t *lowest_mask) if (rq->rt.highest_prio > lowest_prio) { /* new low - clear old data */ lowest_prio = rq->rt.highest_prio; - if (count) { - cpus_clear(*lowest_mask); - count = 0; - } + lowest_cpu = cpu; + count = 0; } - cpu_set(rq->cpu, *lowest_mask); count++; + } else + cpu_clear(cpu, *lowest_mask); + } + + /* + * Clear out all the set bits that represent + * runqueues that were of higher prio than + * the lowest_prio. + */ + if (lowest_cpu > 0) { + /* + * Perhaps we could add another cpumask op to + * zero out bits. Like cpu_zero_bits(cpumask, nrbits); + * Then that could be optimized to use memset and such. + */ + for_each_cpu_mask(cpu, *lowest_mask) { + if (cpu >= lowest_cpu) + break; + cpu_clear(cpu, *lowest_mask); } } -- cgit v1.2.3 From 0d1311a536a0face6267e7346223f2e68b002018 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:14 +0100 Subject: sched: RT-balance on new task rt-balance when creating new tasks. Signed-off-by: Ingo Molnar --- kernel/sched.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index c591abd9ca3..36bd8ff2a66 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -1722,6 +1722,7 @@ void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags) inc_nr_running(p, rq); } check_preempt_curr(rq, p); + wakeup_balance_rt(rq, p); task_rq_unlock(rq, &flags); } -- cgit v1.2.3 From 79064fbf75796c4c6a53e40729dbe52f789a91fd Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:14 +0100 Subject: sched: clean up pick_next_highest_task_rt() clean up pick_next_highest_task_rt(). Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 61d198845f0..b8435fd47f7 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -242,8 +242,7 @@ static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) } /* Return the second highest RT task, NULL otherwise */ -static struct task_struct *pick_next_highest_task_rt(struct rq *rq, - int cpu) +static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu) { struct rt_prio_array *array = &rq->rt.active; struct task_struct *next; @@ -270,7 +269,8 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, if (queue->next->next != queue) { /* same prio task */ - next = list_entry(queue->next->next, struct task_struct, run_list); + next = list_entry(queue->next->next, struct task_struct, + run_list); if (pick_rt_task(rq, next, cpu)) goto out; } -- cgit v1.2.3 From 4df64c0bfb7e0e260d10ebc005f7d0ba1308eed7 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:15 +0100 Subject: sched: clean up find_lock_lowest_rq() clean up find_lock_lowest_rq(). Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index b8435fd47f7..0749c1837b1 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -438,12 +438,11 @@ static int find_lowest_rq(struct task_struct *task) } /* Will lock the rq it finds */ -static struct rq *find_lock_lowest_rq(struct task_struct *task, - struct rq *rq) +static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) { struct rq *lowest_rq = NULL; - int cpu; int tries; + int cpu; for (tries = 0; tries < RT_MAX_TRIES; tries++) { cpu = find_lowest_rq(task); @@ -462,9 +461,11 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, * Also make sure that it wasn't scheduled on its rq. */ if (unlikely(task_rq(task) != rq || - !cpu_isset(lowest_rq->cpu, task->cpus_allowed) || + !cpu_isset(lowest_rq->cpu, + task->cpus_allowed) || task_running(rq, task) || !task->se.on_rq)) { + spin_unlock(&lowest_rq->lock); lowest_rq = NULL; break; -- cgit v1.2.3 From deeeccd41bd94a9db133d7b923f9a7479a47305d Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:15 +0100 Subject: sched: clean up overlong line in kernel/sched_debug.c clean up overlong line in kernel/sched_debug.c. Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 0749c1837b1..b591b89710a 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -762,6 +762,7 @@ move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, /* don't touch RT tasks */ return 0; } + static void set_cpus_allowed_rt(struct task_struct *p, cpumask_t *new_mask) { int weight = cpus_weight(*new_mask); @@ -775,9 +776,9 @@ static void set_cpus_allowed_rt(struct task_struct *p, cpumask_t *new_mask) if (p->se.on_rq && (weight != p->nr_cpus_allowed)) { struct rq *rq = task_rq(p); - if ((p->nr_cpus_allowed <= 1) && (weight > 1)) + if ((p->nr_cpus_allowed <= 1) && (weight > 1)) { rq->rt.rt_nr_migratory++; - else if((p->nr_cpus_allowed > 1) && (weight <= 1)) { + } else if ((p->nr_cpus_allowed > 1) && (weight <= 1)) { BUG_ON(!rq->rt.rt_nr_migratory); rq->rt.rt_nr_migratory--; } @@ -788,6 +789,7 @@ static void set_cpus_allowed_rt(struct task_struct *p, cpumask_t *new_mask) p->cpus_allowed = *new_mask; p->nr_cpus_allowed = weight; } + #else /* CONFIG_SMP */ # define schedule_tail_balance_rt(rq) do { } while (0) # define schedule_balance_rt(rq, prev) do { } while (0) -- cgit v1.2.3 From 84de4274893691aa8c471a1f7336d51e555d23a0 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:15 +0100 Subject: sched: clean up kernel/sched_rt.c clean up whitespace damage and missing comments in kernel/sched_rt.c. Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index b591b89710a..1a2d8f0aa65 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -4,16 +4,24 @@ */ #ifdef CONFIG_SMP + +/* + * The "RT overload" flag: it gets set if a CPU has more than + * one runnable RT task. + */ static cpumask_t rt_overload_mask; static atomic_t rto_count; + static inline int rt_overloaded(void) { return atomic_read(&rto_count); } + static inline cpumask_t *rt_overload(void) { return &rt_overload_mask; } + static inline void rt_set_overload(struct rq *rq) { rq->rt.overloaded = 1; @@ -28,6 +36,7 @@ static inline void rt_set_overload(struct rq *rq) wmb(); atomic_inc(&rto_count); } + static inline void rt_clear_overload(struct rq *rq) { /* the order here really doesn't matter */ -- cgit v1.2.3 From 6e1938d3ad58c940ec4119d387dd92a787cb238c Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:16 +0100 Subject: sched: remove rt_overload() remove rt_overload() - it's an unnecessary indirection. Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 1a2d8f0aa65..deff0c77d70 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -17,11 +17,6 @@ static inline int rt_overloaded(void) return atomic_read(&rto_count); } -static inline cpumask_t *rt_overload(void) -{ - return &rt_overload_mask; -} - static inline void rt_set_overload(struct rq *rq) { rq->rt.overloaded = 1; @@ -590,7 +585,6 @@ static int pull_rt_task(struct rq *this_rq) struct task_struct *next; struct task_struct *p; struct rq *src_rq; - cpumask_t *rto_cpumask; int this_cpu = this_rq->cpu; int cpu; int ret = 0; @@ -608,9 +602,7 @@ static int pull_rt_task(struct rq *this_rq) next = pick_next_task_rt(this_rq); - rto_cpumask = rt_overload(); - - for_each_cpu_mask(cpu, *rto_cpumask) { + for_each_cpu_mask(cpu, rt_overload_mask) { if (this_cpu == cpu) continue; -- cgit v1.2.3 From 00597c3ed78e424bdafff123565c078d8b6088cf Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:16 +0100 Subject: sched: remove leftover debugging remove leftover debugging. Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 8 -------- 1 file changed, 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index deff0c77d70..cc38521c572 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -253,8 +253,6 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu) struct list_head *queue; int idx; - assert_spin_locked(&rq->lock); - if (likely(rq->rt.rt_nr_running < 2)) return NULL; @@ -500,8 +498,6 @@ static int push_rt_task(struct rq *rq) int ret = 0; int paranoid = RT_MAX_TRIES; - assert_spin_locked(&rq->lock); - if (!rq->rt.overloaded) return 0; @@ -546,8 +542,6 @@ static int push_rt_task(struct rq *rq) goto out; } - assert_spin_locked(&lowest_rq->lock); - deactivate_task(rq, next_task, 0); set_task_cpu(next_task, lowest_rq->cpu); activate_task(lowest_rq, next_task, 0); @@ -589,8 +583,6 @@ static int pull_rt_task(struct rq *this_rq) int cpu; int ret = 0; - assert_spin_locked(&this_rq->lock); - /* * If cpusets are used, and we have overlapping * run queue cpusets, then this algorithm may not catch all. -- cgit v1.2.3 From 80bf3171dcdf0f5d236e2e48afe2a95c7ce23879 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:17 +0100 Subject: sched: clean up pull_rt_task() clean up pull_rt_task(). Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index cc38521c572..05ada7d4480 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -576,12 +576,9 @@ static void push_rt_tasks(struct rq *rq) static int pull_rt_task(struct rq *this_rq) { - struct task_struct *next; - struct task_struct *p; + int this_cpu = this_rq->cpu, ret = 0, cpu; + struct task_struct *p, *next; struct rq *src_rq; - int this_cpu = this_rq->cpu; - int cpu; - int ret = 0; /* * If cpusets are used, and we have overlapping @@ -608,23 +605,25 @@ static int pull_rt_task(struct rq *this_rq) if (double_lock_balance(this_rq, src_rq)) { /* unlocked our runqueue lock */ struct task_struct *old_next = next; + next = pick_next_task_rt(this_rq); if (next != old_next) ret = 1; } - if (likely(src_rq->rt.rt_nr_running <= 1)) + if (likely(src_rq->rt.rt_nr_running <= 1)) { /* * Small chance that this_rq->curr changed * but it's really harmless here. */ rt_clear_overload(this_rq); - else + } else { /* * Heh, the src_rq is now overloaded, since * we already have the src_rq lock, go straight * to pulling tasks from it. */ goto try_pulling; + } spin_unlock(&src_rq->lock); continue; } @@ -638,6 +637,7 @@ static int pull_rt_task(struct rq *this_rq) */ if (double_lock_balance(this_rq, src_rq)) { struct task_struct *old_next = next; + next = pick_next_task_rt(this_rq); if (next != old_next) ret = 1; @@ -674,7 +674,7 @@ static int pull_rt_task(struct rq *this_rq) */ if (p->prio < src_rq->curr->prio || (next && next->prio < src_rq->curr->prio)) - goto bail; + goto out; ret = 1; @@ -686,9 +686,7 @@ static int pull_rt_task(struct rq *this_rq) * case there's an even higher prio task * in another runqueue. (low likelyhood * but possible) - */ - - /* + * * Update next so that we won't pick a task * on another cpu with a priority lower (or equal) * than the one we just picked. @@ -696,7 +694,7 @@ static int pull_rt_task(struct rq *this_rq) next = p; } - bail: + out: spin_unlock(&src_rq->lock); } -- cgit v1.2.3 From 7f51f298204ec0528422cd9b23feac12612c5665 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:17 +0100 Subject: sched: clean up schedule_balance_rt() clean up schedule_balance_rt(). Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 05ada7d4480..4d0a60e47df 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -701,12 +701,10 @@ static int pull_rt_task(struct rq *this_rq) return ret; } -static void schedule_balance_rt(struct rq *rq, - struct task_struct *prev) +static void schedule_balance_rt(struct rq *rq, struct task_struct *prev) { /* Try to pull RT tasks here if we lower this rq's prio */ - if (unlikely(rt_task(prev)) && - rq->rt.highest_prio > prev->prio) + if (unlikely(rt_task(prev)) && rq->rt.highest_prio > prev->prio) pull_rt_task(rq); } -- cgit v1.2.3 From 57d885fea0da0e9541d7730a9e1dcf734981a173 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:18 +0100 Subject: sched: add sched-domain roots We add the notion of a root-domain which will be used later to rescope global variables to per-domain variables. Each exclusive cpuset essentially defines an island domain by fully partitioning the member cpus from any other cpuset. However, we currently still maintain some policy/state as global variables which transcend all cpusets. Consider, for instance, rt-overload state. Whenever a new exclusive cpuset is created, we also create a new root-domain object and move each cpu member to the root-domain's span. By default the system creates a single root-domain with all cpus as members (mimicking the global state we have today). We add some plumbing for storing class specific data in our root-domain. Whenever a RQ is switching root-domains (because of repartitioning) we give each sched_class the opportunity to remove any state from its old domain and add state to the new one. This logic doesn't have any clients yet but it will later in the series. Signed-off-by: Gregory Haskins CC: Christoph Lameter CC: Paul Jackson CC: Simon Derr Signed-off-by: Ingo Molnar --- kernel/sched.c | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 36bd8ff2a66..34b7d721d73 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -349,6 +349,28 @@ struct rt_rq { int overloaded; }; +#ifdef CONFIG_SMP + +/* + * We add the notion of a root-domain which will be used to define per-domain + * variables. Each exclusive cpuset essentially defines an island domain by + * fully partitioning the member cpus from any other cpuset. Whenever a new + * exclusive cpuset is created, we also create and attach a new root-domain + * object. + * + * By default the system creates a single root-domain with all cpus as + * members (mimicking the global state we have today). + */ +struct root_domain { + atomic_t refcount; + cpumask_t span; + cpumask_t online; +}; + +static struct root_domain def_root_domain; + +#endif + /* * This is the main, per-CPU runqueue data structure. * @@ -406,6 +428,7 @@ struct rq { atomic_t nr_iowait; #ifdef CONFIG_SMP + struct root_domain *rd; struct sched_domain *sd; /* For active balancing */ @@ -5550,6 +5573,15 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) case CPU_ONLINE_FROZEN: /* Strictly unnecessary, as first user will wake it. */ wake_up_process(cpu_rq(cpu)->migration_thread); + + /* Update our root-domain */ + rq = cpu_rq(cpu); + spin_lock_irqsave(&rq->lock, flags); + if (rq->rd) { + BUG_ON(!cpu_isset(cpu, rq->rd->span)); + cpu_set(cpu, rq->rd->online); + } + spin_unlock_irqrestore(&rq->lock, flags); break; #ifdef CONFIG_HOTPLUG_CPU @@ -5600,6 +5632,17 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) } spin_unlock_irq(&rq->lock); break; + + case CPU_DOWN_PREPARE: + /* Update our root-domain */ + rq = cpu_rq(cpu); + spin_lock_irqsave(&rq->lock, flags); + if (rq->rd) { + BUG_ON(!cpu_isset(cpu, rq->rd->span)); + cpu_clear(cpu, rq->rd->online); + } + spin_unlock_irqrestore(&rq->lock, flags); + break; #endif } return NOTIFY_OK; @@ -5788,11 +5831,69 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) return 1; } +static void rq_attach_root(struct rq *rq, struct root_domain *rd) +{ + unsigned long flags; + const struct sched_class *class; + + spin_lock_irqsave(&rq->lock, flags); + + if (rq->rd) { + struct root_domain *old_rd = rq->rd; + + for (class = sched_class_highest; class; class = class->next) + if (class->leave_domain) + class->leave_domain(rq); + + if (atomic_dec_and_test(&old_rd->refcount)) + kfree(old_rd); + } + + atomic_inc(&rd->refcount); + rq->rd = rd; + + for (class = sched_class_highest; class; class = class->next) + if (class->join_domain) + class->join_domain(rq); + + spin_unlock_irqrestore(&rq->lock, flags); +} + +static void init_rootdomain(struct root_domain *rd, const cpumask_t *map) +{ + memset(rd, 0, sizeof(*rd)); + + rd->span = *map; + cpus_and(rd->online, rd->span, cpu_online_map); +} + +static void init_defrootdomain(void) +{ + cpumask_t cpus = CPU_MASK_ALL; + + init_rootdomain(&def_root_domain, &cpus); + atomic_set(&def_root_domain.refcount, 1); +} + +static struct root_domain *alloc_rootdomain(const cpumask_t *map) +{ + struct root_domain *rd; + + rd = kmalloc(sizeof(*rd), GFP_KERNEL); + if (!rd) + return NULL; + + init_rootdomain(rd, map); + + return rd; +} + /* * Attach the domain 'sd' to 'cpu' as its base domain. Callers must * hold the hotplug lock. */ -static void cpu_attach_domain(struct sched_domain *sd, int cpu) +static void cpu_attach_domain(struct sched_domain *sd, + struct root_domain *rd, int cpu) { struct rq *rq = cpu_rq(cpu); struct sched_domain *tmp; @@ -5817,6 +5918,7 @@ static void cpu_attach_domain(struct sched_domain *sd, int cpu) sched_domain_debug(sd, cpu); + rq_attach_root(rq, rd); rcu_assign_pointer(rq->sd, sd); } @@ -6185,6 +6287,7 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd) static int build_sched_domains(const cpumask_t *cpu_map) { int i; + struct root_domain *rd; #ifdef CONFIG_NUMA struct sched_group **sched_group_nodes = NULL; int sd_allnodes = 0; @@ -6201,6 +6304,12 @@ static int build_sched_domains(const cpumask_t *cpu_map) sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes; #endif + rd = alloc_rootdomain(cpu_map); + if (!rd) { + printk(KERN_WARNING "Cannot alloc root domain\n"); + return -ENOMEM; + } + /* * Set up domains for cpus specified by the cpu_map. */ @@ -6417,7 +6526,7 @@ static int build_sched_domains(const cpumask_t *cpu_map) #else sd = &per_cpu(phys_domains, i); #endif - cpu_attach_domain(sd, i); + cpu_attach_domain(sd, rd, i); } return 0; @@ -6475,7 +6584,7 @@ static void detach_destroy_domains(const cpumask_t *cpu_map) unregister_sched_domain_sysctl(); for_each_cpu_mask(i, *cpu_map) - cpu_attach_domain(NULL, i); + cpu_attach_domain(NULL, &def_root_domain, i); synchronize_sched(); arch_destroy_sched_domains(cpu_map); } @@ -6727,6 +6836,10 @@ void __init sched_init(void) int highest_cpu = 0; int i, j; +#ifdef CONFIG_SMP + init_defrootdomain(); +#endif + for_each_possible_cpu(i) { struct rt_prio_array *array; struct rq *rq; @@ -6765,6 +6878,8 @@ void __init sched_init(void) rq->cpu_load[j] = 0; #ifdef CONFIG_SMP rq->sd = NULL; + rq->rd = NULL; + rq_attach_root(rq, &def_root_domain); rq->active_balance = 0; rq->next_balance = jiffies; rq->push_cpu = 0; -- cgit v1.2.3 From 637f50851b57a32f7ec67c50fc16f1601ab1a87a Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:18 +0100 Subject: sched: only balance our RT tasks within our domain We move the rt-overload data as the first global to per-domain reclassification. This limits the scope of overload related cache-line bouncing to stay with a specified partition instead of affecting all cpus in the system. Finally, we limit the scope of find_lowest_cpu searches to the domain instead of the entire system. Note that we would always respect domain boundaries even without this patch, but we first would scan potentially all cpus before whittling the list down. Now we can avoid looking at RQs that are out of scope, again reducing cache-line hits. Note: In some cases, task->cpus_allowed will effectively reduce our search to within our domain. However, I believe there are cases where the cpus_allowed mask may be all ones and therefore we err on the side of caution. If it can be optimized later, so be it. Signed-off-by: Gregory Haskins CC: Christoph Lameter Signed-off-by: Ingo Molnar --- kernel/sched.c | 7 +++++++ kernel/sched_rt.c | 57 ++++++++++++++++++++++++++++++------------------------- 2 files changed, 38 insertions(+), 26 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 34b7d721d73..35ef06c9921 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -365,6 +365,13 @@ struct root_domain { atomic_t refcount; cpumask_t span; cpumask_t online; + + /* + * The "RT overload" flag: it gets set if a CPU has more than + * one runnable RT task. + */ + cpumask_t rto_mask; + atomic_t rto_count; }; static struct root_domain def_root_domain; diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 4d0a60e47df..b049e5110ee 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -5,22 +5,14 @@ #ifdef CONFIG_SMP -/* - * The "RT overload" flag: it gets set if a CPU has more than - * one runnable RT task. - */ -static cpumask_t rt_overload_mask; -static atomic_t rto_count; - -static inline int rt_overloaded(void) +static inline int rt_overloaded(struct rq *rq) { - return atomic_read(&rto_count); + return atomic_read(&rq->rd->rto_count); } static inline void rt_set_overload(struct rq *rq) { - rq->rt.overloaded = 1; - cpu_set(rq->cpu, rt_overload_mask); + cpu_set(rq->cpu, rq->rd->rto_mask); /* * Make sure the mask is visible before we set * the overload count. That is checked to determine @@ -29,23 +21,25 @@ static inline void rt_set_overload(struct rq *rq) * updated yet. */ wmb(); - atomic_inc(&rto_count); + atomic_inc(&rq->rd->rto_count); } static inline void rt_clear_overload(struct rq *rq) { /* the order here really doesn't matter */ - atomic_dec(&rto_count); - cpu_clear(rq->cpu, rt_overload_mask); - rq->rt.overloaded = 0; + atomic_dec(&rq->rd->rto_count); + cpu_clear(rq->cpu, rq->rd->rto_mask); } static void update_rt_migration(struct rq *rq) { - if (rq->rt.rt_nr_migratory && (rq->rt.rt_nr_running > 1)) + if (rq->rt.rt_nr_migratory && (rq->rt.rt_nr_running > 1)) { rt_set_overload(rq); - else + rq->rt.overloaded = 1; + } else { rt_clear_overload(rq); + rq->rt.overloaded = 0; + } } #endif /* CONFIG_SMP */ @@ -306,7 +300,7 @@ static int find_lowest_cpus(struct task_struct *task, cpumask_t *lowest_mask) int count = 0; int cpu; - cpus_and(*lowest_mask, cpu_online_map, task->cpus_allowed); + cpus_and(*lowest_mask, task_rq(task)->rd->online, task->cpus_allowed); /* * Scan each rq for the lowest prio. @@ -580,18 +574,12 @@ static int pull_rt_task(struct rq *this_rq) struct task_struct *p, *next; struct rq *src_rq; - /* - * If cpusets are used, and we have overlapping - * run queue cpusets, then this algorithm may not catch all. - * This is just the price you pay on trying to keep - * dirtying caches down on large SMP machines. - */ - if (likely(!rt_overloaded())) + if (likely(!rt_overloaded(this_rq))) return 0; next = pick_next_task_rt(this_rq); - for_each_cpu_mask(cpu, rt_overload_mask) { + for_each_cpu_mask(cpu, this_rq->rd->rto_mask) { if (this_cpu == cpu) continue; @@ -811,6 +799,20 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p) } } +/* Assumes rq->lock is held */ +static void join_domain_rt(struct rq *rq) +{ + if (rq->rt.overloaded) + rt_set_overload(rq); +} + +/* Assumes rq->lock is held */ +static void leave_domain_rt(struct rq *rq) +{ + if (rq->rt.overloaded) + rt_clear_overload(rq); +} + static void set_curr_task_rt(struct rq *rq) { struct task_struct *p = rq->curr; @@ -840,4 +842,7 @@ const struct sched_class rt_sched_class = { .set_curr_task = set_curr_task_rt, .task_tick = task_tick_rt, + + .join_domain = join_domain_rt, + .leave_domain = leave_domain_rt, }; -- cgit v1.2.3 From bdd7c81b4973e72b670eff6b5725bab189b723d6 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:18 +0100 Subject: sched: fix sched_rt.c:join/leave_domain fix build bug in sched_rt.c:join/leave_domain and make them only be included on SMP builds. Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index b049e5110ee..3ea0cae513d 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -767,6 +767,20 @@ static void set_cpus_allowed_rt(struct task_struct *p, cpumask_t *new_mask) p->nr_cpus_allowed = weight; } +/* Assumes rq->lock is held */ +static void join_domain_rt(struct rq *rq) +{ + if (rq->rt.overloaded) + rt_set_overload(rq); +} + +/* Assumes rq->lock is held */ +static void leave_domain_rt(struct rq *rq) +{ + if (rq->rt.overloaded) + rt_clear_overload(rq); +} + #else /* CONFIG_SMP */ # define schedule_tail_balance_rt(rq) do { } while (0) # define schedule_balance_rt(rq, prev) do { } while (0) @@ -799,20 +813,6 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p) } } -/* Assumes rq->lock is held */ -static void join_domain_rt(struct rq *rq) -{ - if (rq->rt.overloaded) - rt_set_overload(rq); -} - -/* Assumes rq->lock is held */ -static void leave_domain_rt(struct rq *rq) -{ - if (rq->rt.overloaded) - rt_clear_overload(rq); -} - static void set_curr_task_rt(struct rq *rq) { struct task_struct *p = rq->curr; @@ -838,11 +838,10 @@ const struct sched_class rt_sched_class = { .load_balance = load_balance_rt, .move_one_task = move_one_task_rt, .set_cpus_allowed = set_cpus_allowed_rt, + .join_domain = join_domain_rt, + .leave_domain = leave_domain_rt, #endif .set_curr_task = set_curr_task_rt, .task_tick = task_tick_rt, - - .join_domain = join_domain_rt, - .leave_domain = leave_domain_rt, }; -- cgit v1.2.3 From d7876a08db50895ed9808ede4a259cccf65eba47 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:19 +0100 Subject: sched: remove unused JIFFIES_TO_NS() macro remove unused JIFFIES_TO_NS() macro. Signed-off-by: Ingo Molnar --- kernel/sched.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 35ef06c9921..461ee900d1a 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -96,10 +96,9 @@ unsigned long long __attribute__((weak)) sched_clock(void) #define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) /* - * Some helpers for converting nanosecond timing to jiffy resolution + * Helpers for converting nanosecond timing to jiffy resolution */ #define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ)) -#define JIFFIES_TO_NS(TIME) ((TIME) * (NSEC_PER_SEC / HZ)) #define NICE_0_LOAD SCHED_LOAD_SCALE #define NICE_0_SHIFT SCHED_LOAD_SHIFT -- cgit v1.2.3 From 0eab9146571dfa9b50ea952ec2ab27d591f26b63 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:19 +0100 Subject: sched: style cleanup, #2 style cleanup of various changes that were done recently. no code changed: text data bss dec hex filename 26399 2578 48 29025 7161 sched.o.before 26399 2578 48 29025 7161 sched.o.after Signed-off-by: Ingo Molnar --- kernel/sched.c | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 461ee900d1a..23b9925a1df 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -235,17 +235,17 @@ static void set_se_shares(struct sched_entity *se, unsigned long shares); * Every task in system belong to this group at bootup. */ struct task_group init_task_group = { - .se = init_sched_entity_p, + .se = init_sched_entity_p, .cfs_rq = init_cfs_rq_p, }; #ifdef CONFIG_FAIR_USER_SCHED -# define INIT_TASK_GROUP_LOAD 2*NICE_0_LOAD +# define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD) #else # define INIT_TASK_GROUP_LOAD NICE_0_LOAD #endif -#define MIN_GROUP_SHARES 2 +#define MIN_GROUP_SHARES 2 static int init_task_group_load = INIT_TASK_GROUP_LOAD; @@ -352,8 +352,8 @@ struct rt_rq { /* * We add the notion of a root-domain which will be used to define per-domain - * variables. Each exclusive cpuset essentially defines an island domain by - * fully partitioning the member cpus from any other cpuset. Whenever a new + * variables. Each exclusive cpuset essentially defines an island domain by + * fully partitioning the member cpus from any other cpuset. Whenever a new * exclusive cpuset is created, we also create and attach a new root-domain * object. * @@ -365,12 +365,12 @@ struct root_domain { cpumask_t span; cpumask_t online; - /* + /* * The "RT overload" flag: it gets set if a CPU has more than * one runnable RT task. */ cpumask_t rto_mask; - atomic_t rto_count; + atomic_t rto_count; }; static struct root_domain def_root_domain; @@ -434,7 +434,7 @@ struct rq { atomic_t nr_iowait; #ifdef CONFIG_SMP - struct root_domain *rd; + struct root_domain *rd; struct sched_domain *sd; /* For active balancing */ @@ -5066,7 +5066,7 @@ int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask) if (p->sched_class->set_cpus_allowed) p->sched_class->set_cpus_allowed(p, &new_mask); else { - p->cpus_allowed = new_mask; + p->cpus_allowed = new_mask; p->nr_cpus_allowed = cpus_weight(new_mask); } @@ -5847,9 +5847,10 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) if (rq->rd) { struct root_domain *old_rd = rq->rd; - for (class = sched_class_highest; class; class = class->next) + for (class = sched_class_highest; class; class = class->next) { if (class->leave_domain) class->leave_domain(rq); + } if (atomic_dec_and_test(&old_rd->refcount)) kfree(old_rd); @@ -5858,9 +5859,10 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) atomic_inc(&rd->refcount); rq->rd = rd; - for (class = sched_class_highest; class; class = class->next) + for (class = sched_class_highest; class; class = class->next) { if (class->join_domain) class->join_domain(rq); + } spin_unlock_irqrestore(&rq->lock, flags); } @@ -5895,11 +5897,11 @@ static struct root_domain *alloc_rootdomain(const cpumask_t *map) } /* - * Attach the domain 'sd' to 'cpu' as its base domain. Callers must + * Attach the domain 'sd' to 'cpu' as its base domain. Callers must * hold the hotplug lock. */ -static void cpu_attach_domain(struct sched_domain *sd, - struct root_domain *rd, int cpu) +static void +cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) { struct rq *rq = cpu_rq(cpu); struct sched_domain *tmp; @@ -7095,7 +7097,7 @@ static int rebalance_shares(struct sched_domain *sd, int this_cpu) for_each_cpu_mask(i, sdspan) total_load += tg->cfs_rq[i]->load.weight; - /* Nothing to do if this group has no load */ + /* Nothing to do if this group has no load */ if (!total_load) continue; -- cgit v1.2.3 From b913176917399e92e6f741672038c73d7ce93be5 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:19 +0100 Subject: sched: add credits for RT balancing improvements add credits for RT balancing improvements. Signed-off-by: Ingo Molnar --- kernel/sched.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 23b9925a1df..55c521780f9 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -22,6 +22,8 @@ * by Peter Williams * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri + * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins, + * Thomas Gleixner, Mike Kravetz */ #include -- cgit v1.2.3 From 9ec3b77e11b9398ab40b492c4fde7d8aac04a718 Mon Sep 17 00:00:00 2001 From: Dmitry Adamushko Date: Fri, 25 Jan 2008 21:08:21 +0100 Subject: sched: no need for 'affine wakeup' balancing No need to do a check for 'affine wakeup and passive balancing possibilities' in select_task_rq_fair() when task_cpu(p) == this_cpu. I guess, this part got missed upon introduction of per-sched_class select_task_rq() in try_to_wake_up(). Signed-off-by: Dmitry Adamushko Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index f881fc5e035..2208692dc4a 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -925,6 +925,9 @@ static int select_task_rq_fair(struct task_struct *p, int sync) this_cpu = smp_processor_id(); new_cpu = cpu; + if (cpu == this_cpu) + goto out_set_cpu; + for_each_domain(this_cpu, sd) { if (cpu_isset(cpu, sd->span)) { this_sd = sd; -- cgit v1.2.3 From 5d2f5a616d65e3c08acde3195694c4ab8afbc1b7 Mon Sep 17 00:00:00 2001 From: Dmitry Adamushko Date: Fri, 25 Jan 2008 21:08:21 +0100 Subject: sched: get rid of 'new_cpu' in try_to_wake_up() Clean-up try_to_wake_up(). Get rid of the 'new_cpu' variable in try_to_wake_up() [ that's, one #ifdef section less ]. Also remove a few redundant blank lines. Signed-off-by: Dmitry Adamushko Signed-off-by: Ingo Molnar --- kernel/sched.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 55c521780f9..9d6fb731559 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -1559,9 +1559,6 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync) unsigned long flags; long old_state; struct rq *rq; -#ifdef CONFIG_SMP - int new_cpu; -#endif rq = task_rq_lock(p, &flags); old_state = p->state; @@ -1579,9 +1576,9 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync) if (unlikely(task_running(rq, p))) goto out_activate; - new_cpu = p->sched_class->select_task_rq(p, sync); - if (new_cpu != cpu) { - set_task_cpu(p, new_cpu); + cpu = p->sched_class->select_task_rq(p, sync); + if (cpu != orig_cpu) { + set_task_cpu(p, cpu); task_rq_unlock(rq, &flags); /* might preempt at this point */ rq = task_rq_lock(p, &flags); @@ -1608,10 +1605,8 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync) } } } - #endif - out_activate: #endif /* CONFIG_SMP */ schedstat_inc(p, se.nr_wakeups); -- cgit v1.2.3 From 4bf0b77158d581c952af237aec79d0604b78fe27 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:21 +0100 Subject: sched: remove do_div() from __sched_slice() Yanmin Zhang noticed a nice optimization: p = l * nr / nl, nl = l/g -> p = g * nr which eliminates a do_div() from __sched_period(). Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 2208692dc4a..10aa6e1ae3d 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -248,8 +248,8 @@ static u64 __sched_period(unsigned long nr_running) unsigned long nr_latency = sched_nr_latency; if (unlikely(nr_running > nr_latency)) { + period = sysctl_sched_min_granularity; period *= nr_running; - do_div(period, nr_latency); } return period; -- cgit v1.2.3 From 9a897c5a6701bcb6f099f7ca20194999102729fd Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:22 +0100 Subject: sched: RT-balance, replace hooks with pre/post schedule and wakeup methods To make the main sched.c code more agnostic to the schedule classes. Instead of having specific hooks in the schedule code for the RT class balancing. They are replaced with a pre_schedule, post_schedule and task_wake_up methods. These methods may be used by any of the classes but currently, only the sched_rt class implements them. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched.c | 20 ++++++++++++++++---- kernel/sched_rt.c | 17 +++++++---------- 2 files changed, 23 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 9d6fb731559..2368a0d882e 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -1625,7 +1625,10 @@ out_activate: out_running: p->state = TASK_RUNNING; - wakeup_balance_rt(rq, p); +#ifdef CONFIG_SMP + if (p->sched_class->task_wake_up) + p->sched_class->task_wake_up(rq, p); +#endif out: task_rq_unlock(rq, &flags); @@ -1748,7 +1751,10 @@ void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags) inc_nr_running(p, rq); } check_preempt_curr(rq, p); - wakeup_balance_rt(rq, p); +#ifdef CONFIG_SMP + if (p->sched_class->task_wake_up) + p->sched_class->task_wake_up(rq, p); +#endif task_rq_unlock(rq, &flags); } @@ -1869,7 +1875,10 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev) prev_state = prev->state; finish_arch_switch(prev); finish_lock_switch(rq, prev); - schedule_tail_balance_rt(rq); +#ifdef CONFIG_SMP + if (current->sched_class->post_schedule) + current->sched_class->post_schedule(rq); +#endif fire_sched_in_preempt_notifiers(current); if (mm) @@ -3638,7 +3647,10 @@ need_resched_nonpreemptible: switch_count = &prev->nvcsw; } - schedule_balance_rt(rq, prev); +#ifdef CONFIG_SMP + if (prev->sched_class->pre_schedule) + prev->sched_class->pre_schedule(rq, prev); +#endif if (unlikely(!rq->nr_running)) idle_balance(cpu, rq); diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 3ea0cae513d..a5a45104603 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -689,14 +689,14 @@ static int pull_rt_task(struct rq *this_rq) return ret; } -static void schedule_balance_rt(struct rq *rq, struct task_struct *prev) +static void pre_schedule_rt(struct rq *rq, struct task_struct *prev) { /* Try to pull RT tasks here if we lower this rq's prio */ if (unlikely(rt_task(prev)) && rq->rt.highest_prio > prev->prio) pull_rt_task(rq); } -static void schedule_tail_balance_rt(struct rq *rq) +static void post_schedule_rt(struct rq *rq) { /* * If we have more than one rt_task queued, then @@ -713,10 +713,9 @@ static void schedule_tail_balance_rt(struct rq *rq) } -static void wakeup_balance_rt(struct rq *rq, struct task_struct *p) +static void task_wake_up_rt(struct rq *rq, struct task_struct *p) { - if (unlikely(rt_task(p)) && - !task_running(rq, p) && + if (!task_running(rq, p) && (p->prio >= rq->rt.highest_prio) && rq->rt.overloaded) push_rt_tasks(rq); @@ -780,11 +779,6 @@ static void leave_domain_rt(struct rq *rq) if (rq->rt.overloaded) rt_clear_overload(rq); } - -#else /* CONFIG_SMP */ -# define schedule_tail_balance_rt(rq) do { } while (0) -# define schedule_balance_rt(rq, prev) do { } while (0) -# define wakeup_balance_rt(rq, p) do { } while (0) #endif /* CONFIG_SMP */ static void task_tick_rt(struct rq *rq, struct task_struct *p) @@ -840,6 +834,9 @@ const struct sched_class rt_sched_class = { .set_cpus_allowed = set_cpus_allowed_rt, .join_domain = join_domain_rt, .leave_domain = leave_domain_rt, + .pre_schedule = pre_schedule_rt, + .post_schedule = post_schedule_rt, + .task_wake_up = task_wake_up_rt, #endif .set_curr_task = set_curr_task_rt, -- cgit v1.2.3 From cb46984504048db946cd551c261df4e70d59a8ea Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 25 Jan 2008 21:08:22 +0100 Subject: sched: RT-balance, add new methods to sched_class Dmitry Adamushko found that the current implementation of the RT balancing code left out changes to the sched_setscheduler and rt_mutex_setprio. This patch addresses this issue by adding methods to the schedule classes to handle being switched out of (switched_from) and being switched into (switched_to) a sched_class. Also a method for changing of priorities is also added (prio_changed). This patch also removes some duplicate logic between rt_mutex_setprio and sched_setscheduler. Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched.c | 42 +++++++++++------------ kernel/sched_fair.c | 39 ++++++++++++++++++++++ kernel/sched_idletask.c | 31 +++++++++++++++++ kernel/sched_rt.c | 89 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 2368a0d882e..5834c7fb79a 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -1152,6 +1152,18 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) #endif } +static inline void check_class_changed(struct rq *rq, struct task_struct *p, + const struct sched_class *prev_class, + int oldprio, int running) +{ + if (prev_class != p->sched_class) { + if (prev_class->switched_from) + prev_class->switched_from(rq, p, running); + p->sched_class->switched_to(rq, p, running); + } else + p->sched_class->prio_changed(rq, p, oldprio, running); +} + #ifdef CONFIG_SMP /* @@ -4017,6 +4029,7 @@ void rt_mutex_setprio(struct task_struct *p, int prio) unsigned long flags; int oldprio, on_rq, running; struct rq *rq; + const struct sched_class *prev_class = p->sched_class; BUG_ON(prio < 0 || prio > MAX_PRIO); @@ -4042,18 +4055,10 @@ void rt_mutex_setprio(struct task_struct *p, int prio) if (on_rq) { if (running) p->sched_class->set_curr_task(rq); + enqueue_task(rq, p, 0); - /* - * Reschedule if we are currently running on this runqueue and - * our priority decreased, or if we are not currently running on - * this runqueue and our priority is higher than the current's - */ - if (running) { - if (p->prio > oldprio) - resched_task(rq->curr); - } else { - check_preempt_curr(rq, p); - } + + check_class_changed(rq, p, prev_class, oldprio, running); } task_rq_unlock(rq, &flags); } @@ -4253,6 +4258,7 @@ int sched_setscheduler(struct task_struct *p, int policy, { int retval, oldprio, oldpolicy = -1, on_rq, running; unsigned long flags; + const struct sched_class *prev_class = p->sched_class; struct rq *rq; /* may grab non-irq protected spin_locks */ @@ -4346,18 +4352,10 @@ recheck: if (on_rq) { if (running) p->sched_class->set_curr_task(rq); + activate_task(rq, p, 0); - /* - * Reschedule if we are currently running on this runqueue and - * our priority decreased, or if we are not currently running on - * this runqueue and our priority is higher than the current's - */ - if (running) { - if (p->prio > oldprio) - resched_task(rq->curr); - } else { - check_preempt_curr(rq, p); - } + + check_class_changed(rq, p, prev_class, oldprio, running); } __task_rq_unlock(rq); spin_unlock_irqrestore(&p->pi_lock, flags); diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 10aa6e1ae3d..dfa18d55561 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -1280,6 +1280,42 @@ static void task_new_fair(struct rq *rq, struct task_struct *p) resched_task(rq->curr); } +/* + * Priority of the task has changed. Check to see if we preempt + * the current task. + */ +static void prio_changed_fair(struct rq *rq, struct task_struct *p, + int oldprio, int running) +{ + /* + * Reschedule if we are currently running on this runqueue and + * our priority decreased, or if we are not currently running on + * this runqueue and our priority is higher than the current's + */ + if (running) { + if (p->prio > oldprio) + resched_task(rq->curr); + } else + check_preempt_curr(rq, p); +} + +/* + * We switched to the sched_fair class. + */ +static void switched_to_fair(struct rq *rq, struct task_struct *p, + int running) +{ + /* + * We were most likely switched from sched_rt, so + * kick off the schedule if running, otherwise just see + * if we can still preempt the current task. + */ + if (running) + resched_task(rq->curr); + else + check_preempt_curr(rq, p); +} + /* Account for a task changing its policy or group. * * This routine is mostly called to set cfs_rq->curr field when a task @@ -1318,6 +1354,9 @@ static const struct sched_class fair_sched_class = { .set_curr_task = set_curr_task_fair, .task_tick = task_tick_fair, .task_new = task_new_fair, + + .prio_changed = prio_changed_fair, + .switched_to = switched_to_fair, }; #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sched_idletask.c b/kernel/sched_idletask.c index ca5374860ae..ef7a2661fa1 100644 --- a/kernel/sched_idletask.c +++ b/kernel/sched_idletask.c @@ -69,6 +69,33 @@ static void set_curr_task_idle(struct rq *rq) { } +static void switched_to_idle(struct rq *rq, struct task_struct *p, + int running) +{ + /* Can this actually happen?? */ + if (running) + resched_task(rq->curr); + else + check_preempt_curr(rq, p); +} + +static void prio_changed_idle(struct rq *rq, struct task_struct *p, + int oldprio, int running) +{ + /* This can happen for hot plug CPUS */ + + /* + * Reschedule if we are currently running on this runqueue and + * our priority decreased, or if we are not currently running on + * this runqueue and our priority is higher than the current's + */ + if (running) { + if (p->prio > oldprio) + resched_task(rq->curr); + } else + check_preempt_curr(rq, p); +} + /* * Simple, special scheduling class for the per-CPU idle tasks: */ @@ -94,5 +121,9 @@ const struct sched_class idle_sched_class = { .set_curr_task = set_curr_task_idle, .task_tick = task_tick_idle, + + .prio_changed = prio_changed_idle, + .switched_to = switched_to_idle, + /* no .task_new for idle tasks */ }; diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index a5a45104603..57fa3d96847 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -779,7 +779,92 @@ static void leave_domain_rt(struct rq *rq) if (rq->rt.overloaded) rt_clear_overload(rq); } + +/* + * When switch from the rt queue, we bring ourselves to a position + * that we might want to pull RT tasks from other runqueues. + */ +static void switched_from_rt(struct rq *rq, struct task_struct *p, + int running) +{ + /* + * If there are other RT tasks then we will reschedule + * and the scheduling of the other RT tasks will handle + * the balancing. But if we are the last RT task + * we may need to handle the pulling of RT tasks + * now. + */ + if (!rq->rt.rt_nr_running) + pull_rt_task(rq); +} +#endif /* CONFIG_SMP */ + +/* + * When switching a task to RT, we may overload the runqueue + * with RT tasks. In this case we try to push them off to + * other runqueues. + */ +static void switched_to_rt(struct rq *rq, struct task_struct *p, + int running) +{ + int check_resched = 1; + + /* + * If we are already running, then there's nothing + * that needs to be done. But if we are not running + * we may need to preempt the current running task. + * If that current running task is also an RT task + * then see if we can move to another run queue. + */ + if (!running) { +#ifdef CONFIG_SMP + if (rq->rt.overloaded && push_rt_task(rq) && + /* Don't resched if we changed runqueues */ + rq != task_rq(p)) + check_resched = 0; +#endif /* CONFIG_SMP */ + if (check_resched && p->prio < rq->curr->prio) + resched_task(rq->curr); + } +} + +/* + * Priority of the task has changed. This may cause + * us to initiate a push or pull. + */ +static void prio_changed_rt(struct rq *rq, struct task_struct *p, + int oldprio, int running) +{ + if (running) { +#ifdef CONFIG_SMP + /* + * If our priority decreases while running, we + * may need to pull tasks to this runqueue. + */ + if (oldprio < p->prio) + pull_rt_task(rq); + /* + * If there's a higher priority task waiting to run + * then reschedule. + */ + if (p->prio > rq->rt.highest_prio) + resched_task(p); +#else + /* For UP simply resched on drop of prio */ + if (oldprio < p->prio) + resched_task(p); #endif /* CONFIG_SMP */ + } else { + /* + * This task is not running, but if it is + * greater than the current running task + * then reschedule. + */ + if (p->prio < rq->curr->prio) + resched_task(rq->curr); + } +} + static void task_tick_rt(struct rq *rq, struct task_struct *p) { @@ -837,8 +922,12 @@ const struct sched_class rt_sched_class = { .pre_schedule = pre_schedule_rt, .post_schedule = post_schedule_rt, .task_wake_up = task_wake_up_rt, + .switched_from = switched_from_rt, #endif .set_curr_task = set_curr_task_rt, .task_tick = task_tick_rt, + + .prio_changed = prio_changed_rt, + .switched_to = switched_to_rt, }; -- cgit v1.2.3 From cdc8eb984ce47a7c90a049f45229f7b0d59ba781 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:23 +0100 Subject: sched: RT-balance, only adjust overload state when changing The overload set/clears were originally idempotent when this logic was first implemented. But that is no longer true due to the addition of the atomic counter and this logic was never updated to work properly with that change. So only adjust the overload state if it is actually changing to avoid getting out of sync. Signed-off-by: Gregory Haskins Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 57fa3d96847..a386758ffeb 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -34,9 +34,11 @@ static inline void rt_clear_overload(struct rq *rq) static void update_rt_migration(struct rq *rq) { if (rq->rt.rt_nr_migratory && (rq->rt.rt_nr_running > 1)) { - rt_set_overload(rq); - rq->rt.overloaded = 1; - } else { + if (!rq->rt.overloaded) { + rt_set_overload(rq); + rq->rt.overloaded = 1; + } + } else if (rq->rt.overloaded) { rt_clear_overload(rq); rq->rt.overloaded = 0; } -- cgit v1.2.3 From c49443c538c1bbf50eda27e4a3711e9fc15176b0 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:23 +0100 Subject: sched: remove some old cpuset logic We had support for overlapping cpuset based rto logic in early prototypes that is no longer used, so remove it. Signed-off-by: Gregory Haskins Signed-off-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 33 --------------------------------- 1 file changed, 33 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index a386758ffeb..9affb3c9d3d 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -586,38 +586,6 @@ static int pull_rt_task(struct rq *this_rq) continue; src_rq = cpu_rq(cpu); - if (unlikely(src_rq->rt.rt_nr_running <= 1)) { - /* - * It is possible that overlapping cpusets - * will miss clearing a non overloaded runqueue. - * Clear it now. - */ - if (double_lock_balance(this_rq, src_rq)) { - /* unlocked our runqueue lock */ - struct task_struct *old_next = next; - - next = pick_next_task_rt(this_rq); - if (next != old_next) - ret = 1; - } - if (likely(src_rq->rt.rt_nr_running <= 1)) { - /* - * Small chance that this_rq->curr changed - * but it's really harmless here. - */ - rt_clear_overload(this_rq); - } else { - /* - * Heh, the src_rq is now overloaded, since - * we already have the src_rq lock, go straight - * to pulling tasks from it. - */ - goto try_pulling; - } - spin_unlock(&src_rq->lock); - continue; - } - /* * We can potentially drop this_rq's lock in * double_lock_balance, and another CPU could @@ -641,7 +609,6 @@ static int pull_rt_task(struct rq *this_rq) continue; } - try_pulling: p = pick_next_highest_task_rt(src_rq, this_cpu); /* -- cgit v1.2.3 From c2d727aa2ff17a1c8e5ed1e5e231bb8579b27e82 Mon Sep 17 00:00:00 2001 From: Dipankar Sarma Date: Fri, 25 Jan 2008 21:08:23 +0100 Subject: Preempt-RCU: Use softirq instead of tasklets for This patch makes RCU use softirq instead of tasklets. It also adds a memory barrier after raising the softirq inorder to ensure that the cpu sees the most recently updated value of rcu->cur while processing callbacks. The discussion of the related theoretical race pointed out by James Huang can be found here --> http://lkml.org/lkml/2007/11/20/603 Signed-off-by: Gautham R Shenoy Signed-off-by: Steven Rostedt Signed-off-by: Dipankar Sarma Reviewed-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/rcupdate.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c index f2c1a04e9b1..4dfa0b792ef 100644 --- a/kernel/rcupdate.c +++ b/kernel/rcupdate.c @@ -73,8 +73,6 @@ static struct rcu_ctrlblk rcu_bh_ctrlblk = { DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L }; DEFINE_PER_CPU(struct rcu_data, rcu_bh_data) = { 0L }; -/* Fake initialization required by compiler */ -static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL}; static int blimit = 10; static int qhimark = 10000; static int qlowmark = 100; @@ -231,6 +229,18 @@ void rcu_barrier(void) } EXPORT_SYMBOL_GPL(rcu_barrier); +/* Raises the softirq for processing rcu_callbacks. */ +static inline void raise_rcu_softirq(void) +{ + raise_softirq(RCU_SOFTIRQ); + /* + * The smp_mb() here is required to ensure that this cpu's + * __rcu_process_callbacks() reads the most recently updated + * value of rcu->cur. + */ + smp_mb(); +} + /* * Invoke the completed RCU callbacks. They are expected to be in * a per-cpu list. @@ -260,7 +270,7 @@ static void rcu_do_batch(struct rcu_data *rdp) if (!rdp->donelist) rdp->donetail = &rdp->donelist; else - tasklet_schedule(&per_cpu(rcu_tasklet, rdp->cpu)); + raise_rcu_softirq(); } /* @@ -412,7 +422,6 @@ static void rcu_offline_cpu(int cpu) &per_cpu(rcu_bh_data, cpu)); put_cpu_var(rcu_data); put_cpu_var(rcu_bh_data); - tasklet_kill_immediate(&per_cpu(rcu_tasklet, cpu), cpu); } #else @@ -424,7 +433,7 @@ static void rcu_offline_cpu(int cpu) #endif /* - * This does the RCU processing work from tasklet context. + * This does the RCU processing work from softirq context. */ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp, struct rcu_data *rdp) @@ -469,7 +478,7 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp, rcu_do_batch(rdp); } -static void rcu_process_callbacks(unsigned long unused) +static void rcu_process_callbacks(struct softirq_action *unused) { __rcu_process_callbacks(&rcu_ctrlblk, &__get_cpu_var(rcu_data)); __rcu_process_callbacks(&rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data)); @@ -533,7 +542,7 @@ void rcu_check_callbacks(int cpu, int user) rcu_bh_qsctr_inc(cpu); } else if (!in_softirq()) rcu_bh_qsctr_inc(cpu); - tasklet_schedule(&per_cpu(rcu_tasklet, cpu)); + raise_rcu_softirq(); } static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp, @@ -556,7 +565,7 @@ static void __cpuinit rcu_online_cpu(int cpu) rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp); rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp); - tasklet_init(&per_cpu(rcu_tasklet, cpu), rcu_process_callbacks, 0UL); + open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL); } static int __cpuinit rcu_cpu_notify(struct notifier_block *self, -- cgit v1.2.3 From 01c1c660f4b8086cad7a62345fd04290f3d82c8f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 25 Jan 2008 21:08:24 +0100 Subject: Preempt-RCU: reorganize RCU code into rcuclassic.c and rcupdate.c This patch re-organizes the RCU code to enable multiple implementations of RCU. Users of RCU continues to include rcupdate.h and the RCU interfaces remain the same. This is in preparation for subsequently merging the preemptible RCU implementation. Signed-off-by: Gautham R Shenoy Signed-off-by: Dipankar Sarma Signed-off-by: Paul E. McKenney Reviewed-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/Makefile | 2 +- kernel/rcuclassic.c | 576 ++++++++++++++++++++++++++++++++++++++++++++++++++++ kernel/rcupdate.c | 575 +++------------------------------------------------ 3 files changed, 602 insertions(+), 551 deletions(-) create mode 100644 kernel/rcuclassic.c (limited to 'kernel') diff --git a/kernel/Makefile b/kernel/Makefile index dfa96956dae..def5dd6097a 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -6,7 +6,7 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \ exit.o itimer.o time.o softirq.o resource.o \ sysctl.o capability.o ptrace.o timer.o user.o user_namespace.o \ signal.o sys.o kmod.o workqueue.o pid.o \ - rcupdate.o extable.o params.o posix-timers.o \ + rcupdate.o rcuclassic.o extable.o params.o posix-timers.o \ kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ hrtimer.o rwsem.o latency.o nsproxy.o srcu.o \ utsname.o notifier.o diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c new file mode 100644 index 00000000000..18369e3386e --- /dev/null +++ b/kernel/rcuclassic.c @@ -0,0 +1,576 @@ +/* + * Read-Copy Update mechanism for mutual exclusion + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright IBM Corporation, 2001 + * + * Authors: Dipankar Sarma + * Manfred Spraul + * + * Based on the original work by Paul McKenney + * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. + * Papers: + * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf + * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001) + * + * For detailed explanation of Read-Copy Update mechanism see - + * Documentation/RCU + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +/* #include @@@ */ +#include +#include + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +static struct lock_class_key rcu_lock_key; +struct lockdep_map rcu_lock_map = + STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key); +EXPORT_SYMBOL_GPL(rcu_lock_map); +#endif + + +/* Definition for rcupdate control block. */ +static struct rcu_ctrlblk rcu_ctrlblk = { + .cur = -300, + .completed = -300, + .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock), + .cpumask = CPU_MASK_NONE, +}; +static struct rcu_ctrlblk rcu_bh_ctrlblk = { + .cur = -300, + .completed = -300, + .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock), + .cpumask = CPU_MASK_NONE, +}; + +DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L }; +DEFINE_PER_CPU(struct rcu_data, rcu_bh_data) = { 0L }; + +static int blimit = 10; +static int qhimark = 10000; +static int qlowmark = 100; + +#ifdef CONFIG_SMP +static void force_quiescent_state(struct rcu_data *rdp, + struct rcu_ctrlblk *rcp) +{ + int cpu; + cpumask_t cpumask; + set_need_resched(); + if (unlikely(!rcp->signaled)) { + rcp->signaled = 1; + /* + * Don't send IPI to itself. With irqs disabled, + * rdp->cpu is the current cpu. + */ + cpumask = rcp->cpumask; + cpu_clear(rdp->cpu, cpumask); + for_each_cpu_mask(cpu, cpumask) + smp_send_reschedule(cpu); + } +} +#else +static inline void force_quiescent_state(struct rcu_data *rdp, + struct rcu_ctrlblk *rcp) +{ + set_need_resched(); +} +#endif + +/** + * call_rcu - Queue an RCU callback for invocation after a grace period. + * @head: structure to be used for queueing the RCU updates. + * @func: actual update function to be invoked after the grace period + * + * The update function will be invoked some time after a full grace + * period elapses, in other words after all currently executing RCU + * read-side critical sections have completed. RCU read-side critical + * sections are delimited by rcu_read_lock() and rcu_read_unlock(), + * and may be nested. + */ +void call_rcu(struct rcu_head *head, + void (*func)(struct rcu_head *rcu)) +{ + unsigned long flags; + struct rcu_data *rdp; + + head->func = func; + head->next = NULL; + local_irq_save(flags); + rdp = &__get_cpu_var(rcu_data); + *rdp->nxttail = head; + rdp->nxttail = &head->next; + if (unlikely(++rdp->qlen > qhimark)) { + rdp->blimit = INT_MAX; + force_quiescent_state(rdp, &rcu_ctrlblk); + } + local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(call_rcu); + +/** + * call_rcu_bh - Queue an RCU for invocation after a quicker grace period. + * @head: structure to be used for queueing the RCU updates. + * @func: actual update function to be invoked after the grace period + * + * The update function will be invoked some time after a full grace + * period elapses, in other words after all currently executing RCU + * read-side critical sections have completed. call_rcu_bh() assumes + * that the read-side critical sections end on completion of a softirq + * handler. This means that read-side critical sections in process + * context must not be interrupted by softirqs. This interface is to be + * used when most of the read-side critical sections are in softirq context. + * RCU read-side critical sections are delimited by rcu_read_lock() and + * rcu_read_unlock(), * if in interrupt context or rcu_read_lock_bh() + * and rcu_read_unlock_bh(), if in process context. These may be nested. + */ +void call_rcu_bh(struct rcu_head *head, + void (*func)(struct rcu_head *rcu)) +{ + unsigned long flags; + struct rcu_data *rdp; + + head->func = func; + head->next = NULL; + local_irq_save(flags); + rdp = &__get_cpu_var(rcu_bh_data); + *rdp->nxttail = head; + rdp->nxttail = &head->next; + + if (unlikely(++rdp->qlen > qhimark)) { + rdp->blimit = INT_MAX; + force_quiescent_state(rdp, &rcu_bh_ctrlblk); + } + + local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(call_rcu_bh); + +/* + * Return the number of RCU batches processed thus far. Useful + * for debug and statistics. + */ +long rcu_batches_completed(void) +{ + return rcu_ctrlblk.completed; +} +EXPORT_SYMBOL_GPL(rcu_batches_completed); + +/* + * Return the number of RCU batches processed thus far. Useful + * for debug and statistics. + */ +long rcu_batches_completed_bh(void) +{ + return rcu_bh_ctrlblk.completed; +} +EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); + +/* Raises the softirq for processing rcu_callbacks. */ +static inline void raise_rcu_softirq(void) +{ + raise_softirq(RCU_SOFTIRQ); + /* + * The smp_mb() here is required to ensure that this cpu's + * __rcu_process_callbacks() reads the most recently updated + * value of rcu->cur. + */ + smp_mb(); +} + +/* + * Invoke the completed RCU callbacks. They are expected to be in + * a per-cpu list. + */ +static void rcu_do_batch(struct rcu_data *rdp) +{ + struct rcu_head *next, *list; + int count = 0; + + list = rdp->donelist; + while (list) { + next = list->next; + prefetch(next); + list->func(list); + list = next; + if (++count >= rdp->blimit) + break; + } + rdp->donelist = list; + + local_irq_disable(); + rdp->qlen -= count; + local_irq_enable(); + if (rdp->blimit == INT_MAX && rdp->qlen <= qlowmark) + rdp->blimit = blimit; + + if (!rdp->donelist) + rdp->donetail = &rdp->donelist; + else + raise_rcu_softirq(); +} + +/* + * Grace period handling: + * The grace period handling consists out of two steps: + * - A new grace period is started. + * This is done by rcu_start_batch. The start is not broadcasted to + * all cpus, they must pick this up by comparing rcp->cur with + * rdp->quiescbatch. All cpus are recorded in the + * rcu_ctrlblk.cpumask bitmap. + * - All cpus must go through a quiescent state. + * Since the start of the grace period is not broadcasted, at least two + * calls to rcu_check_quiescent_state are required: + * The first call just notices that a new grace period is running. The + * following calls check if there was a quiescent state since the beginning + * of the grace period. If so, it updates rcu_ctrlblk.cpumask. If + * the bitmap is empty, then the grace period is completed. + * rcu_check_quiescent_state calls rcu_start_batch(0) to start the next grace + * period (if necessary). + */ +/* + * Register a new batch of callbacks, and start it up if there is currently no + * active batch and the batch to be registered has not already occurred. + * Caller must hold rcu_ctrlblk.lock. + */ +static void rcu_start_batch(struct rcu_ctrlblk *rcp) +{ + if (rcp->next_pending && + rcp->completed == rcp->cur) { + rcp->next_pending = 0; + /* + * next_pending == 0 must be visible in + * __rcu_process_callbacks() before it can see new value of cur. + */ + smp_wmb(); + rcp->cur++; + + /* + * Accessing nohz_cpu_mask before incrementing rcp->cur needs a + * Barrier Otherwise it can cause tickless idle CPUs to be + * included in rcp->cpumask, which will extend graceperiods + * unnecessarily. + */ + smp_mb(); + cpus_andnot(rcp->cpumask, cpu_online_map, nohz_cpu_mask); + + rcp->signaled = 0; + } +} + +/* + * cpu went through a quiescent state since the beginning of the grace period. + * Clear it from the cpu mask and complete the grace period if it was the last + * cpu. Start another grace period if someone has further entries pending + */ +static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp) +{ + cpu_clear(cpu, rcp->cpumask); + if (cpus_empty(rcp->cpumask)) { + /* batch completed ! */ + rcp->completed = rcp->cur; + rcu_start_batch(rcp); + } +} + +/* + * Check if the cpu has gone through a quiescent state (say context + * switch). If so and if it already hasn't done so in this RCU + * quiescent cycle, then indicate that it has done so. + */ +static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp, + struct rcu_data *rdp) +{ + if (rdp->quiescbatch != rcp->cur) { + /* start new grace period: */ + rdp->qs_pending = 1; + rdp->passed_quiesc = 0; + rdp->quiescbatch = rcp->cur; + return; + } + + /* Grace period already completed for this cpu? + * qs_pending is checked instead of the actual bitmap to avoid + * cacheline trashing. + */ + if (!rdp->qs_pending) + return; + + /* + * Was there a quiescent state since the beginning of the grace + * period? If no, then exit and wait for the next call. + */ + if (!rdp->passed_quiesc) + return; + rdp->qs_pending = 0; + + spin_lock(&rcp->lock); + /* + * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync + * during cpu startup. Ignore the quiescent state. + */ + if (likely(rdp->quiescbatch == rcp->cur)) + cpu_quiet(rdp->cpu, rcp); + + spin_unlock(&rcp->lock); +} + + +#ifdef CONFIG_HOTPLUG_CPU + +/* warning! helper for rcu_offline_cpu. do not use elsewhere without reviewing + * locking requirements, the list it's pulling from has to belong to a cpu + * which is dead and hence not processing interrupts. + */ +static void rcu_move_batch(struct rcu_data *this_rdp, struct rcu_head *list, + struct rcu_head **tail) +{ + local_irq_disable(); + *this_rdp->nxttail = list; + if (list) + this_rdp->nxttail = tail; + local_irq_enable(); +} + +static void __rcu_offline_cpu(struct rcu_data *this_rdp, + struct rcu_ctrlblk *rcp, struct rcu_data *rdp) +{ + /* if the cpu going offline owns the grace period + * we can block indefinitely waiting for it, so flush + * it here + */ + spin_lock_bh(&rcp->lock); + if (rcp->cur != rcp->completed) + cpu_quiet(rdp->cpu, rcp); + spin_unlock_bh(&rcp->lock); + rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail); + rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail); + rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail); +} + +static void rcu_offline_cpu(int cpu) +{ + struct rcu_data *this_rdp = &get_cpu_var(rcu_data); + struct rcu_data *this_bh_rdp = &get_cpu_var(rcu_bh_data); + + __rcu_offline_cpu(this_rdp, &rcu_ctrlblk, + &per_cpu(rcu_data, cpu)); + __rcu_offline_cpu(this_bh_rdp, &rcu_bh_ctrlblk, + &per_cpu(rcu_bh_data, cpu)); + put_cpu_var(rcu_data); + put_cpu_var(rcu_bh_data); +} + +#else + +static void rcu_offline_cpu(int cpu) +{ +} + +#endif + +/* + * This does the RCU processing work from softirq context. + */ +static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp, + struct rcu_data *rdp) +{ + if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) { + *rdp->donetail = rdp->curlist; + rdp->donetail = rdp->curtail; + rdp->curlist = NULL; + rdp->curtail = &rdp->curlist; + } + + if (rdp->nxtlist && !rdp->curlist) { + local_irq_disable(); + rdp->curlist = rdp->nxtlist; + rdp->curtail = rdp->nxttail; + rdp->nxtlist = NULL; + rdp->nxttail = &rdp->nxtlist; + local_irq_enable(); + + /* + * start the next batch of callbacks + */ + + /* determine batch number */ + rdp->batch = rcp->cur + 1; + /* see the comment and corresponding wmb() in + * the rcu_start_batch() + */ + smp_rmb(); + + if (!rcp->next_pending) { + /* and start it/schedule start if it's a new batch */ + spin_lock(&rcp->lock); + rcp->next_pending = 1; + rcu_start_batch(rcp); + spin_unlock(&rcp->lock); + } + } + + rcu_check_quiescent_state(rcp, rdp); + if (rdp->donelist) + rcu_do_batch(rdp); +} + +static void rcu_process_callbacks(struct softirq_action *unused) +{ + __rcu_process_callbacks(&rcu_ctrlblk, &__get_cpu_var(rcu_data)); + __rcu_process_callbacks(&rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data)); +} + +static int __rcu_pending(struct rcu_ctrlblk *rcp, struct rcu_data *rdp) +{ + /* This cpu has pending rcu entries and the grace period + * for them has completed. + */ + if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) + return 1; + + /* This cpu has no pending entries, but there are new entries */ + if (!rdp->curlist && rdp->nxtlist) + return 1; + + /* This cpu has finished callbacks to invoke */ + if (rdp->donelist) + return 1; + + /* The rcu core waits for a quiescent state from the cpu */ + if (rdp->quiescbatch != rcp->cur || rdp->qs_pending) + return 1; + + /* nothing to do */ + return 0; +} + +/* + * Check to see if there is any immediate RCU-related work to be done + * by the current CPU, returning 1 if so. This function is part of the + * RCU implementation; it is -not- an exported member of the RCU API. + */ +int rcu_pending(int cpu) +{ + return __rcu_pending(&rcu_ctrlblk, &per_cpu(rcu_data, cpu)) || + __rcu_pending(&rcu_bh_ctrlblk, &per_cpu(rcu_bh_data, cpu)); +} + +/* + * Check to see if any future RCU-related work will need to be done + * by the current CPU, even if none need be done immediately, returning + * 1 if so. This function is part of the RCU implementation; it is -not- + * an exported member of the RCU API. + */ +int rcu_needs_cpu(int cpu) +{ + struct rcu_data *rdp = &per_cpu(rcu_data, cpu); + struct rcu_data *rdp_bh = &per_cpu(rcu_bh_data, cpu); + + return (!!rdp->curlist || !!rdp_bh->curlist || rcu_pending(cpu)); +} + +void rcu_check_callbacks(int cpu, int user) +{ + if (user || + (idle_cpu(cpu) && !in_softirq() && + hardirq_count() <= (1 << HARDIRQ_SHIFT))) { + rcu_qsctr_inc(cpu); + rcu_bh_qsctr_inc(cpu); + } else if (!in_softirq()) + rcu_bh_qsctr_inc(cpu); + raise_rcu_softirq(); +} + +static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp, + struct rcu_data *rdp) +{ + memset(rdp, 0, sizeof(*rdp)); + rdp->curtail = &rdp->curlist; + rdp->nxttail = &rdp->nxtlist; + rdp->donetail = &rdp->donelist; + rdp->quiescbatch = rcp->completed; + rdp->qs_pending = 0; + rdp->cpu = cpu; + rdp->blimit = blimit; +} + +static void __cpuinit rcu_online_cpu(int cpu) +{ + struct rcu_data *rdp = &per_cpu(rcu_data, cpu); + struct rcu_data *bh_rdp = &per_cpu(rcu_bh_data, cpu); + + rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp); + rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp); + open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL); +} + +static int __cpuinit rcu_cpu_notify(struct notifier_block *self, + unsigned long action, void *hcpu) +{ + long cpu = (long)hcpu; + + switch (action) { + case CPU_UP_PREPARE: + case CPU_UP_PREPARE_FROZEN: + rcu_online_cpu(cpu); + break; + case CPU_DEAD: + case CPU_DEAD_FROZEN: + rcu_offline_cpu(cpu); + break; + default: + break; + } + return NOTIFY_OK; +} + +static struct notifier_block __cpuinitdata rcu_nb = { + .notifier_call = rcu_cpu_notify, +}; + +/* + * Initializes rcu mechanism. Assumed to be called early. + * That is before local timer(SMP) or jiffie timer (uniproc) is setup. + * Note that rcu_qsctr and friends are implicitly + * initialized due to the choice of ``0'' for RCU_CTR_INVALID. + */ +void __init __rcu_init(void) +{ + rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, + (void *)(long)smp_processor_id()); + /* Register notifier for non-boot CPUs */ + register_cpu_notifier(&rcu_nb); +} + +module_param(blimit, int, 0); +module_param(qhimark, int, 0); +module_param(qlowmark, int, 0); diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c index 4dfa0b792ef..0ccd0095ebd 100644 --- a/kernel/rcupdate.c +++ b/kernel/rcupdate.c @@ -15,7 +15,7 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * - * Copyright (C) IBM Corporation, 2001 + * Copyright IBM Corporation, 2001 * * Authors: Dipankar Sarma * Manfred Spraul @@ -35,163 +35,57 @@ #include #include #include -#include #include #include #include #include -#include #include -#include #include #include #include #include +#include -#ifdef CONFIG_DEBUG_LOCK_ALLOC -static struct lock_class_key rcu_lock_key; -struct lockdep_map rcu_lock_map = - STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key); - -EXPORT_SYMBOL_GPL(rcu_lock_map); -#endif - -/* Definition for rcupdate control block. */ -static struct rcu_ctrlblk rcu_ctrlblk = { - .cur = -300, - .completed = -300, - .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock), - .cpumask = CPU_MASK_NONE, -}; -static struct rcu_ctrlblk rcu_bh_ctrlblk = { - .cur = -300, - .completed = -300, - .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock), - .cpumask = CPU_MASK_NONE, +struct rcu_synchronize { + struct rcu_head head; + struct completion completion; }; -DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L }; -DEFINE_PER_CPU(struct rcu_data, rcu_bh_data) = { 0L }; - -static int blimit = 10; -static int qhimark = 10000; -static int qlowmark = 100; - +static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL}; static atomic_t rcu_barrier_cpu_count; static DEFINE_MUTEX(rcu_barrier_mutex); static struct completion rcu_barrier_completion; -#ifdef CONFIG_SMP -static void force_quiescent_state(struct rcu_data *rdp, - struct rcu_ctrlblk *rcp) -{ - int cpu; - cpumask_t cpumask; - set_need_resched(); - if (unlikely(!rcp->signaled)) { - rcp->signaled = 1; - /* - * Don't send IPI to itself. With irqs disabled, - * rdp->cpu is the current cpu. - */ - cpumask = rcp->cpumask; - cpu_clear(rdp->cpu, cpumask); - for_each_cpu_mask(cpu, cpumask) - smp_send_reschedule(cpu); - } -} -#else -static inline void force_quiescent_state(struct rcu_data *rdp, - struct rcu_ctrlblk *rcp) +/* Because of FASTCALL declaration of complete, we use this wrapper */ +static void wakeme_after_rcu(struct rcu_head *head) { - set_need_resched(); + struct rcu_synchronize *rcu; + + rcu = container_of(head, struct rcu_synchronize, head); + complete(&rcu->completion); } -#endif /** - * call_rcu - Queue an RCU callback for invocation after a grace period. - * @head: structure to be used for queueing the RCU updates. - * @func: actual update function to be invoked after the grace period + * synchronize_rcu - wait until a grace period has elapsed. * - * The update function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU + * Control will return to the caller some time after a full grace + * period has elapsed, in other words after all currently executing RCU * read-side critical sections have completed. RCU read-side critical * sections are delimited by rcu_read_lock() and rcu_read_unlock(), * and may be nested. */ -void fastcall call_rcu(struct rcu_head *head, - void (*func)(struct rcu_head *rcu)) -{ - unsigned long flags; - struct rcu_data *rdp; - - head->func = func; - head->next = NULL; - local_irq_save(flags); - rdp = &__get_cpu_var(rcu_data); - *rdp->nxttail = head; - rdp->nxttail = &head->next; - if (unlikely(++rdp->qlen > qhimark)) { - rdp->blimit = INT_MAX; - force_quiescent_state(rdp, &rcu_ctrlblk); - } - local_irq_restore(flags); -} - -/** - * call_rcu_bh - Queue an RCU for invocation after a quicker grace period. - * @head: structure to be used for queueing the RCU updates. - * @func: actual update function to be invoked after the grace period - * - * The update function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU - * read-side critical sections have completed. call_rcu_bh() assumes - * that the read-side critical sections end on completion of a softirq - * handler. This means that read-side critical sections in process - * context must not be interrupted by softirqs. This interface is to be - * used when most of the read-side critical sections are in softirq context. - * RCU read-side critical sections are delimited by rcu_read_lock() and - * rcu_read_unlock(), * if in interrupt context or rcu_read_lock_bh() - * and rcu_read_unlock_bh(), if in process context. These may be nested. - */ -void fastcall call_rcu_bh(struct rcu_head *head, - void (*func)(struct rcu_head *rcu)) +void synchronize_rcu(void) { - unsigned long flags; - struct rcu_data *rdp; - - head->func = func; - head->next = NULL; - local_irq_save(flags); - rdp = &__get_cpu_var(rcu_bh_data); - *rdp->nxttail = head; - rdp->nxttail = &head->next; - - if (unlikely(++rdp->qlen > qhimark)) { - rdp->blimit = INT_MAX; - force_quiescent_state(rdp, &rcu_bh_ctrlblk); - } - - local_irq_restore(flags); -} + struct rcu_synchronize rcu; -/* - * Return the number of RCU batches processed thus far. Useful - * for debug and statistics. - */ -long rcu_batches_completed(void) -{ - return rcu_ctrlblk.completed; -} + init_completion(&rcu.completion); + /* Will wake me after RCU finished */ + call_rcu(&rcu.head, wakeme_after_rcu); -/* - * Return the number of RCU batches processed thus far. Useful - * for debug and statistics. - */ -long rcu_batches_completed_bh(void) -{ - return rcu_bh_ctrlblk.completed; + /* Wait for it */ + wait_for_completion(&rcu.completion); } +EXPORT_SYMBOL_GPL(synchronize_rcu); static void rcu_barrier_callback(struct rcu_head *notused) { @@ -205,10 +99,8 @@ static void rcu_barrier_callback(struct rcu_head *notused) static void rcu_barrier_func(void *notused) { int cpu = smp_processor_id(); - struct rcu_data *rdp = &per_cpu(rcu_data, cpu); - struct rcu_head *head; + struct rcu_head *head = &per_cpu(rcu_barrier_head, cpu); - head = &rdp->barrier; atomic_inc(&rcu_barrier_cpu_count); call_rcu(head, rcu_barrier_callback); } @@ -229,425 +121,8 @@ void rcu_barrier(void) } EXPORT_SYMBOL_GPL(rcu_barrier); -/* Raises the softirq for processing rcu_callbacks. */ -static inline void raise_rcu_softirq(void) -{ - raise_softirq(RCU_SOFTIRQ); - /* - * The smp_mb() here is required to ensure that this cpu's - * __rcu_process_callbacks() reads the most recently updated - * value of rcu->cur. - */ - smp_mb(); -} - -/* - * Invoke the completed RCU callbacks. They are expected to be in - * a per-cpu list. - */ -static void rcu_do_batch(struct rcu_data *rdp) -{ - struct rcu_head *next, *list; - int count = 0; - - list = rdp->donelist; - while (list) { - next = list->next; - prefetch(next); - list->func(list); - list = next; - if (++count >= rdp->blimit) - break; - } - rdp->donelist = list; - - local_irq_disable(); - rdp->qlen -= count; - local_irq_enable(); - if (rdp->blimit == INT_MAX && rdp->qlen <= qlowmark) - rdp->blimit = blimit; - - if (!rdp->donelist) - rdp->donetail = &rdp->donelist; - else - raise_rcu_softirq(); -} - -/* - * Grace period handling: - * The grace period handling consists out of two steps: - * - A new grace period is started. - * This is done by rcu_start_batch. The start is not broadcasted to - * all cpus, they must pick this up by comparing rcp->cur with - * rdp->quiescbatch. All cpus are recorded in the - * rcu_ctrlblk.cpumask bitmap. - * - All cpus must go through a quiescent state. - * Since the start of the grace period is not broadcasted, at least two - * calls to rcu_check_quiescent_state are required: - * The first call just notices that a new grace period is running. The - * following calls check if there was a quiescent state since the beginning - * of the grace period. If so, it updates rcu_ctrlblk.cpumask. If - * the bitmap is empty, then the grace period is completed. - * rcu_check_quiescent_state calls rcu_start_batch(0) to start the next grace - * period (if necessary). - */ -/* - * Register a new batch of callbacks, and start it up if there is currently no - * active batch and the batch to be registered has not already occurred. - * Caller must hold rcu_ctrlblk.lock. - */ -static void rcu_start_batch(struct rcu_ctrlblk *rcp) -{ - if (rcp->next_pending && - rcp->completed == rcp->cur) { - rcp->next_pending = 0; - /* - * next_pending == 0 must be visible in - * __rcu_process_callbacks() before it can see new value of cur. - */ - smp_wmb(); - rcp->cur++; - - /* - * Accessing nohz_cpu_mask before incrementing rcp->cur needs a - * Barrier Otherwise it can cause tickless idle CPUs to be - * included in rcp->cpumask, which will extend graceperiods - * unnecessarily. - */ - smp_mb(); - cpus_andnot(rcp->cpumask, cpu_online_map, nohz_cpu_mask); - - rcp->signaled = 0; - } -} - -/* - * cpu went through a quiescent state since the beginning of the grace period. - * Clear it from the cpu mask and complete the grace period if it was the last - * cpu. Start another grace period if someone has further entries pending - */ -static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp) -{ - cpu_clear(cpu, rcp->cpumask); - if (cpus_empty(rcp->cpumask)) { - /* batch completed ! */ - rcp->completed = rcp->cur; - rcu_start_batch(rcp); - } -} - -/* - * Check if the cpu has gone through a quiescent state (say context - * switch). If so and if it already hasn't done so in this RCU - * quiescent cycle, then indicate that it has done so. - */ -static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp, - struct rcu_data *rdp) -{ - if (rdp->quiescbatch != rcp->cur) { - /* start new grace period: */ - rdp->qs_pending = 1; - rdp->passed_quiesc = 0; - rdp->quiescbatch = rcp->cur; - return; - } - - /* Grace period already completed for this cpu? - * qs_pending is checked instead of the actual bitmap to avoid - * cacheline trashing. - */ - if (!rdp->qs_pending) - return; - - /* - * Was there a quiescent state since the beginning of the grace - * period? If no, then exit and wait for the next call. - */ - if (!rdp->passed_quiesc) - return; - rdp->qs_pending = 0; - - spin_lock(&rcp->lock); - /* - * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync - * during cpu startup. Ignore the quiescent state. - */ - if (likely(rdp->quiescbatch == rcp->cur)) - cpu_quiet(rdp->cpu, rcp); - - spin_unlock(&rcp->lock); -} - - -#ifdef CONFIG_HOTPLUG_CPU - -/* warning! helper for rcu_offline_cpu. do not use elsewhere without reviewing - * locking requirements, the list it's pulling from has to belong to a cpu - * which is dead and hence not processing interrupts. - */ -static void rcu_move_batch(struct rcu_data *this_rdp, struct rcu_head *list, - struct rcu_head **tail) -{ - local_irq_disable(); - *this_rdp->nxttail = list; - if (list) - this_rdp->nxttail = tail; - local_irq_enable(); -} - -static void __rcu_offline_cpu(struct rcu_data *this_rdp, - struct rcu_ctrlblk *rcp, struct rcu_data *rdp) -{ - /* if the cpu going offline owns the grace period - * we can block indefinitely waiting for it, so flush - * it here - */ - spin_lock_bh(&rcp->lock); - if (rcp->cur != rcp->completed) - cpu_quiet(rdp->cpu, rcp); - spin_unlock_bh(&rcp->lock); - rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail); - rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail); - rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail); -} - -static void rcu_offline_cpu(int cpu) -{ - struct rcu_data *this_rdp = &get_cpu_var(rcu_data); - struct rcu_data *this_bh_rdp = &get_cpu_var(rcu_bh_data); - - __rcu_offline_cpu(this_rdp, &rcu_ctrlblk, - &per_cpu(rcu_data, cpu)); - __rcu_offline_cpu(this_bh_rdp, &rcu_bh_ctrlblk, - &per_cpu(rcu_bh_data, cpu)); - put_cpu_var(rcu_data); - put_cpu_var(rcu_bh_data); -} - -#else - -static void rcu_offline_cpu(int cpu) -{ -} - -#endif - -/* - * This does the RCU processing work from softirq context. - */ -static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp, - struct rcu_data *rdp) -{ - if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) { - *rdp->donetail = rdp->curlist; - rdp->donetail = rdp->curtail; - rdp->curlist = NULL; - rdp->curtail = &rdp->curlist; - } - - if (rdp->nxtlist && !rdp->curlist) { - local_irq_disable(); - rdp->curlist = rdp->nxtlist; - rdp->curtail = rdp->nxttail; - rdp->nxtlist = NULL; - rdp->nxttail = &rdp->nxtlist; - local_irq_enable(); - - /* - * start the next batch of callbacks - */ - - /* determine batch number */ - rdp->batch = rcp->cur + 1; - /* see the comment and corresponding wmb() in - * the rcu_start_batch() - */ - smp_rmb(); - - if (!rcp->next_pending) { - /* and start it/schedule start if it's a new batch */ - spin_lock(&rcp->lock); - rcp->next_pending = 1; - rcu_start_batch(rcp); - spin_unlock(&rcp->lock); - } - } - - rcu_check_quiescent_state(rcp, rdp); - if (rdp->donelist) - rcu_do_batch(rdp); -} - -static void rcu_process_callbacks(struct softirq_action *unused) -{ - __rcu_process_callbacks(&rcu_ctrlblk, &__get_cpu_var(rcu_data)); - __rcu_process_callbacks(&rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data)); -} - -static int __rcu_pending(struct rcu_ctrlblk *rcp, struct rcu_data *rdp) -{ - /* This cpu has pending rcu entries and the grace period - * for them has completed. - */ - if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) - return 1; - - /* This cpu has no pending entries, but there are new entries */ - if (!rdp->curlist && rdp->nxtlist) - return 1; - - /* This cpu has finished callbacks to invoke */ - if (rdp->donelist) - return 1; - - /* The rcu core waits for a quiescent state from the cpu */ - if (rdp->quiescbatch != rcp->cur || rdp->qs_pending) - return 1; - - /* nothing to do */ - return 0; -} - -/* - * Check to see if there is any immediate RCU-related work to be done - * by the current CPU, returning 1 if so. This function is part of the - * RCU implementation; it is -not- an exported member of the RCU API. - */ -int rcu_pending(int cpu) -{ - return __rcu_pending(&rcu_ctrlblk, &per_cpu(rcu_data, cpu)) || - __rcu_pending(&rcu_bh_ctrlblk, &per_cpu(rcu_bh_data, cpu)); -} - -/* - * Check to see if any future RCU-related work will need to be done - * by the current CPU, even if none need be done immediately, returning - * 1 if so. This function is part of the RCU implementation; it is -not- - * an exported member of the RCU API. - */ -int rcu_needs_cpu(int cpu) -{ - struct rcu_data *rdp = &per_cpu(rcu_data, cpu); - struct rcu_data *rdp_bh = &per_cpu(rcu_bh_data, cpu); - - return (!!rdp->curlist || !!rdp_bh->curlist || rcu_pending(cpu)); -} - -void rcu_check_callbacks(int cpu, int user) -{ - if (user || - (idle_cpu(cpu) && !in_softirq() && - hardirq_count() <= (1 << HARDIRQ_SHIFT))) { - rcu_qsctr_inc(cpu); - rcu_bh_qsctr_inc(cpu); - } else if (!in_softirq()) - rcu_bh_qsctr_inc(cpu); - raise_rcu_softirq(); -} - -static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp, - struct rcu_data *rdp) -{ - memset(rdp, 0, sizeof(*rdp)); - rdp->curtail = &rdp->curlist; - rdp->nxttail = &rdp->nxtlist; - rdp->donetail = &rdp->donelist; - rdp->quiescbatch = rcp->completed; - rdp->qs_pending = 0; - rdp->cpu = cpu; - rdp->blimit = blimit; -} - -static void __cpuinit rcu_online_cpu(int cpu) -{ - struct rcu_data *rdp = &per_cpu(rcu_data, cpu); - struct rcu_data *bh_rdp = &per_cpu(rcu_bh_data, cpu); - - rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp); - rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp); - open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL); -} - -static int __cpuinit rcu_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - long cpu = (long)hcpu; - switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - rcu_online_cpu(cpu); - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - rcu_offline_cpu(cpu); - break; - default: - break; - } - return NOTIFY_OK; -} - -static struct notifier_block __cpuinitdata rcu_nb = { - .notifier_call = rcu_cpu_notify, -}; - -/* - * Initializes rcu mechanism. Assumed to be called early. - * That is before local timer(SMP) or jiffie timer (uniproc) is setup. - * Note that rcu_qsctr and friends are implicitly - * initialized due to the choice of ``0'' for RCU_CTR_INVALID. - */ void __init rcu_init(void) { - rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, - (void *)(long)smp_processor_id()); - /* Register notifier for non-boot CPUs */ - register_cpu_notifier(&rcu_nb); -} - -struct rcu_synchronize { - struct rcu_head head; - struct completion completion; -}; - -/* Because of FASTCALL declaration of complete, we use this wrapper */ -static void wakeme_after_rcu(struct rcu_head *head) -{ - struct rcu_synchronize *rcu; - - rcu = container_of(head, struct rcu_synchronize, head); - complete(&rcu->completion); -} - -/** - * synchronize_rcu - wait until a grace period has elapsed. - * - * Control will return to the caller some time after a full grace - * period has elapsed, in other words after all currently executing RCU - * read-side critical sections have completed. RCU read-side critical - * sections are delimited by rcu_read_lock() and rcu_read_unlock(), - * and may be nested. - * - * If your read-side code is not protected by rcu_read_lock(), do -not- - * use synchronize_rcu(). - */ -void synchronize_rcu(void) -{ - struct rcu_synchronize rcu; - - init_completion(&rcu.completion); - /* Will wake me after RCU finished */ - call_rcu(&rcu.head, wakeme_after_rcu); - - /* Wait for it */ - wait_for_completion(&rcu.completion); + __rcu_init(); } -module_param(blimit, int, 0); -module_param(qhimark, int, 0); -module_param(qlowmark, int, 0); -EXPORT_SYMBOL_GPL(rcu_batches_completed); -EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); -EXPORT_SYMBOL_GPL(call_rcu); -EXPORT_SYMBOL_GPL(call_rcu_bh); -EXPORT_SYMBOL_GPL(synchronize_rcu); -- cgit v1.2.3 From e0ecfa7917cafe72f4a75f87e8bb5d8d51dc534f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 25 Jan 2008 21:08:24 +0100 Subject: Preempt-RCU: fix rcu_barrier for preemptive environment. Fix rcu_barrier() to work properly in preemptive kernel environment. Also, the ordering of callback must be preserved while moving callbacks to another CPU during CPU hotplug. Signed-off-by: Gautham R Shenoy Signed-off-by: Dipankar Sarma Signed-off-by: Paul E. McKenney Reviewed-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/rcuclassic.c | 2 +- kernel/rcupdate.c | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c index 18369e3386e..ce0cf16cab6 100644 --- a/kernel/rcuclassic.c +++ b/kernel/rcuclassic.c @@ -371,9 +371,9 @@ static void __rcu_offline_cpu(struct rcu_data *this_rdp, if (rcp->cur != rcp->completed) cpu_quiet(rdp->cpu, rcp); spin_unlock_bh(&rcp->lock); + rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail); rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail); rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail); - rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail); } static void rcu_offline_cpu(int cpu) diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c index 0ccd0095ebd..760dfc233a0 100644 --- a/kernel/rcupdate.c +++ b/kernel/rcupdate.c @@ -115,7 +115,17 @@ void rcu_barrier(void) mutex_lock(&rcu_barrier_mutex); init_completion(&rcu_barrier_completion); atomic_set(&rcu_barrier_cpu_count, 0); + /* + * The queueing of callbacks in all CPUs must be atomic with + * respect to RCU, otherwise one CPU may queue a callback, + * wait for a grace period, decrement barrier count and call + * complete(), while other CPUs have not yet queued anything. + * So, we need to make sure that grace periods cannot complete + * until all the callbacks are queued. + */ + rcu_read_lock(); on_each_cpu(rcu_barrier_func, NULL, 0, 1); + rcu_read_unlock(); wait_for_completion(&rcu_barrier_completion); mutex_unlock(&rcu_barrier_mutex); } -- cgit v1.2.3 From e260be673a15b6125068270e0216a3bfbfc12f87 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 25 Jan 2008 21:08:24 +0100 Subject: Preempt-RCU: implementation This patch implements a new version of RCU which allows its read-side critical sections to be preempted. It uses a set of counter pairs to keep track of the read-side critical sections and flips them when all tasks exit read-side critical section. The details of this implementation can be found in this paper - http://www.rdrop.com/users/paulmck/RCU/OLSrtRCU.2006.08.11a.pdf and the article- http://lwn.net/Articles/253651/ This patch was developed as a part of the -rt kernel development and meant to provide better latencies when read-side critical sections of RCU don't disable preemption. As a consequence of keeping track of RCU readers, the readers have a slight overhead (optimizations in the paper). This implementation co-exists with the "classic" RCU implementations and can be switched to at compiler. Also includes RCU tracing summarized in debugfs. [ akpm@linux-foundation.org: build fixes on non-preempt architectures ] Signed-off-by: Gautham R Shenoy Signed-off-by: Dipankar Sarma Signed-off-by: Paul E. McKenney Reviewed-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/Kconfig.preempt | 10 + kernel/Makefile | 7 +- kernel/fork.c | 4 + kernel/rcuclassic.c | 1 - kernel/rcupreempt.c | 816 ++++++++++++++++++++++++++++++++++++++++++++++ kernel/rcupreempt_trace.c | 330 +++++++++++++++++++ 6 files changed, 1166 insertions(+), 2 deletions(-) create mode 100644 kernel/rcupreempt.c create mode 100644 kernel/rcupreempt_trace.c (limited to 'kernel') diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index c64ce9c1420..61fa116efcd 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -63,3 +63,13 @@ config PREEMPT_BKL Say Y here if you are building a kernel for a desktop system. Say N if you are unsure. +config RCU_TRACE + bool "Enable tracing for RCU - currently stats in debugfs" + select DEBUG_FS + default y + help + This option provides tracing in RCU which presents stats + in debugfs for debugging RCU implementation. + + Say Y here if you want to enable RCU tracing + Say N if you are unsure. diff --git a/kernel/Makefile b/kernel/Makefile index def5dd6097a..68755cd9a7e 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -6,7 +6,7 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \ exit.o itimer.o time.o softirq.o resource.o \ sysctl.o capability.o ptrace.o timer.o user.o user_namespace.o \ signal.o sys.o kmod.o workqueue.o pid.o \ - rcupdate.o rcuclassic.o extable.o params.o posix-timers.o \ + rcupdate.o extable.o params.o posix-timers.o \ kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ hrtimer.o rwsem.o latency.o nsproxy.o srcu.o \ utsname.o notifier.o @@ -52,6 +52,11 @@ obj-$(CONFIG_DETECT_SOFTLOCKUP) += softlockup.o obj-$(CONFIG_GENERIC_HARDIRQS) += irq/ obj-$(CONFIG_SECCOMP) += seccomp.o obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o +obj-$(CONFIG_CLASSIC_RCU) += rcuclassic.o +obj-$(CONFIG_PREEMPT_RCU) += rcupreempt.o +ifeq ($(CONFIG_PREEMPT_RCU),y) +obj-$(CONFIG_RCU_TRACE) += rcupreempt_trace.o +endif obj-$(CONFIG_RELAY) += relay.o obj-$(CONFIG_SYSCTL) += utsname_sysctl.o obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o diff --git a/kernel/fork.c b/kernel/fork.c index 930c51865ab..9f8ef32cbc7 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1045,6 +1045,10 @@ static struct task_struct *copy_process(unsigned long clone_flags, copy_flags(clone_flags, p); INIT_LIST_HEAD(&p->children); INIT_LIST_HEAD(&p->sibling); +#ifdef CONFIG_PREEMPT_RCU + p->rcu_read_lock_nesting = 0; + p->rcu_flipctr_idx = 0; +#endif /* #ifdef CONFIG_PREEMPT_RCU */ p->vfork_done = NULL; spin_lock_init(&p->alloc_lock); diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c index ce0cf16cab6..f4ffbd0f306 100644 --- a/kernel/rcuclassic.c +++ b/kernel/rcuclassic.c @@ -45,7 +45,6 @@ #include #include #include -/* #include @@@ */ #include #include diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c new file mode 100644 index 00000000000..a5aabb1677f --- /dev/null +++ b/kernel/rcupreempt.c @@ -0,0 +1,816 @@ +/* + * Read-Copy Update mechanism for mutual exclusion, realtime implementation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright IBM Corporation, 2006 + * + * Authors: Paul E. McKenney + * With thanks to Esben Nielsen, Bill Huey, and Ingo Molnar + * for pushing me away from locks and towards counters, and + * to Suparna Bhattacharya for pushing me completely away + * from atomic instructions on the read side. + * + * Papers: http://www.rdrop.com/users/paulmck/RCU + * + * Design Document: http://lwn.net/Articles/253651/ + * + * For detailed explanation of Read-Copy Update mechanism see - + * Documentation/RCU/ *.txt + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Macro that prevents the compiler from reordering accesses, but does + * absolutely -nothing- to prevent CPUs from reordering. This is used + * only to mediate communication between mainline code and hardware + * interrupt and NMI handlers. + */ +#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x)) + +/* + * PREEMPT_RCU data structures. + */ + +/* + * GP_STAGES specifies the number of times the state machine has + * to go through the all the rcu_try_flip_states (see below) + * in a single Grace Period. + * + * GP in GP_STAGES stands for Grace Period ;) + */ +#define GP_STAGES 2 +struct rcu_data { + spinlock_t lock; /* Protect rcu_data fields. */ + long completed; /* Number of last completed batch. */ + int waitlistcount; + struct tasklet_struct rcu_tasklet; + struct rcu_head *nextlist; + struct rcu_head **nexttail; + struct rcu_head *waitlist[GP_STAGES]; + struct rcu_head **waittail[GP_STAGES]; + struct rcu_head *donelist; + struct rcu_head **donetail; + long rcu_flipctr[2]; +#ifdef CONFIG_RCU_TRACE + struct rcupreempt_trace trace; +#endif /* #ifdef CONFIG_RCU_TRACE */ +}; + +/* + * States for rcu_try_flip() and friends. + */ + +enum rcu_try_flip_states { + + /* + * Stay here if nothing is happening. Flip the counter if somthing + * starts happening. Denoted by "I" + */ + rcu_try_flip_idle_state, + + /* + * Wait here for all CPUs to notice that the counter has flipped. This + * prevents the old set of counters from ever being incremented once + * we leave this state, which in turn is necessary because we cannot + * test any individual counter for zero -- we can only check the sum. + * Denoted by "A". + */ + rcu_try_flip_waitack_state, + + /* + * Wait here for the sum of the old per-CPU counters to reach zero. + * Denoted by "Z". + */ + rcu_try_flip_waitzero_state, + + /* + * Wait here for each of the other CPUs to execute a memory barrier. + * This is necessary to ensure that these other CPUs really have + * completed executing their RCU read-side critical sections, despite + * their CPUs wildly reordering memory. Denoted by "M". + */ + rcu_try_flip_waitmb_state, +}; + +struct rcu_ctrlblk { + spinlock_t fliplock; /* Protect state-machine transitions. */ + long completed; /* Number of last completed batch. */ + enum rcu_try_flip_states rcu_try_flip_state; /* The current state of + the rcu state machine */ +}; + +static DEFINE_PER_CPU(struct rcu_data, rcu_data); +static struct rcu_ctrlblk rcu_ctrlblk = { + .fliplock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.fliplock), + .completed = 0, + .rcu_try_flip_state = rcu_try_flip_idle_state, +}; + + +#ifdef CONFIG_RCU_TRACE +static char *rcu_try_flip_state_names[] = + { "idle", "waitack", "waitzero", "waitmb" }; +#endif /* #ifdef CONFIG_RCU_TRACE */ + +/* + * Enum and per-CPU flag to determine when each CPU has seen + * the most recent counter flip. + */ + +enum rcu_flip_flag_values { + rcu_flip_seen, /* Steady/initial state, last flip seen. */ + /* Only GP detector can update. */ + rcu_flipped /* Flip just completed, need confirmation. */ + /* Only corresponding CPU can update. */ +}; +static DEFINE_PER_CPU_SHARED_ALIGNED(enum rcu_flip_flag_values, rcu_flip_flag) + = rcu_flip_seen; + +/* + * Enum and per-CPU flag to determine when each CPU has executed the + * needed memory barrier to fence in memory references from its last RCU + * read-side critical section in the just-completed grace period. + */ + +enum rcu_mb_flag_values { + rcu_mb_done, /* Steady/initial state, no mb()s required. */ + /* Only GP detector can update. */ + rcu_mb_needed /* Flip just completed, need an mb(). */ + /* Only corresponding CPU can update. */ +}; +static DEFINE_PER_CPU_SHARED_ALIGNED(enum rcu_mb_flag_values, rcu_mb_flag) + = rcu_mb_done; + +/* + * RCU_DATA_ME: find the current CPU's rcu_data structure. + * RCU_DATA_CPU: find the specified CPU's rcu_data structure. + */ +#define RCU_DATA_ME() (&__get_cpu_var(rcu_data)) +#define RCU_DATA_CPU(cpu) (&per_cpu(rcu_data, cpu)) + +/* + * Helper macro for tracing when the appropriate rcu_data is not + * cached in a local variable, but where the CPU number is so cached. + */ +#define RCU_TRACE_CPU(f, cpu) RCU_TRACE(f, &(RCU_DATA_CPU(cpu)->trace)); + +/* + * Helper macro for tracing when the appropriate rcu_data is not + * cached in a local variable. + */ +#define RCU_TRACE_ME(f) RCU_TRACE(f, &(RCU_DATA_ME()->trace)); + +/* + * Helper macro for tracing when the appropriate rcu_data is pointed + * to by a local variable. + */ +#define RCU_TRACE_RDP(f, rdp) RCU_TRACE(f, &((rdp)->trace)); + +/* + * Return the number of RCU batches processed thus far. Useful + * for debug and statistics. + */ +long rcu_batches_completed(void) +{ + return rcu_ctrlblk.completed; +} +EXPORT_SYMBOL_GPL(rcu_batches_completed); + +EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); + +void __rcu_read_lock(void) +{ + int idx; + struct task_struct *t = current; + int nesting; + + nesting = ACCESS_ONCE(t->rcu_read_lock_nesting); + if (nesting != 0) { + + /* An earlier rcu_read_lock() covers us, just count it. */ + + t->rcu_read_lock_nesting = nesting + 1; + + } else { + unsigned long flags; + + /* + * We disable interrupts for the following reasons: + * - If we get scheduling clock interrupt here, and we + * end up acking the counter flip, it's like a promise + * that we will never increment the old counter again. + * Thus we will break that promise if that + * scheduling clock interrupt happens between the time + * we pick the .completed field and the time that we + * increment our counter. + * + * - We don't want to be preempted out here. + * + * NMIs can still occur, of course, and might themselves + * contain rcu_read_lock(). + */ + + local_irq_save(flags); + + /* + * Outermost nesting of rcu_read_lock(), so increment + * the current counter for the current CPU. Use volatile + * casts to prevent the compiler from reordering. + */ + + idx = ACCESS_ONCE(rcu_ctrlblk.completed) & 0x1; + ACCESS_ONCE(RCU_DATA_ME()->rcu_flipctr[idx])++; + + /* + * Now that the per-CPU counter has been incremented, we + * are protected from races with rcu_read_lock() invoked + * from NMI handlers on this CPU. We can therefore safely + * increment the nesting counter, relieving further NMIs + * of the need to increment the per-CPU counter. + */ + + ACCESS_ONCE(t->rcu_read_lock_nesting) = nesting + 1; + + /* + * Now that we have preventing any NMIs from storing + * to the ->rcu_flipctr_idx, we can safely use it to + * remember which counter to decrement in the matching + * rcu_read_unlock(). + */ + + ACCESS_ONCE(t->rcu_flipctr_idx) = idx; + local_irq_restore(flags); + } +} +EXPORT_SYMBOL_GPL(__rcu_read_lock); + +void __rcu_read_unlock(void) +{ + int idx; + struct task_struct *t = current; + int nesting; + + nesting = ACCESS_ONCE(t->rcu_read_lock_nesting); + if (nesting > 1) { + + /* + * We are still protected by the enclosing rcu_read_lock(), + * so simply decrement the counter. + */ + + t->rcu_read_lock_nesting = nesting - 1; + + } else { + unsigned long flags; + + /* + * Disable local interrupts to prevent the grace-period + * detection state machine from seeing us half-done. + * NMIs can still occur, of course, and might themselves + * contain rcu_read_lock() and rcu_read_unlock(). + */ + + local_irq_save(flags); + + /* + * Outermost nesting of rcu_read_unlock(), so we must + * decrement the current counter for the current CPU. + * This must be done carefully, because NMIs can + * occur at any point in this code, and any rcu_read_lock() + * and rcu_read_unlock() pairs in the NMI handlers + * must interact non-destructively with this code. + * Lots of volatile casts, and -very- careful ordering. + * + * Changes to this code, including this one, must be + * inspected, validated, and tested extremely carefully!!! + */ + + /* + * First, pick up the index. + */ + + idx = ACCESS_ONCE(t->rcu_flipctr_idx); + + /* + * Now that we have fetched the counter index, it is + * safe to decrement the per-task RCU nesting counter. + * After this, any interrupts or NMIs will increment and + * decrement the per-CPU counters. + */ + ACCESS_ONCE(t->rcu_read_lock_nesting) = nesting - 1; + + /* + * It is now safe to decrement this task's nesting count. + * NMIs that occur after this statement will route their + * rcu_read_lock() calls through this "else" clause, and + * will thus start incrementing the per-CPU counter on + * their own. They will also clobber ->rcu_flipctr_idx, + * but that is OK, since we have already fetched it. + */ + + ACCESS_ONCE(RCU_DATA_ME()->rcu_flipctr[idx])--; + local_irq_restore(flags); + } +} +EXPORT_SYMBOL_GPL(__rcu_read_unlock); + +/* + * If a global counter flip has occurred since the last time that we + * advanced callbacks, advance them. Hardware interrupts must be + * disabled when calling this function. + */ +static void __rcu_advance_callbacks(struct rcu_data *rdp) +{ + int cpu; + int i; + int wlc = 0; + + if (rdp->completed != rcu_ctrlblk.completed) { + if (rdp->waitlist[GP_STAGES - 1] != NULL) { + *rdp->donetail = rdp->waitlist[GP_STAGES - 1]; + rdp->donetail = rdp->waittail[GP_STAGES - 1]; + RCU_TRACE_RDP(rcupreempt_trace_move2done, rdp); + } + for (i = GP_STAGES - 2; i >= 0; i--) { + if (rdp->waitlist[i] != NULL) { + rdp->waitlist[i + 1] = rdp->waitlist[i]; + rdp->waittail[i + 1] = rdp->waittail[i]; + wlc++; + } else { + rdp->waitlist[i + 1] = NULL; + rdp->waittail[i + 1] = + &rdp->waitlist[i + 1]; + } + } + if (rdp->nextlist != NULL) { + rdp->waitlist[0] = rdp->nextlist; + rdp->waittail[0] = rdp->nexttail; + wlc++; + rdp->nextlist = NULL; + rdp->nexttail = &rdp->nextlist; + RCU_TRACE_RDP(rcupreempt_trace_move2wait, rdp); + } else { + rdp->waitlist[0] = NULL; + rdp->waittail[0] = &rdp->waitlist[0]; + } + rdp->waitlistcount = wlc; + rdp->completed = rcu_ctrlblk.completed; + } + + /* + * Check to see if this CPU needs to report that it has seen + * the most recent counter flip, thereby declaring that all + * subsequent rcu_read_lock() invocations will respect this flip. + */ + + cpu = raw_smp_processor_id(); + if (per_cpu(rcu_flip_flag, cpu) == rcu_flipped) { + smp_mb(); /* Subsequent counter accesses must see new value */ + per_cpu(rcu_flip_flag, cpu) = rcu_flip_seen; + smp_mb(); /* Subsequent RCU read-side critical sections */ + /* seen -after- acknowledgement. */ + } +} + +/* + * Get here when RCU is idle. Decide whether we need to + * move out of idle state, and return non-zero if so. + * "Straightforward" approach for the moment, might later + * use callback-list lengths, grace-period duration, or + * some such to determine when to exit idle state. + * Might also need a pre-idle test that does not acquire + * the lock, but let's get the simple case working first... + */ + +static int +rcu_try_flip_idle(void) +{ + int cpu; + + RCU_TRACE_ME(rcupreempt_trace_try_flip_i1); + if (!rcu_pending(smp_processor_id())) { + RCU_TRACE_ME(rcupreempt_trace_try_flip_ie1); + return 0; + } + + /* + * Do the flip. + */ + + RCU_TRACE_ME(rcupreempt_trace_try_flip_g1); + rcu_ctrlblk.completed++; /* stands in for rcu_try_flip_g2 */ + + /* + * Need a memory barrier so that other CPUs see the new + * counter value before they see the subsequent change of all + * the rcu_flip_flag instances to rcu_flipped. + */ + + smp_mb(); /* see above block comment. */ + + /* Now ask each CPU for acknowledgement of the flip. */ + + for_each_possible_cpu(cpu) + per_cpu(rcu_flip_flag, cpu) = rcu_flipped; + + return 1; +} + +/* + * Wait for CPUs to acknowledge the flip. + */ + +static int +rcu_try_flip_waitack(void) +{ + int cpu; + + RCU_TRACE_ME(rcupreempt_trace_try_flip_a1); + for_each_possible_cpu(cpu) + if (per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) { + RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1); + return 0; + } + + /* + * Make sure our checks above don't bleed into subsequent + * waiting for the sum of the counters to reach zero. + */ + + smp_mb(); /* see above block comment. */ + RCU_TRACE_ME(rcupreempt_trace_try_flip_a2); + return 1; +} + +/* + * Wait for collective ``last'' counter to reach zero, + * then tell all CPUs to do an end-of-grace-period memory barrier. + */ + +static int +rcu_try_flip_waitzero(void) +{ + int cpu; + int lastidx = !(rcu_ctrlblk.completed & 0x1); + int sum = 0; + + /* Check to see if the sum of the "last" counters is zero. */ + + RCU_TRACE_ME(rcupreempt_trace_try_flip_z1); + for_each_possible_cpu(cpu) + sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx]; + if (sum != 0) { + RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1); + return 0; + } + + /* + * This ensures that the other CPUs see the call for + * memory barriers -after- the sum to zero has been + * detected here + */ + smp_mb(); /* ^^^^^^^^^^^^ */ + + /* Call for a memory barrier from each CPU. */ + for_each_possible_cpu(cpu) + per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed; + + RCU_TRACE_ME(rcupreempt_trace_try_flip_z2); + return 1; +} + +/* + * Wait for all CPUs to do their end-of-grace-period memory barrier. + * Return 0 once all CPUs have done so. + */ + +static int +rcu_try_flip_waitmb(void) +{ + int cpu; + + RCU_TRACE_ME(rcupreempt_trace_try_flip_m1); + for_each_possible_cpu(cpu) + if (per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) { + RCU_TRACE_ME(rcupreempt_trace_try_flip_me1); + return 0; + } + + smp_mb(); /* Ensure that the above checks precede any following flip. */ + RCU_TRACE_ME(rcupreempt_trace_try_flip_m2); + return 1; +} + +/* + * Attempt a single flip of the counters. Remember, a single flip does + * -not- constitute a grace period. Instead, the interval between + * at least GP_STAGES consecutive flips is a grace period. + * + * If anyone is nuts enough to run this CONFIG_PREEMPT_RCU implementation + * on a large SMP, they might want to use a hierarchical organization of + * the per-CPU-counter pairs. + */ +static void rcu_try_flip(void) +{ + unsigned long flags; + + RCU_TRACE_ME(rcupreempt_trace_try_flip_1); + if (unlikely(!spin_trylock_irqsave(&rcu_ctrlblk.fliplock, flags))) { + RCU_TRACE_ME(rcupreempt_trace_try_flip_e1); + return; + } + + /* + * Take the next transition(s) through the RCU grace-period + * flip-counter state machine. + */ + + switch (rcu_ctrlblk.rcu_try_flip_state) { + case rcu_try_flip_idle_state: + if (rcu_try_flip_idle()) + rcu_ctrlblk.rcu_try_flip_state = + rcu_try_flip_waitack_state; + break; + case rcu_try_flip_waitack_state: + if (rcu_try_flip_waitack()) + rcu_ctrlblk.rcu_try_flip_state = + rcu_try_flip_waitzero_state; + break; + case rcu_try_flip_waitzero_state: + if (rcu_try_flip_waitzero()) + rcu_ctrlblk.rcu_try_flip_state = + rcu_try_flip_waitmb_state; + break; + case rcu_try_flip_waitmb_state: + if (rcu_try_flip_waitmb()) + rcu_ctrlblk.rcu_try_flip_state = + rcu_try_flip_idle_state; + } + spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); +} + +/* + * Check to see if this CPU needs to do a memory barrier in order to + * ensure that any prior RCU read-side critical sections have committed + * their counter manipulations and critical-section memory references + * before declaring the grace period to be completed. + */ +static void rcu_check_mb(int cpu) +{ + if (per_cpu(rcu_mb_flag, cpu) == rcu_mb_needed) { + smp_mb(); /* Ensure RCU read-side accesses are visible. */ + per_cpu(rcu_mb_flag, cpu) = rcu_mb_done; + } +} + +void rcu_check_callbacks(int cpu, int user) +{ + unsigned long flags; + struct rcu_data *rdp = RCU_DATA_CPU(cpu); + + rcu_check_mb(cpu); + if (rcu_ctrlblk.completed == rdp->completed) + rcu_try_flip(); + spin_lock_irqsave(&rdp->lock, flags); + RCU_TRACE_RDP(rcupreempt_trace_check_callbacks, rdp); + __rcu_advance_callbacks(rdp); + if (rdp->donelist == NULL) { + spin_unlock_irqrestore(&rdp->lock, flags); + } else { + spin_unlock_irqrestore(&rdp->lock, flags); + raise_softirq(RCU_SOFTIRQ); + } +} + +/* + * Needed by dynticks, to make sure all RCU processing has finished + * when we go idle: + */ +void rcu_advance_callbacks(int cpu, int user) +{ + unsigned long flags; + struct rcu_data *rdp = RCU_DATA_CPU(cpu); + + if (rcu_ctrlblk.completed == rdp->completed) { + rcu_try_flip(); + if (rcu_ctrlblk.completed == rdp->completed) + return; + } + spin_lock_irqsave(&rdp->lock, flags); + RCU_TRACE_RDP(rcupreempt_trace_check_callbacks, rdp); + __rcu_advance_callbacks(rdp); + spin_unlock_irqrestore(&rdp->lock, flags); +} + +static void rcu_process_callbacks(struct softirq_action *unused) +{ + unsigned long flags; + struct rcu_head *next, *list; + struct rcu_data *rdp = RCU_DATA_ME(); + + spin_lock_irqsave(&rdp->lock, flags); + list = rdp->donelist; + if (list == NULL) { + spin_unlock_irqrestore(&rdp->lock, flags); + return; + } + rdp->donelist = NULL; + rdp->donetail = &rdp->donelist; + RCU_TRACE_RDP(rcupreempt_trace_done_remove, rdp); + spin_unlock_irqrestore(&rdp->lock, flags); + while (list) { + next = list->next; + list->func(list); + list = next; + RCU_TRACE_ME(rcupreempt_trace_invoke); + } +} + +void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) +{ + unsigned long flags; + struct rcu_data *rdp; + + head->func = func; + head->next = NULL; + local_irq_save(flags); + rdp = RCU_DATA_ME(); + spin_lock(&rdp->lock); + __rcu_advance_callbacks(rdp); + *rdp->nexttail = head; + rdp->nexttail = &head->next; + RCU_TRACE_RDP(rcupreempt_trace_next_add, rdp); + spin_unlock(&rdp->lock); + local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(call_rcu); + +/* + * Wait until all currently running preempt_disable() code segments + * (including hardware-irq-disable segments) complete. Note that + * in -rt this does -not- necessarily result in all currently executing + * interrupt -handlers- having completed. + */ +void __synchronize_sched(void) +{ + cpumask_t oldmask; + int cpu; + + if (sched_getaffinity(0, &oldmask) < 0) + oldmask = cpu_possible_map; + for_each_online_cpu(cpu) { + sched_setaffinity(0, cpumask_of_cpu(cpu)); + schedule(); + } + sched_setaffinity(0, oldmask); +} +EXPORT_SYMBOL_GPL(__synchronize_sched); + +/* + * Check to see if any future RCU-related work will need to be done + * by the current CPU, even if none need be done immediately, returning + * 1 if so. Assumes that notifiers would take care of handling any + * outstanding requests from the RCU core. + * + * This function is part of the RCU implementation; it is -not- + * an exported member of the RCU API. + */ +int rcu_needs_cpu(int cpu) +{ + struct rcu_data *rdp = RCU_DATA_CPU(cpu); + + return (rdp->donelist != NULL || + !!rdp->waitlistcount || + rdp->nextlist != NULL); +} + +int rcu_pending(int cpu) +{ + struct rcu_data *rdp = RCU_DATA_CPU(cpu); + + /* The CPU has at least one callback queued somewhere. */ + + if (rdp->donelist != NULL || + !!rdp->waitlistcount || + rdp->nextlist != NULL) + return 1; + + /* The RCU core needs an acknowledgement from this CPU. */ + + if ((per_cpu(rcu_flip_flag, cpu) == rcu_flipped) || + (per_cpu(rcu_mb_flag, cpu) == rcu_mb_needed)) + return 1; + + /* This CPU has fallen behind the global grace-period number. */ + + if (rdp->completed != rcu_ctrlblk.completed) + return 1; + + /* Nothing needed from this CPU. */ + + return 0; +} + +void __init __rcu_init(void) +{ + int cpu; + int i; + struct rcu_data *rdp; + + printk(KERN_NOTICE "Preemptible RCU implementation.\n"); + for_each_possible_cpu(cpu) { + rdp = RCU_DATA_CPU(cpu); + spin_lock_init(&rdp->lock); + rdp->completed = 0; + rdp->waitlistcount = 0; + rdp->nextlist = NULL; + rdp->nexttail = &rdp->nextlist; + for (i = 0; i < GP_STAGES; i++) { + rdp->waitlist[i] = NULL; + rdp->waittail[i] = &rdp->waitlist[i]; + } + rdp->donelist = NULL; + rdp->donetail = &rdp->donelist; + rdp->rcu_flipctr[0] = 0; + rdp->rcu_flipctr[1] = 0; + } + open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL); +} + +/* + * Deprecated, use synchronize_rcu() or synchronize_sched() instead. + */ +void synchronize_kernel(void) +{ + synchronize_rcu(); +} + +#ifdef CONFIG_RCU_TRACE +long *rcupreempt_flipctr(int cpu) +{ + return &RCU_DATA_CPU(cpu)->rcu_flipctr[0]; +} +EXPORT_SYMBOL_GPL(rcupreempt_flipctr); + +int rcupreempt_flip_flag(int cpu) +{ + return per_cpu(rcu_flip_flag, cpu); +} +EXPORT_SYMBOL_GPL(rcupreempt_flip_flag); + +int rcupreempt_mb_flag(int cpu) +{ + return per_cpu(rcu_mb_flag, cpu); +} +EXPORT_SYMBOL_GPL(rcupreempt_mb_flag); + +char *rcupreempt_try_flip_state_name(void) +{ + return rcu_try_flip_state_names[rcu_ctrlblk.rcu_try_flip_state]; +} +EXPORT_SYMBOL_GPL(rcupreempt_try_flip_state_name); + +struct rcupreempt_trace *rcupreempt_trace_cpu(int cpu) +{ + struct rcu_data *rdp = RCU_DATA_CPU(cpu); + + return &rdp->trace; +} +EXPORT_SYMBOL_GPL(rcupreempt_trace_cpu); + +#endif /* #ifdef RCU_TRACE */ diff --git a/kernel/rcupreempt_trace.c b/kernel/rcupreempt_trace.c new file mode 100644 index 00000000000..49ac4947af2 --- /dev/null +++ b/kernel/rcupreempt_trace.c @@ -0,0 +1,330 @@ +/* + * Read-Copy Update tracing for realtime implementation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright IBM Corporation, 2006 + * + * Papers: http://www.rdrop.com/users/paulmck/RCU + * + * For detailed explanation of Read-Copy Update mechanism see - + * Documentation/RCU/ *.txt + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct mutex rcupreempt_trace_mutex; +static char *rcupreempt_trace_buf; +#define RCUPREEMPT_TRACE_BUF_SIZE 4096 + +void rcupreempt_trace_move2done(struct rcupreempt_trace *trace) +{ + trace->done_length += trace->wait_length; + trace->done_add += trace->wait_length; + trace->wait_length = 0; +} +void rcupreempt_trace_move2wait(struct rcupreempt_trace *trace) +{ + trace->wait_length += trace->next_length; + trace->wait_add += trace->next_length; + trace->next_length = 0; +} +void rcupreempt_trace_try_flip_1(struct rcupreempt_trace *trace) +{ + atomic_inc(&trace->rcu_try_flip_1); +} +void rcupreempt_trace_try_flip_e1(struct rcupreempt_trace *trace) +{ + atomic_inc(&trace->rcu_try_flip_e1); +} +void rcupreempt_trace_try_flip_i1(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_i1++; +} +void rcupreempt_trace_try_flip_ie1(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_ie1++; +} +void rcupreempt_trace_try_flip_g1(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_g1++; +} +void rcupreempt_trace_try_flip_a1(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_a1++; +} +void rcupreempt_trace_try_flip_ae1(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_ae1++; +} +void rcupreempt_trace_try_flip_a2(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_a2++; +} +void rcupreempt_trace_try_flip_z1(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_z1++; +} +void rcupreempt_trace_try_flip_ze1(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_ze1++; +} +void rcupreempt_trace_try_flip_z2(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_z2++; +} +void rcupreempt_trace_try_flip_m1(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_m1++; +} +void rcupreempt_trace_try_flip_me1(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_me1++; +} +void rcupreempt_trace_try_flip_m2(struct rcupreempt_trace *trace) +{ + trace->rcu_try_flip_m2++; +} +void rcupreempt_trace_check_callbacks(struct rcupreempt_trace *trace) +{ + trace->rcu_check_callbacks++; +} +void rcupreempt_trace_done_remove(struct rcupreempt_trace *trace) +{ + trace->done_remove += trace->done_length; + trace->done_length = 0; +} +void rcupreempt_trace_invoke(struct rcupreempt_trace *trace) +{ + atomic_inc(&trace->done_invoked); +} +void rcupreempt_trace_next_add(struct rcupreempt_trace *trace) +{ + trace->next_add++; + trace->next_length++; +} + +static void rcupreempt_trace_sum(struct rcupreempt_trace *sp) +{ + struct rcupreempt_trace *cp; + int cpu; + + memset(sp, 0, sizeof(*sp)); + for_each_possible_cpu(cpu) { + cp = rcupreempt_trace_cpu(cpu); + sp->next_length += cp->next_length; + sp->next_add += cp->next_add; + sp->wait_length += cp->wait_length; + sp->wait_add += cp->wait_add; + sp->done_length += cp->done_length; + sp->done_add += cp->done_add; + sp->done_remove += cp->done_remove; + atomic_set(&sp->done_invoked, atomic_read(&cp->done_invoked)); + sp->rcu_check_callbacks += cp->rcu_check_callbacks; + atomic_set(&sp->rcu_try_flip_1, + atomic_read(&cp->rcu_try_flip_1)); + atomic_set(&sp->rcu_try_flip_e1, + atomic_read(&cp->rcu_try_flip_e1)); + sp->rcu_try_flip_i1 += cp->rcu_try_flip_i1; + sp->rcu_try_flip_ie1 += cp->rcu_try_flip_ie1; + sp->rcu_try_flip_g1 += cp->rcu_try_flip_g1; + sp->rcu_try_flip_a1 += cp->rcu_try_flip_a1; + sp->rcu_try_flip_ae1 += cp->rcu_try_flip_ae1; + sp->rcu_try_flip_a2 += cp->rcu_try_flip_a2; + sp->rcu_try_flip_z1 += cp->rcu_try_flip_z1; + sp->rcu_try_flip_ze1 += cp->rcu_try_flip_ze1; + sp->rcu_try_flip_z2 += cp->rcu_try_flip_z2; + sp->rcu_try_flip_m1 += cp->rcu_try_flip_m1; + sp->rcu_try_flip_me1 += cp->rcu_try_flip_me1; + sp->rcu_try_flip_m2 += cp->rcu_try_flip_m2; + } +} + +static ssize_t rcustats_read(struct file *filp, char __user *buffer, + size_t count, loff_t *ppos) +{ + struct rcupreempt_trace trace; + ssize_t bcount; + int cnt = 0; + + rcupreempt_trace_sum(&trace); + mutex_lock(&rcupreempt_trace_mutex); + snprintf(&rcupreempt_trace_buf[cnt], RCUPREEMPT_TRACE_BUF_SIZE - cnt, + "ggp=%ld rcc=%ld\n", + rcu_batches_completed(), + trace.rcu_check_callbacks); + snprintf(&rcupreempt_trace_buf[cnt], RCUPREEMPT_TRACE_BUF_SIZE - cnt, + "na=%ld nl=%ld wa=%ld wl=%ld da=%ld dl=%ld dr=%ld di=%d\n" + "1=%d e1=%d i1=%ld ie1=%ld g1=%ld a1=%ld ae1=%ld a2=%ld\n" + "z1=%ld ze1=%ld z2=%ld m1=%ld me1=%ld m2=%ld\n", + + trace.next_add, trace.next_length, + trace.wait_add, trace.wait_length, + trace.done_add, trace.done_length, + trace.done_remove, atomic_read(&trace.done_invoked), + atomic_read(&trace.rcu_try_flip_1), + atomic_read(&trace.rcu_try_flip_e1), + trace.rcu_try_flip_i1, trace.rcu_try_flip_ie1, + trace.rcu_try_flip_g1, + trace.rcu_try_flip_a1, trace.rcu_try_flip_ae1, + trace.rcu_try_flip_a2, + trace.rcu_try_flip_z1, trace.rcu_try_flip_ze1, + trace.rcu_try_flip_z2, + trace.rcu_try_flip_m1, trace.rcu_try_flip_me1, + trace.rcu_try_flip_m2); + bcount = simple_read_from_buffer(buffer, count, ppos, + rcupreempt_trace_buf, strlen(rcupreempt_trace_buf)); + mutex_unlock(&rcupreempt_trace_mutex); + return bcount; +} + +static ssize_t rcugp_read(struct file *filp, char __user *buffer, + size_t count, loff_t *ppos) +{ + long oldgp = rcu_batches_completed(); + ssize_t bcount; + + mutex_lock(&rcupreempt_trace_mutex); + synchronize_rcu(); + snprintf(rcupreempt_trace_buf, RCUPREEMPT_TRACE_BUF_SIZE, + "oldggp=%ld newggp=%ld\n", oldgp, rcu_batches_completed()); + bcount = simple_read_from_buffer(buffer, count, ppos, + rcupreempt_trace_buf, strlen(rcupreempt_trace_buf)); + mutex_unlock(&rcupreempt_trace_mutex); + return bcount; +} + +static ssize_t rcuctrs_read(struct file *filp, char __user *buffer, + size_t count, loff_t *ppos) +{ + int cnt = 0; + int cpu; + int f = rcu_batches_completed() & 0x1; + ssize_t bcount; + + mutex_lock(&rcupreempt_trace_mutex); + + cnt += snprintf(&rcupreempt_trace_buf[cnt], RCUPREEMPT_TRACE_BUF_SIZE, + "CPU last cur F M\n"); + for_each_online_cpu(cpu) { + long *flipctr = rcupreempt_flipctr(cpu); + cnt += snprintf(&rcupreempt_trace_buf[cnt], + RCUPREEMPT_TRACE_BUF_SIZE - cnt, + "%3d %4ld %3ld %d %d\n", + cpu, + flipctr[!f], + flipctr[f], + rcupreempt_flip_flag(cpu), + rcupreempt_mb_flag(cpu)); + } + cnt += snprintf(&rcupreempt_trace_buf[cnt], + RCUPREEMPT_TRACE_BUF_SIZE - cnt, + "ggp = %ld, state = %s\n", + rcu_batches_completed(), + rcupreempt_try_flip_state_name()); + cnt += snprintf(&rcupreempt_trace_buf[cnt], + RCUPREEMPT_TRACE_BUF_SIZE - cnt, + "\n"); + bcount = simple_read_from_buffer(buffer, count, ppos, + rcupreempt_trace_buf, strlen(rcupreempt_trace_buf)); + mutex_unlock(&rcupreempt_trace_mutex); + return bcount; +} + +static struct file_operations rcustats_fops = { + .owner = THIS_MODULE, + .read = rcustats_read, +}; + +static struct file_operations rcugp_fops = { + .owner = THIS_MODULE, + .read = rcugp_read, +}; + +static struct file_operations rcuctrs_fops = { + .owner = THIS_MODULE, + .read = rcuctrs_read, +}; + +static struct dentry *rcudir, *statdir, *ctrsdir, *gpdir; +static int rcupreempt_debugfs_init(void) +{ + rcudir = debugfs_create_dir("rcu", NULL); + if (!rcudir) + goto out; + statdir = debugfs_create_file("rcustats", 0444, rcudir, + NULL, &rcustats_fops); + if (!statdir) + goto free_out; + + gpdir = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops); + if (!gpdir) + goto free_out; + + ctrsdir = debugfs_create_file("rcuctrs", 0444, rcudir, + NULL, &rcuctrs_fops); + if (!ctrsdir) + goto free_out; + return 0; +free_out: + if (statdir) + debugfs_remove(statdir); + if (gpdir) + debugfs_remove(gpdir); + debugfs_remove(rcudir); +out: + return 1; +} + +static int __init rcupreempt_trace_init(void) +{ + mutex_init(&rcupreempt_trace_mutex); + rcupreempt_trace_buf = kmalloc(RCUPREEMPT_TRACE_BUF_SIZE, GFP_KERNEL); + if (!rcupreempt_trace_buf) + return 1; + return rcupreempt_debugfs_init(); +} + +static void __exit rcupreempt_trace_cleanup(void) +{ + debugfs_remove(statdir); + debugfs_remove(gpdir); + debugfs_remove(ctrsdir); + debugfs_remove(rcudir); + kfree(rcupreempt_trace_buf); +} + + +module_init(rcupreempt_trace_init); +module_exit(rcupreempt_trace_cleanup); -- cgit v1.2.3 From eaf649e9fe6685f4c5a392cd0e16df5fd6660b7c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 25 Jan 2008 21:08:25 +0100 Subject: Preempt-RCU: CPU Hotplug handling This patch allows preemptible RCU to tolerate CPU-hotplug operations. It accomplishes this by maintaining a local copy of a map of online CPUs, which it accesses under its own lock. Signed-off-by: Gautham R Shenoy Signed-off-by: Paul E. McKenney Reviewed-by: Steven Rostedt Signed-off-by: Ingo Molnar --- kernel/rcupreempt.c | 147 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 142 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c index a5aabb1677f..987cfb7ade8 100644 --- a/kernel/rcupreempt.c +++ b/kernel/rcupreempt.c @@ -147,6 +147,8 @@ static char *rcu_try_flip_state_names[] = { "idle", "waitack", "waitzero", "waitmb" }; #endif /* #ifdef CONFIG_RCU_TRACE */ +static cpumask_t rcu_cpu_online_map __read_mostly = CPU_MASK_NONE; + /* * Enum and per-CPU flag to determine when each CPU has seen * the most recent counter flip. @@ -445,7 +447,7 @@ rcu_try_flip_idle(void) /* Now ask each CPU for acknowledgement of the flip. */ - for_each_possible_cpu(cpu) + for_each_cpu_mask(cpu, rcu_cpu_online_map) per_cpu(rcu_flip_flag, cpu) = rcu_flipped; return 1; @@ -461,7 +463,7 @@ rcu_try_flip_waitack(void) int cpu; RCU_TRACE_ME(rcupreempt_trace_try_flip_a1); - for_each_possible_cpu(cpu) + for_each_cpu_mask(cpu, rcu_cpu_online_map) if (per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) { RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1); return 0; @@ -492,7 +494,7 @@ rcu_try_flip_waitzero(void) /* Check to see if the sum of the "last" counters is zero. */ RCU_TRACE_ME(rcupreempt_trace_try_flip_z1); - for_each_possible_cpu(cpu) + for_each_cpu_mask(cpu, rcu_cpu_online_map) sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx]; if (sum != 0) { RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1); @@ -507,7 +509,7 @@ rcu_try_flip_waitzero(void) smp_mb(); /* ^^^^^^^^^^^^ */ /* Call for a memory barrier from each CPU. */ - for_each_possible_cpu(cpu) + for_each_cpu_mask(cpu, rcu_cpu_online_map) per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed; RCU_TRACE_ME(rcupreempt_trace_try_flip_z2); @@ -525,7 +527,7 @@ rcu_try_flip_waitmb(void) int cpu; RCU_TRACE_ME(rcupreempt_trace_try_flip_m1); - for_each_possible_cpu(cpu) + for_each_cpu_mask(cpu, rcu_cpu_online_map) if (per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) { RCU_TRACE_ME(rcupreempt_trace_try_flip_me1); return 0; @@ -637,6 +639,98 @@ void rcu_advance_callbacks(int cpu, int user) spin_unlock_irqrestore(&rdp->lock, flags); } +#ifdef CONFIG_HOTPLUG_CPU +#define rcu_offline_cpu_enqueue(srclist, srctail, dstlist, dsttail) do { \ + *dsttail = srclist; \ + if (srclist != NULL) { \ + dsttail = srctail; \ + srclist = NULL; \ + srctail = &srclist;\ + } \ + } while (0) + +void rcu_offline_cpu(int cpu) +{ + int i; + struct rcu_head *list = NULL; + unsigned long flags; + struct rcu_data *rdp = RCU_DATA_CPU(cpu); + struct rcu_head **tail = &list; + + /* + * Remove all callbacks from the newly dead CPU, retaining order. + * Otherwise rcu_barrier() will fail + */ + + spin_lock_irqsave(&rdp->lock, flags); + rcu_offline_cpu_enqueue(rdp->donelist, rdp->donetail, list, tail); + for (i = GP_STAGES - 1; i >= 0; i--) + rcu_offline_cpu_enqueue(rdp->waitlist[i], rdp->waittail[i], + list, tail); + rcu_offline_cpu_enqueue(rdp->nextlist, rdp->nexttail, list, tail); + spin_unlock_irqrestore(&rdp->lock, flags); + rdp->waitlistcount = 0; + + /* Disengage the newly dead CPU from the grace-period computation. */ + + spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags); + rcu_check_mb(cpu); + if (per_cpu(rcu_flip_flag, cpu) == rcu_flipped) { + smp_mb(); /* Subsequent counter accesses must see new value */ + per_cpu(rcu_flip_flag, cpu) = rcu_flip_seen; + smp_mb(); /* Subsequent RCU read-side critical sections */ + /* seen -after- acknowledgement. */ + } + + RCU_DATA_ME()->rcu_flipctr[0] += RCU_DATA_CPU(cpu)->rcu_flipctr[0]; + RCU_DATA_ME()->rcu_flipctr[1] += RCU_DATA_CPU(cpu)->rcu_flipctr[1]; + + RCU_DATA_CPU(cpu)->rcu_flipctr[0] = 0; + RCU_DATA_CPU(cpu)->rcu_flipctr[1] = 0; + + cpu_clear(cpu, rcu_cpu_online_map); + + spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); + + /* + * Place the removed callbacks on the current CPU's queue. + * Make them all start a new grace period: simple approach, + * in theory could starve a given set of callbacks, but + * you would need to be doing some serious CPU hotplugging + * to make this happen. If this becomes a problem, adding + * a synchronize_rcu() to the hotplug path would be a simple + * fix. + */ + + rdp = RCU_DATA_ME(); + spin_lock_irqsave(&rdp->lock, flags); + *rdp->nexttail = list; + if (list) + rdp->nexttail = tail; + spin_unlock_irqrestore(&rdp->lock, flags); +} + +void __devinit rcu_online_cpu(int cpu) +{ + unsigned long flags; + + spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags); + cpu_set(cpu, rcu_cpu_online_map); + spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); +} + +#else /* #ifdef CONFIG_HOTPLUG_CPU */ + +void rcu_offline_cpu(int cpu) +{ +} + +void __devinit rcu_online_cpu(int cpu) +{ +} + +#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ + static void rcu_process_callbacks(struct softirq_action *unused) { unsigned long flags; @@ -746,6 +840,32 @@ int rcu_pending(int cpu) return 0; } +static int __cpuinit rcu_cpu_notify(struct notifier_block *self, + unsigned long action, void *hcpu) +{ + long cpu = (long)hcpu; + + switch (action) { + case CPU_UP_PREPARE: + case CPU_UP_PREPARE_FROZEN: + rcu_online_cpu(cpu); + break; + case CPU_UP_CANCELED: + case CPU_UP_CANCELED_FROZEN: + case CPU_DEAD: + case CPU_DEAD_FROZEN: + rcu_offline_cpu(cpu); + break; + default: + break; + } + return NOTIFY_OK; +} + +static struct notifier_block __cpuinitdata rcu_nb = { + .notifier_call = rcu_cpu_notify, +}; + void __init __rcu_init(void) { int cpu; @@ -769,6 +889,23 @@ void __init __rcu_init(void) rdp->rcu_flipctr[0] = 0; rdp->rcu_flipctr[1] = 0; } + register_cpu_notifier(&rcu_nb); + + /* + * We don't need protection against CPU-Hotplug here + * since + * a) If a CPU comes online while we are iterating over the + * cpu_online_map below, we would only end up making a + * duplicate call to rcu_online_cpu() which sets the corresponding + * CPU's mask in the rcu_cpu_online_map. + * + * b) A CPU cannot go offline at this point in time since the user + * does not have access to the sysfs interface, nor do we + * suspend the system. + */ + for_each_online_cpu(cpu) + rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long) cpu); + open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL); } -- cgit v1.2.3 From dc938520d2bf343b239795cfa24e4f44649358dc Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Fri, 25 Jan 2008 21:08:26 +0100 Subject: sched: dynamically update the root-domain span/online maps The baseline code statically builds the span maps when the domain is formed. Previous attempts at dynamically updating the maps caused a suspend-to-ram regression, which should now be fixed. Signed-off-by: Gregory Haskins CC: Gautham R Shenoy Signed-off-by: Ingo Molnar --- kernel/sched.c | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 5834c7fb79a..02d468844a9 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -359,8 +359,6 @@ struct rt_rq { * exclusive cpuset is created, we also create and attach a new root-domain * object. * - * By default the system creates a single root-domain with all cpus as - * members (mimicking the global state we have today). */ struct root_domain { atomic_t refcount; @@ -375,6 +373,10 @@ struct root_domain { atomic_t rto_count; }; +/* + * By default the system creates a single root-domain with all cpus as + * members (mimicking the global state we have today). + */ static struct root_domain def_root_domain; #endif @@ -5859,6 +5861,9 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) class->leave_domain(rq); } + cpu_clear(rq->cpu, old_rd->span); + cpu_clear(rq->cpu, old_rd->online); + if (atomic_dec_and_test(&old_rd->refcount)) kfree(old_rd); } @@ -5866,6 +5871,10 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) atomic_inc(&rd->refcount); rq->rd = rd; + cpu_set(rq->cpu, rd->span); + if (cpu_isset(rq->cpu, cpu_online_map)) + cpu_set(rq->cpu, rd->online); + for (class = sched_class_highest; class; class = class->next) { if (class->join_domain) class->join_domain(rq); @@ -5874,23 +5883,21 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) spin_unlock_irqrestore(&rq->lock, flags); } -static void init_rootdomain(struct root_domain *rd, const cpumask_t *map) +static void init_rootdomain(struct root_domain *rd) { memset(rd, 0, sizeof(*rd)); - rd->span = *map; - cpus_and(rd->online, rd->span, cpu_online_map); + cpus_clear(rd->span); + cpus_clear(rd->online); } static void init_defrootdomain(void) { - cpumask_t cpus = CPU_MASK_ALL; - - init_rootdomain(&def_root_domain, &cpus); + init_rootdomain(&def_root_domain); atomic_set(&def_root_domain.refcount, 1); } -static struct root_domain *alloc_rootdomain(const cpumask_t *map) +static struct root_domain *alloc_rootdomain(void) { struct root_domain *rd; @@ -5898,7 +5905,7 @@ static struct root_domain *alloc_rootdomain(const cpumask_t *map) if (!rd) return NULL; - init_rootdomain(rd, map); + init_rootdomain(rd); return rd; } @@ -6319,7 +6326,7 @@ static int build_sched_domains(const cpumask_t *cpu_map) sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes; #endif - rd = alloc_rootdomain(cpu_map); + rd = alloc_rootdomain(); if (!rd) { printk(KERN_WARNING "Cannot alloc root domain\n"); return -ENOMEM; @@ -6894,7 +6901,6 @@ void __init sched_init(void) #ifdef CONFIG_SMP rq->sd = NULL; rq->rd = NULL; - rq_attach_root(rq, &def_root_domain); rq->active_balance = 0; rq->next_balance = jiffies; rq->push_cpu = 0; @@ -6903,6 +6909,7 @@ void __init sched_init(void) INIT_LIST_HEAD(&rq->migration_queue); rq->rt.highest_prio = MAX_RT_PRIO; rq->rt.overloaded = 0; + rq_attach_root(rq, &def_root_domain); #endif atomic_set(&rq->nr_iowait, 0); -- cgit v1.2.3 From 8eb703e4f33488bf75829564d51d427e17f7cd4c Mon Sep 17 00:00:00 2001 From: Pavel Emelyanov Date: Fri, 25 Jan 2008 21:08:26 +0100 Subject: uids: merge multiple error paths in alloc_uid() into one There are already 4 error paths in alloc_uid() that do incremental rollbacks. I think it's time to merge them. This costs us 8 lines of code :) Maybe it would be better to merge this patch with the previous one, but I remember that some time ago I sent a similar patch (fixing the error path and cleaning it), but I was told to make two patches in such cases. Signed-off-by: Pavel Emelyanov Acked-by: Dhaval Giani Signed-off-by: Andrew Morton Signed-off-by: Ingo Molnar --- kernel/user.c | 47 ++++++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/kernel/user.c b/kernel/user.c index ab4fd706993..bc1c48d35cb 100644 --- a/kernel/user.c +++ b/kernel/user.c @@ -319,7 +319,7 @@ void free_uid(struct user_struct *up) struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid) { struct hlist_head *hashent = uidhashentry(ns, uid); - struct user_struct *up; + struct user_struct *up, *new; /* Make uid_hash_find() + uids_user_create() + uid_hash_insert() * atomic. @@ -331,13 +331,9 @@ struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid) spin_unlock_irq(&uidhash_lock); if (!up) { - struct user_struct *new; - new = kmem_cache_alloc(uid_cachep, GFP_KERNEL); - if (!new) { - uids_mutex_unlock(); - return NULL; - } + if (!new) + goto out_unlock; new->uid = uid; atomic_set(&new->__count, 1); @@ -353,28 +349,14 @@ struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid) #endif new->locked_shm = 0; - if (alloc_uid_keyring(new, current) < 0) { - kmem_cache_free(uid_cachep, new); - uids_mutex_unlock(); - return NULL; - } + if (alloc_uid_keyring(new, current) < 0) + goto out_free_user; - if (sched_create_user(new) < 0) { - key_put(new->uid_keyring); - key_put(new->session_keyring); - kmem_cache_free(uid_cachep, new); - uids_mutex_unlock(); - return NULL; - } + if (sched_create_user(new) < 0) + goto out_put_keys; - if (uids_user_create(new)) { - sched_destroy_user(new); - key_put(new->uid_keyring); - key_put(new->session_keyring); - kmem_cache_free(uid_cachep, new); - uids_mutex_unlock(); - return NULL; - } + if (uids_user_create(new)) + goto out_destoy_sched; /* * Before adding this, check whether we raced @@ -402,6 +384,17 @@ struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid) uids_mutex_unlock(); return up; + +out_destoy_sched: + sched_destroy_user(new); +out_put_keys: + key_put(new->uid_keyring); + key_put(new->session_keyring); +out_free_user: + kmem_cache_free(uid_cachep, new); +out_unlock: + uids_mutex_unlock(); + return NULL; } void switch_uid(struct user_struct *new_user) -- cgit v1.2.3 From fa717060f1ab7eb6570f2fb49136f838fc9195a9 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:27 +0100 Subject: sched: sched_rt_entity Move the task_struct members specific to rt scheduling together. A future optimization could be to put sched_entity and sched_rt_entity into a union. Signed-off-by: Peter Zijlstra CC: Srivatsa Vaddagiri Signed-off-by: Ingo Molnar --- kernel/sched.c | 2 +- kernel/sched_rt.c | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 02d468844a9..c2cedd09d89 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -1685,7 +1685,7 @@ static void __sched_fork(struct task_struct *p) p->se.wait_max = 0; #endif - INIT_LIST_HEAD(&p->run_list); + INIT_LIST_HEAD(&p->rt.run_list); p->se.on_rq = 0; #ifdef CONFIG_PREEMPT_NOTIFIERS diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 9affb3c9d3d..29963af782a 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -111,7 +111,7 @@ static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup) { struct rt_prio_array *array = &rq->rt.active; - list_add_tail(&p->run_list, array->queue + p->prio); + list_add_tail(&p->rt.run_list, array->queue + p->prio); __set_bit(p->prio, array->bitmap); inc_cpu_load(rq, p->se.load.weight); @@ -127,7 +127,7 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep) update_curr_rt(rq); - list_del(&p->run_list); + list_del(&p->rt.run_list); if (list_empty(array->queue + p->prio)) __clear_bit(p->prio, array->bitmap); dec_cpu_load(rq, p->se.load.weight); @@ -143,7 +143,7 @@ static void requeue_task_rt(struct rq *rq, struct task_struct *p) { struct rt_prio_array *array = &rq->rt.active; - list_move_tail(&p->run_list, array->queue + p->prio); + list_move_tail(&p->rt.run_list, array->queue + p->prio); } static void @@ -212,7 +212,7 @@ static struct task_struct *pick_next_task_rt(struct rq *rq) return NULL; queue = array->queue + idx; - next = list_entry(queue->next, struct task_struct, run_list); + next = list_entry(queue->next, struct task_struct, rt.run_list); next->se.exec_start = rq->clock; @@ -261,14 +261,14 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu) queue = array->queue + idx; BUG_ON(list_empty(queue)); - next = list_entry(queue->next, struct task_struct, run_list); + next = list_entry(queue->next, struct task_struct, rt.run_list); if (unlikely(pick_rt_task(rq, next, cpu))) goto out; if (queue->next->next != queue) { /* same prio task */ next = list_entry(queue->next->next, struct task_struct, - run_list); + rt.run_list); if (pick_rt_task(rq, next, cpu)) goto out; } @@ -282,7 +282,7 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu) queue = array->queue + idx; BUG_ON(list_empty(queue)); - list_for_each_entry(next, queue, run_list) { + list_for_each_entry(next, queue, rt.run_list) { if (pick_rt_task(rq, next, cpu)) goto out; } @@ -846,16 +846,16 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p) if (p->policy != SCHED_RR) return; - if (--p->time_slice) + if (--p->rt.time_slice) return; - p->time_slice = DEF_TIMESLICE; + p->rt.time_slice = DEF_TIMESLICE; /* * Requeue to the end of queue if we are not the only element * on the queue: */ - if (p->run_list.prev != p->run_list.next) { + if (p->rt.run_list.prev != p->rt.run_list.next) { requeue_task_rt(rq, p); set_tsk_need_resched(p); } -- cgit v1.2.3 From 78f2c7db6068fd6ef75b8c120f04a388848eacb5 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:27 +0100 Subject: sched: SCHED_FIFO/SCHED_RR watchdog timer Introduce a new rlimit that allows the user to set a runtime timeout on real-time tasks their slice. Once this limit is exceeded the task will receive SIGXCPU. So it measures runtime since the last sleep. Input and ideas by Thomas Gleixner and Lennart Poettering. Signed-off-by: Peter Zijlstra CC: Lennart Poettering CC: Michael Kerrisk CC: Ulrich Drepper Signed-off-by: Ingo Molnar --- kernel/posix-cpu-timers.c | 29 +++++++++++++++++++++++++++++ kernel/sched_rt.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) (limited to 'kernel') diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index 68c96376e84..2c076b36c4f 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c @@ -967,6 +967,7 @@ static void check_thread_timers(struct task_struct *tsk, { int maxfire; struct list_head *timers = tsk->cpu_timers; + struct signal_struct *const sig = tsk->signal; maxfire = 20; tsk->it_prof_expires = cputime_zero; @@ -1011,6 +1012,34 @@ static void check_thread_timers(struct task_struct *tsk, t->firing = 1; list_move_tail(&t->entry, firing); } + + /* + * Check for the special case thread timers. + */ + if (sig->rlim[RLIMIT_RTTIME].rlim_cur != RLIM_INFINITY) { + unsigned long hard = sig->rlim[RLIMIT_RTTIME].rlim_max; + unsigned long *soft = &sig->rlim[RLIMIT_RTTIME].rlim_cur; + + if (tsk->rt.timeout > DIV_ROUND_UP(hard, USEC_PER_SEC/HZ)) { + /* + * At the hard limit, we just die. + * No need to calculate anything else now. + */ + __group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk); + return; + } + if (tsk->rt.timeout > DIV_ROUND_UP(*soft, USEC_PER_SEC/HZ)) { + /* + * At the soft limit, send a SIGXCPU every second. + */ + if (sig->rlim[RLIMIT_RTTIME].rlim_cur + < sig->rlim[RLIMIT_RTTIME].rlim_max) { + sig->rlim[RLIMIT_RTTIME].rlim_cur += + USEC_PER_SEC; + } + __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk); + } + } } /* diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 29963af782a..f350f7b1515 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -116,6 +116,9 @@ static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup) inc_cpu_load(rq, p->se.load.weight); inc_rt_tasks(p, rq); + + if (wakeup) + p->rt.timeout = 0; } /* @@ -834,11 +837,38 @@ static void prio_changed_rt(struct rq *rq, struct task_struct *p, } } +static void watchdog(struct rq *rq, struct task_struct *p) +{ + unsigned long soft, hard; + + if (!p->signal) + return; + + soft = p->signal->rlim[RLIMIT_RTTIME].rlim_cur; + hard = p->signal->rlim[RLIMIT_RTTIME].rlim_max; + + if (soft != RLIM_INFINITY) { + unsigned long next; + + p->rt.timeout++; + next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ); + if (next > p->rt.timeout) { + u64 next_time = p->se.sum_exec_runtime; + + next_time += next * (NSEC_PER_SEC/HZ); + if (p->it_sched_expires > next_time) + p->it_sched_expires = next_time; + } else + p->it_sched_expires = p->se.sum_exec_runtime; + } +} static void task_tick_rt(struct rq *rq, struct task_struct *p) { update_curr_rt(rq); + watchdog(rq, p); + /* * RR tasks need a special form of timeslice management. * FIFO tasks have no timeslices. -- cgit v1.2.3 From 03319ec8b06849051747a17aa2a0f9aba9277980 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:28 +0100 Subject: sched: documentation, whitespace fixes whitespace fixes. Signed-off-by: Ingo Molnar --- kernel/sched.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index c2cedd09d89..b9ee0f4db66 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -182,7 +182,7 @@ struct task_group { * * Bw(A) = 1000/(1000+2000+3000) * 100 = 16.66% * Bw(B) = 2000/(1000+2000+3000) * 100 = 33.33% - * Bw(C) = 3000/(1000+2000+3000) * 100 = 50% + * Bw(C) = 3000/(1000+2000+3000) * 100 = 50% * * The weight assigned to a task group's schedulable entities on every * cpu (task_group.se[a_cpu]->load.weight) is derived from the task @@ -192,9 +192,9 @@ struct task_group { * tg_A->se[0]->load.weight = tg_A->se[1]->load.weight = 1000; * * Note: It's not necessary that each of a task's group schedulable - * entity have the same weight on all CPUs. If the group - * has 2 of its tasks on CPU0 and 1 task on CPU1, then a - * better distribution of weight could be: + * entity have the same weight on all CPUs. If the group + * has 2 of its tasks on CPU0 and 1 task on CPU1, then a + * better distribution of weight could be: * * tg_A->se[0]->load.weight = 2/3 * 2000 = 1333 * tg_A->se[1]->load.weight = 1/2 * 2000 = 667 -- cgit v1.2.3 From 02b67cc3ba36bdba351d6c3a00593f4ec550d9d3 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Jan 2008 21:08:28 +0100 Subject: sched: do not do cond_resched() when CONFIG_PREEMPT Why do we even have cond_resched when real preemption is on? It seems to be a waste of space and time. remove cond_resched with CONFIG_PREEMPT on. Signed-off-by: Ingo Molnar --- kernel/sched.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index b9ee0f4db66..6ee37602a6d 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -4678,7 +4678,8 @@ static void __cond_resched(void) } while (need_resched()); } -int __sched cond_resched(void) +#if !defined(CONFIG_PREEMPT) || defined(CONFIG_PREEMPT_VOLUNTARY) +int __sched _cond_resched(void) { if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) && system_state == SYSTEM_RUNNING) { @@ -4687,7 +4688,8 @@ int __sched cond_resched(void) } return 0; } -EXPORT_SYMBOL(cond_resched); +EXPORT_SYMBOL(_cond_resched); +#endif /* * cond_resched_lock() - if a reschedule is pending, drop the given lock, -- cgit v1.2.3 From 8f4d37ec073c17e2d4aa8851df5837d798606d6f Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:29 +0100 Subject: sched: high-res preemption tick Use HR-timers (when available) to deliver an accurate preemption tick. The regular scheduler tick that runs at 1/HZ can be too coarse when nice level are used. The fairness system will still keep the cpu utilisation 'fair' by then delaying the task that got an excessive amount of CPU time but try to minimize this by delivering preemption points spot-on. The average frequency of this extra interrupt is sched_latency / nr_latency. Which need not be higher than 1/HZ, its just that the distribution within the sched_latency period is important. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/Kconfig.hz | 2 + kernel/sched.c | 210 +++++++++++++++++++++++++++++++++++++++++++++--- kernel/sched_fair.c | 69 +++++++++++++++- kernel/sched_idletask.c | 2 +- kernel/sched_rt.c | 2 +- 5 files changed, 268 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz index 4af15802ccd..526128a2e62 100644 --- a/kernel/Kconfig.hz +++ b/kernel/Kconfig.hz @@ -54,3 +54,5 @@ config HZ default 300 if HZ_300 default 1000 if HZ_1000 +config SCHED_HRTICK + def_bool HIGH_RES_TIMERS && X86 diff --git a/kernel/sched.c b/kernel/sched.c index 6ee37602a6d..17f93d3eda9 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -65,6 +65,7 @@ #include #include #include +#include #include #include @@ -451,6 +452,12 @@ struct rq { struct list_head migration_queue; #endif +#ifdef CONFIG_SCHED_HRTICK + unsigned long hrtick_flags; + ktime_t hrtick_expire; + struct hrtimer hrtick_timer; +#endif + #ifdef CONFIG_SCHEDSTATS /* latency stats */ struct sched_info rq_sched_info; @@ -572,6 +579,8 @@ enum { SCHED_FEAT_START_DEBIT = 4, SCHED_FEAT_TREE_AVG = 8, SCHED_FEAT_APPROX_AVG = 16, + SCHED_FEAT_HRTICK = 32, + SCHED_FEAT_DOUBLE_TICK = 64, }; const_debug unsigned int sysctl_sched_features = @@ -579,7 +588,9 @@ const_debug unsigned int sysctl_sched_features = SCHED_FEAT_WAKEUP_PREEMPT * 1 | SCHED_FEAT_START_DEBIT * 1 | SCHED_FEAT_TREE_AVG * 0 | - SCHED_FEAT_APPROX_AVG * 0; + SCHED_FEAT_APPROX_AVG * 0 | + SCHED_FEAT_HRTICK * 1 | + SCHED_FEAT_DOUBLE_TICK * 0; #define sched_feat(x) (sysctl_sched_features & SCHED_FEAT_##x) @@ -796,6 +807,173 @@ void sched_clock_idle_wakeup_event(u64 delta_ns) } EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); +static void __resched_task(struct task_struct *p, int tif_bit); + +static inline void resched_task(struct task_struct *p) +{ + __resched_task(p, TIF_NEED_RESCHED); +} + +#ifdef CONFIG_SCHED_HRTICK +/* + * Use HR-timers to deliver accurate preemption points. + * + * Its all a bit involved since we cannot program an hrt while holding the + * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a + * reschedule event. + * + * When we get rescheduled we reprogram the hrtick_timer outside of the + * rq->lock. + */ +static inline void resched_hrt(struct task_struct *p) +{ + __resched_task(p, TIF_HRTICK_RESCHED); +} + +static inline void resched_rq(struct rq *rq) +{ + unsigned long flags; + + spin_lock_irqsave(&rq->lock, flags); + resched_task(rq->curr); + spin_unlock_irqrestore(&rq->lock, flags); +} + +enum { + HRTICK_SET, /* re-programm hrtick_timer */ + HRTICK_RESET, /* not a new slice */ +}; + +/* + * Use hrtick when: + * - enabled by features + * - hrtimer is actually high res + */ +static inline int hrtick_enabled(struct rq *rq) +{ + if (!sched_feat(HRTICK)) + return 0; + return hrtimer_is_hres_active(&rq->hrtick_timer); +} + +/* + * Called to set the hrtick timer state. + * + * called with rq->lock held and irqs disabled + */ +static void hrtick_start(struct rq *rq, u64 delay, int reset) +{ + assert_spin_locked(&rq->lock); + + /* + * preempt at: now + delay + */ + rq->hrtick_expire = + ktime_add_ns(rq->hrtick_timer.base->get_time(), delay); + /* + * indicate we need to program the timer + */ + __set_bit(HRTICK_SET, &rq->hrtick_flags); + if (reset) + __set_bit(HRTICK_RESET, &rq->hrtick_flags); + + /* + * New slices are called from the schedule path and don't need a + * forced reschedule. + */ + if (reset) + resched_hrt(rq->curr); +} + +static void hrtick_clear(struct rq *rq) +{ + if (hrtimer_active(&rq->hrtick_timer)) + hrtimer_cancel(&rq->hrtick_timer); +} + +/* + * Update the timer from the possible pending state. + */ +static void hrtick_set(struct rq *rq) +{ + ktime_t time; + int set, reset; + unsigned long flags; + + WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); + + spin_lock_irqsave(&rq->lock, flags); + set = __test_and_clear_bit(HRTICK_SET, &rq->hrtick_flags); + reset = __test_and_clear_bit(HRTICK_RESET, &rq->hrtick_flags); + time = rq->hrtick_expire; + clear_thread_flag(TIF_HRTICK_RESCHED); + spin_unlock_irqrestore(&rq->lock, flags); + + if (set) { + hrtimer_start(&rq->hrtick_timer, time, HRTIMER_MODE_ABS); + if (reset && !hrtimer_active(&rq->hrtick_timer)) + resched_rq(rq); + } else + hrtick_clear(rq); +} + +/* + * High-resolution timer tick. + * Runs from hardirq context with interrupts disabled. + */ +static enum hrtimer_restart hrtick(struct hrtimer *timer) +{ + struct rq *rq = container_of(timer, struct rq, hrtick_timer); + + WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); + + spin_lock(&rq->lock); + __update_rq_clock(rq); + rq->curr->sched_class->task_tick(rq, rq->curr, 1); + spin_unlock(&rq->lock); + + return HRTIMER_NORESTART; +} + +static inline void init_rq_hrtick(struct rq *rq) +{ + rq->hrtick_flags = 0; + hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + rq->hrtick_timer.function = hrtick; + rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ; +} + +void hrtick_resched(void) +{ + struct rq *rq; + unsigned long flags; + + if (!test_thread_flag(TIF_HRTICK_RESCHED)) + return; + + local_irq_save(flags); + rq = cpu_rq(smp_processor_id()); + hrtick_set(rq); + local_irq_restore(flags); +} +#else +static inline void hrtick_clear(struct rq *rq) +{ +} + +static inline void hrtick_set(struct rq *rq) +{ +} + +static inline void init_rq_hrtick(struct rq *rq) +{ +} + +void hrtick_resched(void) +{ +} +#endif + /* * resched_task - mark a task 'to be rescheduled now'. * @@ -809,16 +987,16 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); #define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG) #endif -static void resched_task(struct task_struct *p) +static void __resched_task(struct task_struct *p, int tif_bit) { int cpu; assert_spin_locked(&task_rq(p)->lock); - if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED))) + if (unlikely(test_tsk_thread_flag(p, tif_bit))) return; - set_tsk_thread_flag(p, TIF_NEED_RESCHED); + set_tsk_thread_flag(p, tif_bit); cpu = task_cpu(p); if (cpu == smp_processor_id()) @@ -841,10 +1019,10 @@ static void resched_cpu(int cpu) spin_unlock_irqrestore(&rq->lock, flags); } #else -static inline void resched_task(struct task_struct *p) +static void __resched_task(struct task_struct *p, int tif_bit) { assert_spin_locked(&task_rq(p)->lock); - set_tsk_need_resched(p); + set_tsk_thread_flag(p, tif_bit); } #endif @@ -3497,7 +3675,7 @@ void scheduler_tick(void) rq->tick_timestamp = rq->clock; update_cpu_load(rq); if (curr != rq->idle) /* FIXME: needed? */ - curr->sched_class->task_tick(rq, curr); + curr->sched_class->task_tick(rq, curr, 0); spin_unlock(&rq->lock); #ifdef CONFIG_SMP @@ -3643,6 +3821,8 @@ need_resched_nonpreemptible: schedule_debug(prev); + hrtick_clear(rq); + /* * Do the rq-clock update outside the rq lock: */ @@ -3680,14 +3860,20 @@ need_resched_nonpreemptible: ++*switch_count; context_switch(rq, prev, next); /* unlocks the rq */ + /* + * the context switch might have flipped the stack from under + * us, hence refresh the local variables. + */ + cpu = smp_processor_id(); + rq = cpu_rq(cpu); } else spin_unlock_irq(&rq->lock); - if (unlikely(reacquire_kernel_lock(current) < 0)) { - cpu = smp_processor_id(); - rq = cpu_rq(cpu); + hrtick_set(rq); + + if (unlikely(reacquire_kernel_lock(current) < 0)) goto need_resched_nonpreemptible; - } + preempt_enable_no_resched(); if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) goto need_resched; @@ -6913,6 +7099,8 @@ void __init sched_init(void) rq->rt.overloaded = 0; rq_attach_root(rq, &def_root_domain); #endif + init_rq_hrtick(rq); + atomic_set(&rq->nr_iowait, 0); array = &rq->rt.active; diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index dfa18d55561..3dab1ff83c4 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -642,13 +642,29 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) cfs_rq->curr = NULL; } -static void entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) +static void +entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) { /* * Update run-time statistics of the 'current'. */ update_curr(cfs_rq); +#ifdef CONFIG_SCHED_HRTICK + /* + * queued ticks are scheduled to match the slice, so don't bother + * validating it and just reschedule. + */ + if (queued) + return resched_task(rq_of(cfs_rq)->curr); + /* + * don't let the period tick interfere with the hrtick preemption + */ + if (!sched_feat(DOUBLE_TICK) && + hrtimer_active(&rq_of(cfs_rq)->hrtick_timer)) + return; +#endif + if (cfs_rq->nr_running > 1 || !sched_feat(WAKEUP_PREEMPT)) check_preempt_tick(cfs_rq, curr); } @@ -754,6 +770,43 @@ static inline struct sched_entity *parent_entity(struct sched_entity *se) #endif /* CONFIG_FAIR_GROUP_SCHED */ +#ifdef CONFIG_SCHED_HRTICK +static void hrtick_start_fair(struct rq *rq, struct task_struct *p) +{ + int requeue = rq->curr == p; + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = cfs_rq_of(se); + + WARN_ON(task_rq(p) != rq); + + if (hrtick_enabled(rq) && cfs_rq->nr_running > 1) { + u64 slice = sched_slice(cfs_rq, se); + u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; + s64 delta = slice - ran; + + if (delta < 0) { + if (rq->curr == p) + resched_task(p); + return; + } + + /* + * Don't schedule slices shorter than 10000ns, that just + * doesn't make sense. Rely on vruntime for fairness. + */ + if (!requeue) + delta = max(10000LL, delta); + + hrtick_start(rq, delta, requeue); + } +} +#else +static inline void +hrtick_start_fair(struct rq *rq, struct task_struct *p) +{ +} +#endif + /* * The enqueue_task method is called before nr_running is * increased. Here we update the fair scheduling stats and @@ -782,6 +835,8 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup) */ if (incload) inc_cpu_load(rq, topse->load.weight); + + hrtick_start_fair(rq, rq->curr); } /* @@ -814,6 +869,8 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep) */ if (decload) dec_cpu_load(rq, topse->load.weight); + + hrtick_start_fair(rq, rq->curr); } /* @@ -1049,6 +1106,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) static struct task_struct *pick_next_task_fair(struct rq *rq) { + struct task_struct *p; struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; @@ -1060,7 +1118,10 @@ static struct task_struct *pick_next_task_fair(struct rq *rq) cfs_rq = group_cfs_rq(se); } while (cfs_rq); - return task_of(se); + p = task_of(se); + hrtick_start_fair(rq, p); + + return p; } /* @@ -1235,14 +1296,14 @@ move_one_task_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, /* * scheduler tick hitting a task of our scheduling class: */ -static void task_tick_fair(struct rq *rq, struct task_struct *curr) +static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) { struct cfs_rq *cfs_rq; struct sched_entity *se = &curr->se; for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); - entity_tick(cfs_rq, se); + entity_tick(cfs_rq, se, queued); } } diff --git a/kernel/sched_idletask.c b/kernel/sched_idletask.c index ef7a2661fa1..2bcafa37563 100644 --- a/kernel/sched_idletask.c +++ b/kernel/sched_idletask.c @@ -61,7 +61,7 @@ move_one_task_idle(struct rq *this_rq, int this_cpu, struct rq *busiest, } #endif -static void task_tick_idle(struct rq *rq, struct task_struct *curr) +static void task_tick_idle(struct rq *rq, struct task_struct *curr, int queued) { } diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index f350f7b1515..83fbbcb8019 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -863,7 +863,7 @@ static void watchdog(struct rq *rq, struct task_struct *p) } } -static void task_tick_rt(struct rq *rq, struct task_struct *p) +static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued) { update_curr_rt(rq); -- cgit v1.2.3 From fa85ae2418e6843953107cd6a06f645752829bc0 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:29 +0100 Subject: sched: rt time limit Very simple time limit on the realtime scheduling classes. Allow the rq's realtime class to consume sched_rt_ratio of every sched_rt_period slice. If the class exceeds this quota the fair class will preempt the realtime class. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/sched.c | 70 +++++++++++++++++++++++++++++++++++++++---------------- kernel/sched_rt.c | 53 +++++++++++++++++++++++++++++++++++++++++ kernel/sysctl.c | 18 +++++++++++++- 3 files changed, 120 insertions(+), 21 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 17f93d3eda9..e9a7beee9b7 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -342,13 +342,14 @@ struct cfs_rq { /* Real-Time classes' related field in a runqueue: */ struct rt_rq { struct rt_prio_array active; - int rt_load_balance_idx; - struct list_head *rt_load_balance_head, *rt_load_balance_curr; unsigned long rt_nr_running; +#ifdef CONFIG_SMP unsigned long rt_nr_migratory; - /* highest queued rt task prio */ - int highest_prio; + int highest_prio; /* highest queued rt task prio */ int overloaded; +#endif + u64 rt_time; + u64 rt_throttled; }; #ifdef CONFIG_SMP @@ -415,6 +416,7 @@ struct rq { struct list_head leaf_cfs_rq_list; #endif struct rt_rq rt; + u64 rt_period_expire; /* * This is part of a global counter where only the total sum @@ -600,6 +602,21 @@ const_debug unsigned int sysctl_sched_features = */ const_debug unsigned int sysctl_sched_nr_migrate = 32; +/* + * period over which we measure -rt task cpu usage in ms. + * default: 1s + */ +const_debug unsigned int sysctl_sched_rt_period = 1000; + +#define SCHED_RT_FRAC_SHIFT 16 +#define SCHED_RT_FRAC (1UL << SCHED_RT_FRAC_SHIFT) + +/* + * ratio of time -rt tasks may consume. + * default: 100% + */ +const_debug unsigned int sysctl_sched_rt_ratio = SCHED_RT_FRAC; + /* * For kernel-internal use: high-speed (but slightly incorrect) per-cpu * clock constructed from sched_clock(): @@ -3674,8 +3691,8 @@ void scheduler_tick(void) rq->clock = next_tick; rq->tick_timestamp = rq->clock; update_cpu_load(rq); - if (curr != rq->idle) /* FIXME: needed? */ - curr->sched_class->task_tick(rq, curr, 0); + curr->sched_class->task_tick(rq, curr, 0); + update_sched_rt_period(rq); spin_unlock(&rq->lock); #ifdef CONFIG_SMP @@ -7041,6 +7058,29 @@ static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq) cfs_rq->min_vruntime = (u64)(-(1LL << 20)); } +static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq) +{ + struct rt_prio_array *array; + int i; + + array = &rt_rq->active; + for (i = 0; i < MAX_RT_PRIO; i++) { + INIT_LIST_HEAD(array->queue + i); + __clear_bit(i, array->bitmap); + } + /* delimiter for bitsearch: */ + __set_bit(MAX_RT_PRIO, array->bitmap); + +#ifdef CONFIG_SMP + rt_rq->rt_nr_migratory = 0; + rt_rq->highest_prio = MAX_RT_PRIO; + rt_rq->overloaded = 0; +#endif + + rt_rq->rt_time = 0; + rt_rq->rt_throttled = 0; +} + void __init sched_init(void) { int highest_cpu = 0; @@ -7051,7 +7091,6 @@ void __init sched_init(void) #endif for_each_possible_cpu(i) { - struct rt_prio_array *array; struct rq *rq; rq = cpu_rq(i); @@ -7083,6 +7122,8 @@ void __init sched_init(void) } init_task_group.shares = init_task_group_load; #endif + init_rt_rq(&rq->rt, rq); + rq->rt_period_expire = 0; for (j = 0; j < CPU_LOAD_IDX_MAX; j++) rq->cpu_load[j] = 0; @@ -7095,22 +7136,11 @@ void __init sched_init(void) rq->cpu = i; rq->migration_thread = NULL; INIT_LIST_HEAD(&rq->migration_queue); - rq->rt.highest_prio = MAX_RT_PRIO; - rq->rt.overloaded = 0; rq_attach_root(rq, &def_root_domain); #endif init_rq_hrtick(rq); - atomic_set(&rq->nr_iowait, 0); - - array = &rq->rt.active; - for (j = 0; j < MAX_RT_PRIO; j++) { - INIT_LIST_HEAD(array->queue + j); - __clear_bit(j, array->bitmap); - } highest_cpu = i; - /* delimiter for bitsearch: */ - __set_bit(MAX_RT_PRIO, array->bitmap); } set_load_weight(&init_task); @@ -7282,7 +7312,7 @@ void set_curr_task(int cpu, struct task_struct *p) #ifdef CONFIG_SMP /* * distribute shares of all task groups among their schedulable entities, - * to reflect load distrbution across cpus. + * to reflect load distribution across cpus. */ static int rebalance_shares(struct sched_domain *sd, int this_cpu) { @@ -7349,7 +7379,7 @@ static int rebalance_shares(struct sched_domain *sd, int this_cpu) * sysctl_sched_max_bal_int_shares represents the maximum interval between * consecutive calls to rebalance_shares() in the same sched domain. * - * These settings allows for the appropriate tradeoff between accuracy of + * These settings allows for the appropriate trade-off between accuracy of * fairness and the associated overhead. * */ diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 83fbbcb8019..fd10d965aa0 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -45,6 +45,50 @@ static void update_rt_migration(struct rq *rq) } #endif /* CONFIG_SMP */ +static int sched_rt_ratio_exceeded(struct rq *rq, struct rt_rq *rt_rq) +{ + u64 period, ratio; + + if (sysctl_sched_rt_ratio == SCHED_RT_FRAC) + return 0; + + if (rt_rq->rt_throttled) + return 1; + + period = (u64)sysctl_sched_rt_period * NSEC_PER_MSEC; + ratio = (period * sysctl_sched_rt_ratio) >> SCHED_RT_FRAC_SHIFT; + + if (rt_rq->rt_time > ratio) { + rt_rq->rt_throttled = rq->clock + period - rt_rq->rt_time; + return 1; + } + + return 0; +} + +static void update_sched_rt_period(struct rq *rq) +{ + while (rq->clock > rq->rt_period_expire) { + u64 period, ratio; + + period = (u64)sysctl_sched_rt_period * NSEC_PER_MSEC; + ratio = (period * sysctl_sched_rt_ratio) >> SCHED_RT_FRAC_SHIFT; + + rq->rt.rt_time -= min(rq->rt.rt_time, ratio); + rq->rt_period_expire += period; + } + + /* + * When the rt throttle is expired, let them rip. + * (XXX: use hrtick when available) + */ + if (rq->rt.rt_throttled && rq->clock > rq->rt.rt_throttled) { + rq->rt.rt_throttled = 0; + if (!sched_rt_ratio_exceeded(rq, &rq->rt)) + resched_task(rq->curr); + } +} + /* * Update the current task's runtime statistics. Skip current tasks that * are not in our scheduling class. @@ -66,6 +110,11 @@ static void update_curr_rt(struct rq *rq) curr->se.sum_exec_runtime += delta_exec; curr->se.exec_start = rq->clock; cpuacct_charge(curr, delta_exec); + + rq->rt.rt_time += delta_exec; + update_sched_rt_period(rq); + if (sched_rt_ratio_exceeded(rq, &rq->rt)) + resched_task(curr); } static inline void inc_rt_tasks(struct task_struct *p, struct rq *rq) @@ -208,8 +257,12 @@ static struct task_struct *pick_next_task_rt(struct rq *rq) struct rt_prio_array *array = &rq->rt.active; struct task_struct *next; struct list_head *queue; + struct rt_rq *rt_rq = &rq->rt; int idx; + if (sched_rt_ratio_exceeded(rq, rt_rq)) + return NULL; + idx = sched_find_first_bit(array->bitmap); if (idx >= MAX_RT_PRIO) return NULL; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 96f31c1bc4f..3afbd25f43e 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -306,7 +306,23 @@ static struct ctl_table kern_table[] = { .procname = "sched_nr_migrate", .data = &sysctl_sched_nr_migrate, .maxlen = sizeof(unsigned int), - .mode = 644, + .mode = 0644, + .proc_handler = &proc_dointvec, + }, + { + .ctl_name = CTL_UNNUMBERED, + .procname = "sched_rt_period_ms", + .data = &sysctl_sched_rt_period, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = &proc_dointvec, + }, + { + .ctl_name = CTL_UNNUMBERED, + .procname = "sched_rt_ratio", + .data = &sysctl_sched_rt_ratio, + .maxlen = sizeof(unsigned int), + .mode = 0644, .proc_handler = &proc_dointvec, }, #if defined(CONFIG_FAIR_GROUP_SCHED) && defined(CONFIG_SMP) -- cgit v1.2.3 From 6f505b16425a51270058e4a93441fe64de3dd435 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:30 +0100 Subject: sched: rt group scheduling Extend group scheduling to also cover the realtime classes. It uses the time limiting introduced by the previous patch to allow multiple realtime groups. The hard time limit is required to keep behaviour deterministic. The algorithms used make the realtime scheduler O(tg), linear scaling wrt the number of task groups. This is the worst case behaviour I can't seem to get out of, the avg. case of the algorithms can be improved, I focused on correctness and worst case. [ akpm@linux-foundation.org: move side-effects out of BUG_ON(). ] Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/fork.c | 2 +- kernel/sched.c | 283 +++++++++++++++++++++++---------- kernel/sched_rt.c | 455 ++++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 537 insertions(+), 203 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 9f8ef32cbc7..0c969f4fade 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1246,7 +1246,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, * parent's CPU). This avoids alot of nasty races. */ p->cpus_allowed = current->cpus_allowed; - p->nr_cpus_allowed = current->nr_cpus_allowed; + p->rt.nr_cpus_allowed = current->rt.nr_cpus_allowed; if (unlikely(!cpu_isset(task_cpu(p), p->cpus_allowed) || !cpu_online(task_cpu(p)))) set_task_cpu(p, smp_processor_id()); diff --git a/kernel/sched.c b/kernel/sched.c index e9a7beee9b7..5ea2c533b43 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -161,6 +161,8 @@ struct rt_prio_array { struct cfs_rq; +static LIST_HEAD(task_groups); + /* task group related information */ struct task_group { #ifdef CONFIG_FAIR_CGROUP_SCHED @@ -171,6 +173,11 @@ struct task_group { /* runqueue "owned" by this group on each cpu */ struct cfs_rq **cfs_rq; + struct sched_rt_entity **rt_se; + struct rt_rq **rt_rq; + + unsigned int rt_ratio; + /* * shares assigned to a task group governs how much of cpu bandwidth * is allocated to the group. The more shares a group has, the more is @@ -208,6 +215,7 @@ struct task_group { unsigned long shares; struct rcu_head rcu; + struct list_head list; }; /* Default task group's sched entity on each cpu */ @@ -215,9 +223,15 @@ static DEFINE_PER_CPU(struct sched_entity, init_sched_entity); /* Default task group's cfs_rq on each cpu */ static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; +static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); +static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp; + static struct sched_entity *init_sched_entity_p[NR_CPUS]; static struct cfs_rq *init_cfs_rq_p[NR_CPUS]; +static struct sched_rt_entity *init_sched_rt_entity_p[NR_CPUS]; +static struct rt_rq *init_rt_rq_p[NR_CPUS]; + /* task_group_mutex serializes add/remove of task groups and also changes to * a task group's cpu shares. */ @@ -240,6 +254,9 @@ static void set_se_shares(struct sched_entity *se, unsigned long shares); struct task_group init_task_group = { .se = init_sched_entity_p, .cfs_rq = init_cfs_rq_p, + + .rt_se = init_sched_rt_entity_p, + .rt_rq = init_rt_rq_p, }; #ifdef CONFIG_FAIR_USER_SCHED @@ -269,10 +286,13 @@ static inline struct task_group *task_group(struct task_struct *p) } /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ -static inline void set_task_cfs_rq(struct task_struct *p, unsigned int cpu) +static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { p->se.cfs_rq = task_group(p)->cfs_rq[cpu]; p->se.parent = task_group(p)->se[cpu]; + + p->rt.rt_rq = task_group(p)->rt_rq[cpu]; + p->rt.parent = task_group(p)->rt_se[cpu]; } static inline void lock_task_group_list(void) @@ -297,7 +317,7 @@ static inline void unlock_doms_cur(void) #else -static inline void set_task_cfs_rq(struct task_struct *p, unsigned int cpu) { } +static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { } static inline void lock_task_group_list(void) { } static inline void unlock_task_group_list(void) { } static inline void lock_doms_cur(void) { } @@ -343,13 +363,22 @@ struct cfs_rq { struct rt_rq { struct rt_prio_array active; unsigned long rt_nr_running; +#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED + int highest_prio; /* highest queued rt task prio */ +#endif #ifdef CONFIG_SMP unsigned long rt_nr_migratory; - int highest_prio; /* highest queued rt task prio */ int overloaded; #endif + int rt_throttled; u64 rt_time; - u64 rt_throttled; + +#ifdef CONFIG_FAIR_GROUP_SCHED + struct rq *rq; + struct list_head leaf_rt_rq_list; + struct task_group *tg; + struct sched_rt_entity *rt_se; +#endif }; #ifdef CONFIG_SMP @@ -411,12 +440,14 @@ struct rq { u64 nr_switches; struct cfs_rq cfs; + struct rt_rq rt; + u64 rt_period_expire; + #ifdef CONFIG_FAIR_GROUP_SCHED /* list of leaf cfs_rq on this cpu: */ struct list_head leaf_cfs_rq_list; + struct list_head leaf_rt_rq_list; #endif - struct rt_rq rt; - u64 rt_period_expire; /* * This is part of a global counter where only the total sum @@ -613,9 +644,9 @@ const_debug unsigned int sysctl_sched_rt_period = 1000; /* * ratio of time -rt tasks may consume. - * default: 100% + * default: 95% */ -const_debug unsigned int sysctl_sched_rt_ratio = SCHED_RT_FRAC; +const_debug unsigned int sysctl_sched_rt_ratio = 62259; /* * For kernel-internal use: high-speed (but slightly incorrect) per-cpu @@ -1337,7 +1368,7 @@ unsigned long weighted_cpuload(const int cpu) static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) { - set_task_cfs_rq(p, cpu); + set_task_rq(p, cpu); #ifdef CONFIG_SMP /* * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be @@ -5281,7 +5312,7 @@ int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask) p->sched_class->set_cpus_allowed(p, &new_mask); else { p->cpus_allowed = new_mask; - p->nr_cpus_allowed = cpus_weight(new_mask); + p->rt.nr_cpus_allowed = cpus_weight(new_mask); } /* Can the task run on the task's current CPU? If so, we're done */ @@ -7079,8 +7110,50 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq) rt_rq->rt_time = 0; rt_rq->rt_throttled = 0; + +#ifdef CONFIG_FAIR_GROUP_SCHED + rt_rq->rq = rq; +#endif } +#ifdef CONFIG_FAIR_GROUP_SCHED +static void init_tg_cfs_entry(struct rq *rq, struct task_group *tg, + struct cfs_rq *cfs_rq, struct sched_entity *se, + int cpu, int add) +{ + tg->cfs_rq[cpu] = cfs_rq; + init_cfs_rq(cfs_rq, rq); + cfs_rq->tg = tg; + if (add) + list_add(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list); + + tg->se[cpu] = se; + se->cfs_rq = &rq->cfs; + se->my_q = cfs_rq; + se->load.weight = tg->shares; + se->load.inv_weight = div64_64(1ULL<<32, se->load.weight); + se->parent = NULL; +} + +static void init_tg_rt_entry(struct rq *rq, struct task_group *tg, + struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, + int cpu, int add) +{ + tg->rt_rq[cpu] = rt_rq; + init_rt_rq(rt_rq, rq); + rt_rq->tg = tg; + rt_rq->rt_se = rt_se; + if (add) + list_add(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list); + + tg->rt_se[cpu] = rt_se; + rt_se->rt_rq = &rq->rt; + rt_se->my_q = rt_rq; + rt_se->parent = NULL; + INIT_LIST_HEAD(&rt_se->run_list); +} +#endif + void __init sched_init(void) { int highest_cpu = 0; @@ -7090,6 +7163,10 @@ void __init sched_init(void) init_defrootdomain(); #endif +#ifdef CONFIG_FAIR_GROUP_SCHED + list_add(&init_task_group.list, &task_groups); +#endif + for_each_possible_cpu(i) { struct rq *rq; @@ -7099,30 +7176,20 @@ void __init sched_init(void) rq->nr_running = 0; rq->clock = 1; init_cfs_rq(&rq->cfs, rq); + init_rt_rq(&rq->rt, rq); #ifdef CONFIG_FAIR_GROUP_SCHED - INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); - { - struct cfs_rq *cfs_rq = &per_cpu(init_cfs_rq, i); - struct sched_entity *se = - &per_cpu(init_sched_entity, i); - - init_cfs_rq_p[i] = cfs_rq; - init_cfs_rq(cfs_rq, rq); - cfs_rq->tg = &init_task_group; - list_add(&cfs_rq->leaf_cfs_rq_list, - &rq->leaf_cfs_rq_list); - - init_sched_entity_p[i] = se; - se->cfs_rq = &rq->cfs; - se->my_q = cfs_rq; - se->load.weight = init_task_group_load; - se->load.inv_weight = - div64_64(1ULL<<32, init_task_group_load); - se->parent = NULL; - } init_task_group.shares = init_task_group_load; + INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); + init_tg_cfs_entry(rq, &init_task_group, + &per_cpu(init_cfs_rq, i), + &per_cpu(init_sched_entity, i), i, 1); + + init_task_group.rt_ratio = sysctl_sched_rt_ratio; /* XXX */ + INIT_LIST_HEAD(&rq->leaf_rt_rq_list); + init_tg_rt_entry(rq, &init_task_group, + &per_cpu(init_rt_rq, i), + &per_cpu(init_sched_rt_entity, i), i, 1); #endif - init_rt_rq(&rq->rt, rq); rq->rt_period_expire = 0; for (j = 0; j < CPU_LOAD_IDX_MAX; j++) @@ -7460,12 +7527,36 @@ static int load_balance_monitor(void *unused) } #endif /* CONFIG_SMP */ +static void free_sched_group(struct task_group *tg) +{ + int i; + + for_each_possible_cpu(i) { + if (tg->cfs_rq) + kfree(tg->cfs_rq[i]); + if (tg->se) + kfree(tg->se[i]); + if (tg->rt_rq) + kfree(tg->rt_rq[i]); + if (tg->rt_se) + kfree(tg->rt_se[i]); + } + + kfree(tg->cfs_rq); + kfree(tg->se); + kfree(tg->rt_rq); + kfree(tg->rt_se); + kfree(tg); +} + /* allocate runqueue etc for a new task group */ struct task_group *sched_create_group(void) { struct task_group *tg; struct cfs_rq *cfs_rq; struct sched_entity *se; + struct rt_rq *rt_rq; + struct sched_rt_entity *rt_se; struct rq *rq; int i; @@ -7479,100 +7570,89 @@ struct task_group *sched_create_group(void) tg->se = kzalloc(sizeof(se) * NR_CPUS, GFP_KERNEL); if (!tg->se) goto err; + tg->rt_rq = kzalloc(sizeof(rt_rq) * NR_CPUS, GFP_KERNEL); + if (!tg->rt_rq) + goto err; + tg->rt_se = kzalloc(sizeof(rt_se) * NR_CPUS, GFP_KERNEL); + if (!tg->rt_se) + goto err; + + tg->shares = NICE_0_LOAD; + tg->rt_ratio = 0; /* XXX */ for_each_possible_cpu(i) { rq = cpu_rq(i); - cfs_rq = kmalloc_node(sizeof(struct cfs_rq), GFP_KERNEL, - cpu_to_node(i)); + cfs_rq = kmalloc_node(sizeof(struct cfs_rq), + GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); if (!cfs_rq) goto err; - se = kmalloc_node(sizeof(struct sched_entity), GFP_KERNEL, - cpu_to_node(i)); + se = kmalloc_node(sizeof(struct sched_entity), + GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); if (!se) goto err; - memset(cfs_rq, 0, sizeof(struct cfs_rq)); - memset(se, 0, sizeof(struct sched_entity)); + rt_rq = kmalloc_node(sizeof(struct rt_rq), + GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); + if (!rt_rq) + goto err; - tg->cfs_rq[i] = cfs_rq; - init_cfs_rq(cfs_rq, rq); - cfs_rq->tg = tg; + rt_se = kmalloc_node(sizeof(struct sched_rt_entity), + GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); + if (!rt_se) + goto err; - tg->se[i] = se; - se->cfs_rq = &rq->cfs; - se->my_q = cfs_rq; - se->load.weight = NICE_0_LOAD; - se->load.inv_weight = div64_64(1ULL<<32, NICE_0_LOAD); - se->parent = NULL; + init_tg_cfs_entry(rq, tg, cfs_rq, se, i, 0); + init_tg_rt_entry(rq, tg, rt_rq, rt_se, i, 0); } - tg->shares = NICE_0_LOAD; - lock_task_group_list(); for_each_possible_cpu(i) { rq = cpu_rq(i); cfs_rq = tg->cfs_rq[i]; list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list); + rt_rq = tg->rt_rq[i]; + list_add_rcu(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list); } + list_add_rcu(&tg->list, &task_groups); unlock_task_group_list(); return tg; err: - for_each_possible_cpu(i) { - if (tg->cfs_rq) - kfree(tg->cfs_rq[i]); - if (tg->se) - kfree(tg->se[i]); - } - kfree(tg->cfs_rq); - kfree(tg->se); - kfree(tg); - + free_sched_group(tg); return ERR_PTR(-ENOMEM); } /* rcu callback to free various structures associated with a task group */ -static void free_sched_group(struct rcu_head *rhp) +static void free_sched_group_rcu(struct rcu_head *rhp) { - struct task_group *tg = container_of(rhp, struct task_group, rcu); - struct cfs_rq *cfs_rq; - struct sched_entity *se; - int i; - /* now it should be safe to free those cfs_rqs */ - for_each_possible_cpu(i) { - cfs_rq = tg->cfs_rq[i]; - kfree(cfs_rq); - - se = tg->se[i]; - kfree(se); - } - - kfree(tg->cfs_rq); - kfree(tg->se); - kfree(tg); + free_sched_group(container_of(rhp, struct task_group, rcu)); } /* Destroy runqueue etc associated with a task group */ void sched_destroy_group(struct task_group *tg) { struct cfs_rq *cfs_rq = NULL; + struct rt_rq *rt_rq = NULL; int i; lock_task_group_list(); for_each_possible_cpu(i) { cfs_rq = tg->cfs_rq[i]; list_del_rcu(&cfs_rq->leaf_cfs_rq_list); + rt_rq = tg->rt_rq[i]; + list_del_rcu(&rt_rq->leaf_rt_rq_list); } + list_del_rcu(&tg->list); unlock_task_group_list(); BUG_ON(!cfs_rq); /* wait for possible concurrent references to cfs_rqs complete */ - call_rcu(&tg->rcu, free_sched_group); + call_rcu(&tg->rcu, free_sched_group_rcu); } /* change task's runqueue when it moves between groups. @@ -7588,11 +7668,6 @@ void sched_move_task(struct task_struct *tsk) rq = task_rq_lock(tsk, &flags); - if (tsk->sched_class != &fair_sched_class) { - set_task_cfs_rq(tsk, task_cpu(tsk)); - goto done; - } - update_rq_clock(rq); running = task_current(rq, tsk); @@ -7604,7 +7679,7 @@ void sched_move_task(struct task_struct *tsk) tsk->sched_class->put_prev_task(rq, tsk); } - set_task_cfs_rq(tsk, task_cpu(tsk)); + set_task_rq(tsk, task_cpu(tsk)); if (on_rq) { if (unlikely(running)) @@ -7612,7 +7687,6 @@ void sched_move_task(struct task_struct *tsk) enqueue_task(rq, tsk, 0); } -done: task_rq_unlock(rq, &flags); } @@ -7697,6 +7771,31 @@ unsigned long sched_group_shares(struct task_group *tg) return tg->shares; } +/* + * Ensure the total rt_ratio <= sysctl_sched_rt_ratio + */ +int sched_group_set_rt_ratio(struct task_group *tg, unsigned long rt_ratio) +{ + struct task_group *tgi; + unsigned long total = 0; + + rcu_read_lock(); + list_for_each_entry_rcu(tgi, &task_groups, list) + total += tgi->rt_ratio; + rcu_read_unlock(); + + if (total + rt_ratio - tg->rt_ratio > sysctl_sched_rt_ratio) + return -EINVAL; + + tg->rt_ratio = rt_ratio; + return 0; +} + +unsigned long sched_group_rt_ratio(struct task_group *tg) +{ + return tg->rt_ratio; +} + #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_FAIR_CGROUP_SCHED @@ -7772,12 +7871,30 @@ static u64 cpu_shares_read_uint(struct cgroup *cgrp, struct cftype *cft) return (u64) tg->shares; } +static int cpu_rt_ratio_write_uint(struct cgroup *cgrp, struct cftype *cftype, + u64 rt_ratio_val) +{ + return sched_group_set_rt_ratio(cgroup_tg(cgrp), rt_ratio_val); +} + +static u64 cpu_rt_ratio_read_uint(struct cgroup *cgrp, struct cftype *cft) +{ + struct task_group *tg = cgroup_tg(cgrp); + + return (u64) tg->rt_ratio; +} + static struct cftype cpu_files[] = { { .name = "shares", .read_uint = cpu_shares_read_uint, .write_uint = cpu_shares_write_uint, }, + { + .name = "rt_ratio", + .read_uint = cpu_rt_ratio_read_uint, + .write_uint = cpu_rt_ratio_write_uint, + }, }; static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont) diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index fd10d965aa0..1178257613a 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -45,47 +45,167 @@ static void update_rt_migration(struct rq *rq) } #endif /* CONFIG_SMP */ -static int sched_rt_ratio_exceeded(struct rq *rq, struct rt_rq *rt_rq) +static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) { + return container_of(rt_se, struct task_struct, rt); +} + +static inline int on_rt_rq(struct sched_rt_entity *rt_se) +{ + return !list_empty(&rt_se->run_list); +} + +#ifdef CONFIG_FAIR_GROUP_SCHED + +static inline unsigned int sched_rt_ratio(struct rt_rq *rt_rq) +{ + if (!rt_rq->tg) + return SCHED_RT_FRAC; + + return rt_rq->tg->rt_ratio; +} + +#define for_each_leaf_rt_rq(rt_rq, rq) \ + list_for_each_entry(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list) + +static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) +{ + return rt_rq->rq; +} + +static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) +{ + return rt_se->rt_rq; +} + +#define for_each_sched_rt_entity(rt_se) \ + for (; rt_se; rt_se = rt_se->parent) + +static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) +{ + return rt_se->my_q; +} + +static void enqueue_rt_entity(struct sched_rt_entity *rt_se); +static void dequeue_rt_entity(struct sched_rt_entity *rt_se); + +static void sched_rt_ratio_enqueue(struct rt_rq *rt_rq) +{ + struct sched_rt_entity *rt_se = rt_rq->rt_se; + + if (rt_se && !on_rt_rq(rt_se) && rt_rq->rt_nr_running) { + enqueue_rt_entity(rt_se); + resched_task(rq_of_rt_rq(rt_rq)->curr); + } +} + +static void sched_rt_ratio_dequeue(struct rt_rq *rt_rq) +{ + struct sched_rt_entity *rt_se = rt_rq->rt_se; + + if (rt_se && on_rt_rq(rt_se)) + dequeue_rt_entity(rt_se); +} + +#else + +static inline unsigned int sched_rt_ratio(struct rt_rq *rt_rq) +{ + return sysctl_sched_rt_ratio; +} + +#define for_each_leaf_rt_rq(rt_rq, rq) \ + for (rt_rq = &rq->rt; rt_rq; rt_rq = NULL) + +static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) +{ + return container_of(rt_rq, struct rq, rt); +} + +static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) +{ + struct task_struct *p = rt_task_of(rt_se); + struct rq *rq = task_rq(p); + + return &rq->rt; +} + +#define for_each_sched_rt_entity(rt_se) \ + for (; rt_se; rt_se = NULL) + +static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) +{ + return NULL; +} + +static inline void sched_rt_ratio_enqueue(struct rt_rq *rt_rq) +{ +} + +static inline void sched_rt_ratio_dequeue(struct rt_rq *rt_rq) +{ +} + +#endif + +static inline int rt_se_prio(struct sched_rt_entity *rt_se) +{ +#ifdef CONFIG_FAIR_GROUP_SCHED + struct rt_rq *rt_rq = group_rt_rq(rt_se); + + if (rt_rq) + return rt_rq->highest_prio; +#endif + + return rt_task_of(rt_se)->prio; +} + +static int sched_rt_ratio_exceeded(struct rt_rq *rt_rq) +{ + unsigned int rt_ratio = sched_rt_ratio(rt_rq); u64 period, ratio; - if (sysctl_sched_rt_ratio == SCHED_RT_FRAC) + if (rt_ratio == SCHED_RT_FRAC) return 0; if (rt_rq->rt_throttled) return 1; period = (u64)sysctl_sched_rt_period * NSEC_PER_MSEC; - ratio = (period * sysctl_sched_rt_ratio) >> SCHED_RT_FRAC_SHIFT; + ratio = (period * rt_ratio) >> SCHED_RT_FRAC_SHIFT; if (rt_rq->rt_time > ratio) { - rt_rq->rt_throttled = rq->clock + period - rt_rq->rt_time; + rt_rq->rt_throttled = 1; + sched_rt_ratio_dequeue(rt_rq); return 1; } return 0; } +static void __update_sched_rt_period(struct rt_rq *rt_rq, u64 period) +{ + unsigned long rt_ratio = sched_rt_ratio(rt_rq); + u64 ratio = (period * rt_ratio) >> SCHED_RT_FRAC_SHIFT; + + rt_rq->rt_time -= min(rt_rq->rt_time, ratio); + if (rt_rq->rt_throttled) { + rt_rq->rt_throttled = 0; + sched_rt_ratio_enqueue(rt_rq); + } +} + static void update_sched_rt_period(struct rq *rq) { - while (rq->clock > rq->rt_period_expire) { - u64 period, ratio; + struct rt_rq *rt_rq; + u64 period; + while (rq->clock > rq->rt_period_expire) { period = (u64)sysctl_sched_rt_period * NSEC_PER_MSEC; - ratio = (period * sysctl_sched_rt_ratio) >> SCHED_RT_FRAC_SHIFT; - - rq->rt.rt_time -= min(rq->rt.rt_time, ratio); rq->rt_period_expire += period; - } - /* - * When the rt throttle is expired, let them rip. - * (XXX: use hrtick when available) - */ - if (rq->rt.rt_throttled && rq->clock > rq->rt.rt_throttled) { - rq->rt.rt_throttled = 0; - if (!sched_rt_ratio_exceeded(rq, &rq->rt)) - resched_task(rq->curr); + for_each_leaf_rt_rq(rt_rq, rq) + __update_sched_rt_period(rt_rq, period); } } @@ -96,6 +216,8 @@ static void update_sched_rt_period(struct rq *rq) static void update_curr_rt(struct rq *rq) { struct task_struct *curr = rq->curr; + struct sched_rt_entity *rt_se = &curr->rt; + struct rt_rq *rt_rq = rt_rq_of_se(rt_se); u64 delta_exec; if (!task_has_rt_policy(curr)) @@ -111,95 +233,184 @@ static void update_curr_rt(struct rq *rq) curr->se.exec_start = rq->clock; cpuacct_charge(curr, delta_exec); - rq->rt.rt_time += delta_exec; - update_sched_rt_period(rq); - if (sched_rt_ratio_exceeded(rq, &rq->rt)) + rt_rq->rt_time += delta_exec; + /* + * might make it a tad more accurate: + * + * update_sched_rt_period(rq); + */ + if (sched_rt_ratio_exceeded(rt_rq)) resched_task(curr); } -static inline void inc_rt_tasks(struct task_struct *p, struct rq *rq) +static inline +void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { - WARN_ON(!rt_task(p)); - rq->rt.rt_nr_running++; + WARN_ON(!rt_prio(rt_se_prio(rt_se))); + rt_rq->rt_nr_running++; +#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED + if (rt_se_prio(rt_se) < rt_rq->highest_prio) + rt_rq->highest_prio = rt_se_prio(rt_se); +#endif #ifdef CONFIG_SMP - if (p->prio < rq->rt.highest_prio) - rq->rt.highest_prio = p->prio; - if (p->nr_cpus_allowed > 1) + if (rt_se->nr_cpus_allowed > 1) { + struct rq *rq = rq_of_rt_rq(rt_rq); rq->rt.rt_nr_migratory++; + } - update_rt_migration(rq); -#endif /* CONFIG_SMP */ + update_rt_migration(rq_of_rt_rq(rt_rq)); +#endif } -static inline void dec_rt_tasks(struct task_struct *p, struct rq *rq) +static inline +void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { - WARN_ON(!rt_task(p)); - WARN_ON(!rq->rt.rt_nr_running); - rq->rt.rt_nr_running--; -#ifdef CONFIG_SMP - if (rq->rt.rt_nr_running) { + WARN_ON(!rt_prio(rt_se_prio(rt_se))); + WARN_ON(!rt_rq->rt_nr_running); + rt_rq->rt_nr_running--; +#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED + if (rt_rq->rt_nr_running) { struct rt_prio_array *array; - WARN_ON(p->prio < rq->rt.highest_prio); - if (p->prio == rq->rt.highest_prio) { + WARN_ON(rt_se_prio(rt_se) < rt_rq->highest_prio); + if (rt_se_prio(rt_se) == rt_rq->highest_prio) { /* recalculate */ - array = &rq->rt.active; - rq->rt.highest_prio = + array = &rt_rq->active; + rt_rq->highest_prio = sched_find_first_bit(array->bitmap); } /* otherwise leave rq->highest prio alone */ } else - rq->rt.highest_prio = MAX_RT_PRIO; - if (p->nr_cpus_allowed > 1) + rt_rq->highest_prio = MAX_RT_PRIO; +#endif +#ifdef CONFIG_SMP + if (rt_se->nr_cpus_allowed > 1) { + struct rq *rq = rq_of_rt_rq(rt_rq); rq->rt.rt_nr_migratory--; + } - update_rt_migration(rq); + update_rt_migration(rq_of_rt_rq(rt_rq)); #endif /* CONFIG_SMP */ } -static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup) +static void enqueue_rt_entity(struct sched_rt_entity *rt_se) { - struct rt_prio_array *array = &rq->rt.active; + struct rt_rq *rt_rq = rt_rq_of_se(rt_se); + struct rt_prio_array *array = &rt_rq->active; + struct rt_rq *group_rq = group_rt_rq(rt_se); - list_add_tail(&p->rt.run_list, array->queue + p->prio); - __set_bit(p->prio, array->bitmap); - inc_cpu_load(rq, p->se.load.weight); + if (group_rq && group_rq->rt_throttled) + return; - inc_rt_tasks(p, rq); + list_add_tail(&rt_se->run_list, array->queue + rt_se_prio(rt_se)); + __set_bit(rt_se_prio(rt_se), array->bitmap); - if (wakeup) - p->rt.timeout = 0; + inc_rt_tasks(rt_se, rt_rq); +} + +static void dequeue_rt_entity(struct sched_rt_entity *rt_se) +{ + struct rt_rq *rt_rq = rt_rq_of_se(rt_se); + struct rt_prio_array *array = &rt_rq->active; + + list_del_init(&rt_se->run_list); + if (list_empty(array->queue + rt_se_prio(rt_se))) + __clear_bit(rt_se_prio(rt_se), array->bitmap); + + dec_rt_tasks(rt_se, rt_rq); +} + +/* + * Because the prio of an upper entry depends on the lower + * entries, we must remove entries top - down. + * + * XXX: O(1/2 h^2) because we can only walk up, not down the chain. + * doesn't matter much for now, as h=2 for GROUP_SCHED. + */ +static void dequeue_rt_stack(struct task_struct *p) +{ + struct sched_rt_entity *rt_se, *top_se; + + /* + * dequeue all, top - down. + */ + do { + rt_se = &p->rt; + top_se = NULL; + for_each_sched_rt_entity(rt_se) { + if (on_rt_rq(rt_se)) + top_se = rt_se; + } + if (top_se) + dequeue_rt_entity(top_se); + } while (top_se); } /* * Adding/removing a task to/from a priority array: */ +static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup) +{ + struct sched_rt_entity *rt_se = &p->rt; + + if (wakeup) + rt_se->timeout = 0; + + dequeue_rt_stack(p); + + /* + * enqueue everybody, bottom - up. + */ + for_each_sched_rt_entity(rt_se) + enqueue_rt_entity(rt_se); + + inc_cpu_load(rq, p->se.load.weight); +} + static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep) { - struct rt_prio_array *array = &rq->rt.active; + struct sched_rt_entity *rt_se = &p->rt; + struct rt_rq *rt_rq; update_curr_rt(rq); - list_del(&p->rt.run_list); - if (list_empty(array->queue + p->prio)) - __clear_bit(p->prio, array->bitmap); - dec_cpu_load(rq, p->se.load.weight); + dequeue_rt_stack(p); + + /* + * re-enqueue all non-empty rt_rq entities. + */ + for_each_sched_rt_entity(rt_se) { + rt_rq = group_rt_rq(rt_se); + if (rt_rq && rt_rq->rt_nr_running) + enqueue_rt_entity(rt_se); + } - dec_rt_tasks(p, rq); + dec_cpu_load(rq, p->se.load.weight); } /* * Put task to the end of the run list without the overhead of dequeue * followed by enqueue. */ +static +void requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) +{ + struct rt_prio_array *array = &rt_rq->active; + + list_move_tail(&rt_se->run_list, array->queue + rt_se_prio(rt_se)); +} + static void requeue_task_rt(struct rq *rq, struct task_struct *p) { - struct rt_prio_array *array = &rq->rt.active; + struct sched_rt_entity *rt_se = &p->rt; + struct rt_rq *rt_rq; - list_move_tail(&p->rt.run_list, array->queue + p->prio); + for_each_sched_rt_entity(rt_se) { + rt_rq = rt_rq_of_se(rt_se); + requeue_rt_entity(rt_rq, rt_se); + } } -static void -yield_task_rt(struct rq *rq) +static void yield_task_rt(struct rq *rq) { requeue_task_rt(rq, rq->curr); } @@ -229,7 +440,7 @@ static int select_task_rq_rt(struct task_struct *p, int sync) * cold cache anyway. */ if (unlikely(rt_task(rq->curr)) && - (p->nr_cpus_allowed > 1)) { + (p->rt.nr_cpus_allowed > 1)) { int cpu = find_lowest_rq(p); return (cpu == -1) ? task_cpu(p) : cpu; @@ -252,27 +463,51 @@ static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p) resched_task(rq->curr); } -static struct task_struct *pick_next_task_rt(struct rq *rq) +static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq, + struct rt_rq *rt_rq) { - struct rt_prio_array *array = &rq->rt.active; - struct task_struct *next; + struct rt_prio_array *array = &rt_rq->active; + struct sched_rt_entity *next = NULL; struct list_head *queue; - struct rt_rq *rt_rq = &rq->rt; int idx; - if (sched_rt_ratio_exceeded(rq, rt_rq)) - return NULL; + if (sched_rt_ratio_exceeded(rt_rq)) + goto out; idx = sched_find_first_bit(array->bitmap); - if (idx >= MAX_RT_PRIO) - return NULL; + BUG_ON(idx >= MAX_RT_PRIO); queue = array->queue + idx; - next = list_entry(queue->next, struct task_struct, rt.run_list); + next = list_entry(queue->next, struct sched_rt_entity, run_list); + out: + return next; +} - next->se.exec_start = rq->clock; +static struct task_struct *pick_next_task_rt(struct rq *rq) +{ + struct sched_rt_entity *rt_se; + struct task_struct *p; + struct rt_rq *rt_rq; - return next; + retry: + rt_rq = &rq->rt; + + if (unlikely(!rt_rq->rt_nr_running)) + return NULL; + + if (sched_rt_ratio_exceeded(rt_rq)) + return NULL; + + do { + rt_se = pick_next_rt_entity(rq, rt_rq); + if (unlikely(!rt_se)) + goto retry; + rt_rq = group_rt_rq(rt_se); + } while (rt_rq); + + p = rt_task_of(rt_se); + p->se.exec_start = rq->clock; + return p; } static void put_prev_task_rt(struct rq *rq, struct task_struct *p) @@ -282,6 +517,7 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p) } #ifdef CONFIG_SMP + /* Only try algorithms three times */ #define RT_MAX_TRIES 3 @@ -292,7 +528,7 @@ static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) { if (!task_running(rq, p) && (cpu < 0 || cpu_isset(cpu, p->cpus_allowed)) && - (p->nr_cpus_allowed > 1)) + (p->rt.nr_cpus_allowed > 1)) return 1; return 0; } @@ -300,52 +536,33 @@ static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) /* Return the second highest RT task, NULL otherwise */ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu) { - struct rt_prio_array *array = &rq->rt.active; - struct task_struct *next; - struct list_head *queue; + struct task_struct *next = NULL; + struct sched_rt_entity *rt_se; + struct rt_prio_array *array; + struct rt_rq *rt_rq; int idx; - if (likely(rq->rt.rt_nr_running < 2)) - return NULL; - - idx = sched_find_first_bit(array->bitmap); - if (unlikely(idx >= MAX_RT_PRIO)) { - WARN_ON(1); /* rt_nr_running is bad */ - return NULL; - } - - queue = array->queue + idx; - BUG_ON(list_empty(queue)); - - next = list_entry(queue->next, struct task_struct, rt.run_list); - if (unlikely(pick_rt_task(rq, next, cpu))) - goto out; - - if (queue->next->next != queue) { - /* same prio task */ - next = list_entry(queue->next->next, struct task_struct, - rt.run_list); - if (pick_rt_task(rq, next, cpu)) - goto out; - } - - retry: - /* slower, but more flexible */ - idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1); - if (unlikely(idx >= MAX_RT_PRIO)) - return NULL; - - queue = array->queue + idx; - BUG_ON(list_empty(queue)); - - list_for_each_entry(next, queue, rt.run_list) { - if (pick_rt_task(rq, next, cpu)) - goto out; + for_each_leaf_rt_rq(rt_rq, rq) { + array = &rt_rq->active; + idx = sched_find_first_bit(array->bitmap); + next_idx: + if (idx >= MAX_RT_PRIO) + continue; + if (next && next->prio < idx) + continue; + list_for_each_entry(rt_se, array->queue + idx, run_list) { + struct task_struct *p = rt_task_of(rt_se); + if (pick_rt_task(rq, p, cpu)) { + next = p; + break; + } + } + if (!next) { + idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1); + goto next_idx; + } } - goto retry; - - out: return next; } @@ -774,12 +991,12 @@ static void set_cpus_allowed_rt(struct task_struct *p, cpumask_t *new_mask) * Update the migration status of the RQ if we have an RT task * which is running AND changing its weight value. */ - if (p->se.on_rq && (weight != p->nr_cpus_allowed)) { + if (p->se.on_rq && (weight != p->rt.nr_cpus_allowed)) { struct rq *rq = task_rq(p); - if ((p->nr_cpus_allowed <= 1) && (weight > 1)) { + if ((p->rt.nr_cpus_allowed <= 1) && (weight > 1)) { rq->rt.rt_nr_migratory++; - } else if ((p->nr_cpus_allowed > 1) && (weight <= 1)) { + } else if ((p->rt.nr_cpus_allowed > 1) && (weight <= 1)) { BUG_ON(!rq->rt.rt_nr_migratory); rq->rt.rt_nr_migratory--; } @@ -788,7 +1005,7 @@ static void set_cpus_allowed_rt(struct task_struct *p, cpumask_t *new_mask) } p->cpus_allowed = *new_mask; - p->nr_cpus_allowed = weight; + p->rt.nr_cpus_allowed = weight; } /* Assumes rq->lock is held */ -- cgit v1.2.3 From 614ee1f61f667b02165c1ae0c1357048dc6d94a0 Mon Sep 17 00:00:00 2001 From: Mike Galbraith Date: Fri, 25 Jan 2008 21:08:30 +0100 Subject: sched: pull_rt_task() cleanup "goto out" is an odd way to spell "skip". Signed-off-by: Mike Galbraith Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 1178257613a..1144bf55669 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -877,10 +877,8 @@ static int pull_rt_task(struct rq *this_rq) /* * Are there still pullable RT tasks? */ - if (src_rq->rt.rt_nr_running <= 1) { - spin_unlock(&src_rq->lock); - continue; - } + if (src_rq->rt.rt_nr_running <= 1) + goto skip; p = pick_next_highest_task_rt(src_rq, this_cpu); @@ -904,7 +902,7 @@ static int pull_rt_task(struct rq *this_rq) */ if (p->prio < src_rq->curr->prio || (next && next->prio < src_rq->curr->prio)) - goto out; + goto skip; ret = 1; @@ -924,7 +922,7 @@ static int pull_rt_task(struct rq *this_rq) next = p; } - out: + skip: spin_unlock(&src_rq->lock); } -- cgit v1.2.3 From 48d5e258216f1c7713633439beb98a38c7290649 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:31 +0100 Subject: sched: rt throttling vs no_hz We need to teach no_hz about the rt throttling because its tick driven. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/sched.c | 23 ++++++++++++++++++++++- kernel/sched_rt.c | 30 ++++++++++++++++-------------- kernel/time/tick-sched.c | 5 +++++ 3 files changed, 43 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 5ea2c533b43..22712b2e058 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -442,6 +442,7 @@ struct rq { struct cfs_rq cfs; struct rt_rq rt; u64 rt_period_expire; + int rt_throttled; #ifdef CONFIG_FAIR_GROUP_SCHED /* list of leaf cfs_rq on this cpu: */ @@ -594,6 +595,23 @@ static void update_rq_clock(struct rq *rq) #define task_rq(p) cpu_rq(task_cpu(p)) #define cpu_curr(cpu) (cpu_rq(cpu)->curr) +unsigned long rt_needs_cpu(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + u64 delta; + + if (!rq->rt_throttled) + return 0; + + if (rq->clock > rq->rt_period_expire) + return 1; + + delta = rq->rt_period_expire - rq->clock; + do_div(delta, NSEC_PER_SEC / HZ); + + return (unsigned long)delta; +} + /* * Tunables that become constants when CONFIG_SCHED_DEBUG is off: */ @@ -7102,9 +7120,11 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq) /* delimiter for bitsearch: */ __set_bit(MAX_RT_PRIO, array->bitmap); +#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED + rt_rq->highest_prio = MAX_RT_PRIO; +#endif #ifdef CONFIG_SMP rt_rq->rt_nr_migratory = 0; - rt_rq->highest_prio = MAX_RT_PRIO; rt_rq->overloaded = 0; #endif @@ -7191,6 +7211,7 @@ void __init sched_init(void) &per_cpu(init_sched_rt_entity, i), i, 1); #endif rq->rt_period_expire = 0; + rq->rt_throttled = 0; for (j = 0; j < CPU_LOAD_IDX_MAX; j++) rq->cpu_load[j] = 0; diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 1144bf55669..8bfdb3f8a52 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -175,7 +175,11 @@ static int sched_rt_ratio_exceeded(struct rt_rq *rt_rq) ratio = (period * rt_ratio) >> SCHED_RT_FRAC_SHIFT; if (rt_rq->rt_time > ratio) { + struct rq *rq = rq_of_rt_rq(rt_rq); + + rq->rt_throttled = 1; rt_rq->rt_throttled = 1; + sched_rt_ratio_dequeue(rt_rq); return 1; } @@ -183,18 +187,6 @@ static int sched_rt_ratio_exceeded(struct rt_rq *rt_rq) return 0; } -static void __update_sched_rt_period(struct rt_rq *rt_rq, u64 period) -{ - unsigned long rt_ratio = sched_rt_ratio(rt_rq); - u64 ratio = (period * rt_ratio) >> SCHED_RT_FRAC_SHIFT; - - rt_rq->rt_time -= min(rt_rq->rt_time, ratio); - if (rt_rq->rt_throttled) { - rt_rq->rt_throttled = 0; - sched_rt_ratio_enqueue(rt_rq); - } -} - static void update_sched_rt_period(struct rq *rq) { struct rt_rq *rt_rq; @@ -204,8 +196,18 @@ static void update_sched_rt_period(struct rq *rq) period = (u64)sysctl_sched_rt_period * NSEC_PER_MSEC; rq->rt_period_expire += period; - for_each_leaf_rt_rq(rt_rq, rq) - __update_sched_rt_period(rt_rq, period); + for_each_leaf_rt_rq(rt_rq, rq) { + unsigned long rt_ratio = sched_rt_ratio(rt_rq); + u64 ratio = (period * rt_ratio) >> SCHED_RT_FRAC_SHIFT; + + rt_rq->rt_time -= min(rt_rq->rt_time, ratio); + if (rt_rq->rt_throttled) { + rt_rq->rt_throttled = 0; + sched_rt_ratio_enqueue(rt_rq); + } + } + + rq->rt_throttled = 0; } } diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index cb89fa8db11..5f9fb645b72 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -153,6 +153,7 @@ void tick_nohz_update_jiffies(void) void tick_nohz_stop_sched_tick(void) { unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags; + unsigned long rt_jiffies; struct tick_sched *ts; ktime_t last_update, expires, now, delta; struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; @@ -216,6 +217,10 @@ void tick_nohz_stop_sched_tick(void) next_jiffies = get_next_timer_interrupt(last_jiffies); delta_jiffies = next_jiffies - last_jiffies; + rt_jiffies = rt_needs_cpu(cpu); + if (rt_jiffies && rt_jiffies < delta_jiffies) + delta_jiffies = rt_jiffies; + if (rcu_needs_cpu(cpu)) delta_jiffies = 1; /* -- cgit v1.2.3 From 2d44ae4d7135b9aee26439b3523b43473381bc5f Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:31 +0100 Subject: hrtimer: clean up cpu->base locking tricks In order to more easily allow for the scheduler to use timers, clean up the locking a bit. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/hrtimer.c | 20 +++++++++++++++++++- kernel/time/tick-sched.c | 8 -------- 2 files changed, 19 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index f994bb8065e..9f850ca032b 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -1063,7 +1063,9 @@ void hrtimer_interrupt(struct clock_event_device *dev) basenow = ktime_add(now, base->offset); while ((node = base->first)) { + enum hrtimer_restart (*fn)(struct hrtimer *); struct hrtimer *timer; + int restart; timer = rb_entry(node, struct hrtimer, node); @@ -1091,13 +1093,29 @@ void hrtimer_interrupt(struct clock_event_device *dev) HRTIMER_STATE_CALLBACK, 0); timer_stats_account_hrtimer(timer); + fn = timer->function; + if (timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ) { + /* + * Used for scheduler timers, avoid lock + * inversion with rq->lock and tasklist_lock. + * + * These timers are required to deal with + * enqueue expiry themselves and are not + * allowed to migrate. + */ + spin_unlock(&cpu_base->lock); + restart = fn(timer); + spin_lock(&cpu_base->lock); + } else + restart = fn(timer); + /* * Note: We clear the CALLBACK bit after * enqueue_hrtimer to avoid reprogramming of * the event hardware. This happens at the end * of this function anyway. */ - if (timer->function(timer) != HRTIMER_NORESTART) { + if (restart != HRTIMER_NORESTART) { BUG_ON(timer->state != HRTIMER_STATE_CALLBACK); enqueue_hrtimer(timer, base, 0); } diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 5f9fb645b72..1a21b6fdb67 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -514,7 +514,6 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) { struct tick_sched *ts = container_of(timer, struct tick_sched, sched_timer); - struct hrtimer_cpu_base *base = timer->base->cpu_base; struct pt_regs *regs = get_irq_regs(); ktime_t now = ktime_get(); int cpu = smp_processor_id(); @@ -552,15 +551,8 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) touch_softlockup_watchdog(); ts->idle_jiffies++; } - /* - * update_process_times() might take tasklist_lock, hence - * drop the base lock. sched-tick hrtimers are per-CPU and - * never accessible by userspace APIs, so this is safe to do. - */ - spin_unlock(&base->lock); update_process_times(user_mode(regs)); profile_tick(CPU_PROFILING); - spin_lock(&base->lock); } /* Do not restart, when we are in the idle loop */ -- cgit v1.2.3 From d3d74453c34f8fd87674a8cf5b8a327c68f22e99 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:31 +0100 Subject: hrtimer: fixup the HRTIMER_CB_IRQSAFE_NO_SOFTIRQ fallback Currently all highres=off timers are run from softirq context, but HRTIMER_CB_IRQSAFE_NO_SOFTIRQ timers expect to run from irq context. Fix this up by splitting it similar to the highres=on case. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/hrtimer.c | 270 ++++++++++++++++++++++++++++--------------------------- kernel/timer.c | 3 +- 2 files changed, 141 insertions(+), 132 deletions(-) (limited to 'kernel') diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 9f850ca032b..061ae28a36a 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -325,6 +325,22 @@ unsigned long ktime_divns(const ktime_t kt, s64 div) } #endif /* BITS_PER_LONG >= 64 */ +/* + * Check, whether the timer is on the callback pending list + */ +static inline int hrtimer_cb_pending(const struct hrtimer *timer) +{ + return timer->state & HRTIMER_STATE_PENDING; +} + +/* + * Remove a timer from the callback pending list + */ +static inline void hrtimer_remove_cb_pending(struct hrtimer *timer) +{ + list_del_init(&timer->cb_entry); +} + /* High resolution timer related functions */ #ifdef CONFIG_HIGH_RES_TIMERS @@ -493,22 +509,6 @@ void hres_timers_resume(void) retrigger_next_event(NULL); } -/* - * Check, whether the timer is on the callback pending list - */ -static inline int hrtimer_cb_pending(const struct hrtimer *timer) -{ - return timer->state & HRTIMER_STATE_PENDING; -} - -/* - * Remove a timer from the callback pending list - */ -static inline void hrtimer_remove_cb_pending(struct hrtimer *timer) -{ - list_del_init(&timer->cb_entry); -} - /* * Initialize the high resolution related parts of cpu_base */ @@ -516,7 +516,6 @@ static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { base->expires_next.tv64 = KTIME_MAX; base->hres_active = 0; - INIT_LIST_HEAD(&base->cb_pending); } /* @@ -524,7 +523,6 @@ static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) */ static inline void hrtimer_init_timer_hres(struct hrtimer *timer) { - INIT_LIST_HEAD(&timer->cb_entry); } /* @@ -618,10 +616,13 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, { return 0; } -static inline int hrtimer_cb_pending(struct hrtimer *timer) { return 0; } -static inline void hrtimer_remove_cb_pending(struct hrtimer *timer) { } static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { } static inline void hrtimer_init_timer_hres(struct hrtimer *timer) { } +static inline int hrtimer_reprogram(struct hrtimer *timer, + struct hrtimer_clock_base *base) +{ + return 0; +} #endif /* CONFIG_HIGH_RES_TIMERS */ @@ -1001,6 +1002,7 @@ void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, clock_id = CLOCK_MONOTONIC; timer->base = &cpu_base->clock_base[clock_id]; + INIT_LIST_HEAD(&timer->cb_entry); hrtimer_init_timer_hres(timer); #ifdef CONFIG_TIMER_STATS @@ -1030,6 +1032,85 @@ int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp) } EXPORT_SYMBOL_GPL(hrtimer_get_res); +static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base) +{ + spin_lock_irq(&cpu_base->lock); + + while (!list_empty(&cpu_base->cb_pending)) { + enum hrtimer_restart (*fn)(struct hrtimer *); + struct hrtimer *timer; + int restart; + + timer = list_entry(cpu_base->cb_pending.next, + struct hrtimer, cb_entry); + + timer_stats_account_hrtimer(timer); + + fn = timer->function; + __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0); + spin_unlock_irq(&cpu_base->lock); + + restart = fn(timer); + + spin_lock_irq(&cpu_base->lock); + + timer->state &= ~HRTIMER_STATE_CALLBACK; + if (restart == HRTIMER_RESTART) { + BUG_ON(hrtimer_active(timer)); + /* + * Enqueue the timer, allow reprogramming of the event + * device + */ + enqueue_hrtimer(timer, timer->base, 1); + } else if (hrtimer_active(timer)) { + /* + * If the timer was rearmed on another CPU, reprogram + * the event device. + */ + if (timer->base->first == &timer->node) + hrtimer_reprogram(timer, timer->base); + } + } + spin_unlock_irq(&cpu_base->lock); +} + +static void __run_hrtimer(struct hrtimer *timer) +{ + struct hrtimer_clock_base *base = timer->base; + struct hrtimer_cpu_base *cpu_base = base->cpu_base; + enum hrtimer_restart (*fn)(struct hrtimer *); + int restart; + + __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); + timer_stats_account_hrtimer(timer); + + fn = timer->function; + if (timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ) { + /* + * Used for scheduler timers, avoid lock inversion with + * rq->lock and tasklist_lock. + * + * These timers are required to deal with enqueue expiry + * themselves and are not allowed to migrate. + */ + spin_unlock(&cpu_base->lock); + restart = fn(timer); + spin_lock(&cpu_base->lock); + } else + restart = fn(timer); + + /* + * Note: We clear the CALLBACK bit after enqueue_hrtimer to avoid + * reprogramming of the event hardware. This happens at the end of this + * function anyway. + */ + if (restart != HRTIMER_NORESTART) { + BUG_ON(timer->state != HRTIMER_STATE_CALLBACK); + enqueue_hrtimer(timer, base, 0); + } + timer->state &= ~HRTIMER_STATE_CALLBACK; +} + #ifdef CONFIG_HIGH_RES_TIMERS /* @@ -1063,9 +1144,7 @@ void hrtimer_interrupt(struct clock_event_device *dev) basenow = ktime_add(now, base->offset); while ((node = base->first)) { - enum hrtimer_restart (*fn)(struct hrtimer *); struct hrtimer *timer; - int restart; timer = rb_entry(node, struct hrtimer, node); @@ -1089,37 +1168,7 @@ void hrtimer_interrupt(struct clock_event_device *dev) continue; } - __remove_hrtimer(timer, base, - HRTIMER_STATE_CALLBACK, 0); - timer_stats_account_hrtimer(timer); - - fn = timer->function; - if (timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ) { - /* - * Used for scheduler timers, avoid lock - * inversion with rq->lock and tasklist_lock. - * - * These timers are required to deal with - * enqueue expiry themselves and are not - * allowed to migrate. - */ - spin_unlock(&cpu_base->lock); - restart = fn(timer); - spin_lock(&cpu_base->lock); - } else - restart = fn(timer); - - /* - * Note: We clear the CALLBACK bit after - * enqueue_hrtimer to avoid reprogramming of - * the event hardware. This happens at the end - * of this function anyway. - */ - if (restart != HRTIMER_NORESTART) { - BUG_ON(timer->state != HRTIMER_STATE_CALLBACK); - enqueue_hrtimer(timer, base, 0); - } - timer->state &= ~HRTIMER_STATE_CALLBACK; + __run_hrtimer(timer); } spin_unlock(&cpu_base->lock); base++; @@ -1140,52 +1189,41 @@ void hrtimer_interrupt(struct clock_event_device *dev) static void run_hrtimer_softirq(struct softirq_action *h) { - struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); - - spin_lock_irq(&cpu_base->lock); - - while (!list_empty(&cpu_base->cb_pending)) { - enum hrtimer_restart (*fn)(struct hrtimer *); - struct hrtimer *timer; - int restart; - - timer = list_entry(cpu_base->cb_pending.next, - struct hrtimer, cb_entry); + run_hrtimer_pending(&__get_cpu_var(hrtimer_bases)); +} - timer_stats_account_hrtimer(timer); +#endif /* CONFIG_HIGH_RES_TIMERS */ - fn = timer->function; - __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0); - spin_unlock_irq(&cpu_base->lock); +/* + * Called from timer softirq every jiffy, expire hrtimers: + * + * For HRT its the fall back code to run the softirq in the timer + * softirq context in case the hrtimer initialization failed or has + * not been done yet. + */ +void hrtimer_run_pending(void) +{ + struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); - restart = fn(timer); + if (hrtimer_hres_active()) + return; - spin_lock_irq(&cpu_base->lock); + /* + * This _is_ ugly: We have to check in the softirq context, + * whether we can switch to highres and / or nohz mode. The + * clocksource switch happens in the timer interrupt with + * xtime_lock held. Notification from there only sets the + * check bit in the tick_oneshot code, otherwise we might + * deadlock vs. xtime_lock. + */ + if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) + hrtimer_switch_to_hres(); - timer->state &= ~HRTIMER_STATE_CALLBACK; - if (restart == HRTIMER_RESTART) { - BUG_ON(hrtimer_active(timer)); - /* - * Enqueue the timer, allow reprogramming of the event - * device - */ - enqueue_hrtimer(timer, timer->base, 1); - } else if (hrtimer_active(timer)) { - /* - * If the timer was rearmed on another CPU, reprogram - * the event device. - */ - if (timer->base->first == &timer->node) - hrtimer_reprogram(timer, timer->base); - } - } - spin_unlock_irq(&cpu_base->lock); + run_hrtimer_pending(cpu_base); } -#endif /* CONFIG_HIGH_RES_TIMERS */ - /* - * Expire the per base hrtimer-queue: + * Called from hardirq context every jiffy */ static inline void run_hrtimer_queue(struct hrtimer_cpu_base *cpu_base, int index) @@ -1199,46 +1237,27 @@ static inline void run_hrtimer_queue(struct hrtimer_cpu_base *cpu_base, if (base->get_softirq_time) base->softirq_time = base->get_softirq_time(); - spin_lock_irq(&cpu_base->lock); + spin_lock(&cpu_base->lock); while ((node = base->first)) { struct hrtimer *timer; - enum hrtimer_restart (*fn)(struct hrtimer *); - int restart; timer = rb_entry(node, struct hrtimer, node); if (base->softirq_time.tv64 <= timer->expires.tv64) break; -#ifdef CONFIG_HIGH_RES_TIMERS - WARN_ON_ONCE(timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ); -#endif - timer_stats_account_hrtimer(timer); - - fn = timer->function; - __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); - spin_unlock_irq(&cpu_base->lock); - - restart = fn(timer); - - spin_lock_irq(&cpu_base->lock); - - timer->state &= ~HRTIMER_STATE_CALLBACK; - if (restart != HRTIMER_NORESTART) { - BUG_ON(hrtimer_active(timer)); - enqueue_hrtimer(timer, base, 0); + if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) { + __remove_hrtimer(timer, base, HRTIMER_STATE_PENDING, 0); + list_add_tail(&timer->cb_entry, + &base->cpu_base->cb_pending); + continue; } + + __run_hrtimer(timer); } - spin_unlock_irq(&cpu_base->lock); + spin_unlock(&cpu_base->lock); } -/* - * Called from timer softirq every jiffy, expire hrtimers: - * - * For HRT its the fall back code to run the softirq in the timer - * softirq context in case the hrtimer initialization failed or has - * not been done yet. - */ void hrtimer_run_queues(void) { struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); @@ -1247,18 +1266,6 @@ void hrtimer_run_queues(void) if (hrtimer_hres_active()) return; - /* - * This _is_ ugly: We have to check in the softirq context, - * whether we can switch to highres and / or nohz mode. The - * clocksource switch happens in the timer interrupt with - * xtime_lock held. Notification from there only sets the - * check bit in the tick_oneshot code, otherwise we might - * deadlock vs. xtime_lock. - */ - if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) - if (hrtimer_switch_to_hres()) - return; - hrtimer_get_softirq_time(cpu_base); for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) @@ -1407,6 +1414,7 @@ static void __cpuinit init_hrtimers_cpu(int cpu) for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) cpu_base->clock_base[i].cpu_base = cpu_base; + INIT_LIST_HEAD(&cpu_base->cb_pending); hrtimer_init_hres(cpu_base); } diff --git a/kernel/timer.c b/kernel/timer.c index 2a00c22203f..f739dfb539c 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -896,7 +896,7 @@ static void run_timer_softirq(struct softirq_action *h) { tvec_base_t *base = __get_cpu_var(tvec_bases); - hrtimer_run_queues(); + hrtimer_run_pending(); if (time_after_eq(jiffies, base->timer_jiffies)) __run_timers(base); @@ -907,6 +907,7 @@ static void run_timer_softirq(struct softirq_action *h) */ void run_local_timers(void) { + hrtimer_run_queues(); raise_softirq(TIMER_SOFTIRQ); softlockup_tick(); } -- cgit v1.2.3 From 37bb6cb4097e29ffee970065b74499cbf10603a3 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:32 +0100 Subject: hrtimer: unlock hrtimer_wakeup hrtimer_wakeup creates a base->lock rq->lock lock dependancy. Avoid this by switching to HRTIMER_CB_IRQSAFE_NO_SOFTIRQ which doesn't hold base->lock. This fully untangles hrtimer locks from the scheduler locks, and allows hrtimer usage in the scheduler proper. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/hrtimer.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 061ae28a36a..bd5d6b5060b 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -1293,7 +1293,7 @@ void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) sl->timer.function = hrtimer_wakeup; sl->task = task; #ifdef CONFIG_HIGH_RES_TIMERS - sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_RESTART; + sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ; #endif } @@ -1304,6 +1304,8 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod do { set_current_state(TASK_INTERRUPTIBLE); hrtimer_start(&t->timer, t->timer.expires, mode); + if (!hrtimer_active(&t->timer)) + t->task = NULL; if (likely(t->task)) schedule(); -- cgit v1.2.3 From 1020387f5f3b52929b387103cf976321981f8e26 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:32 +0100 Subject: sched: rt-group: reduce rescheduling Only reschedule if the new group has a higher prio task. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 8bfdb3f8a52..f1f215db3bd 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -94,8 +94,11 @@ static void sched_rt_ratio_enqueue(struct rt_rq *rt_rq) struct sched_rt_entity *rt_se = rt_rq->rt_se; if (rt_se && !on_rt_rq(rt_se) && rt_rq->rt_nr_running) { + struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr; + enqueue_rt_entity(rt_se); - resched_task(rq_of_rt_rq(rt_rq)->curr); + if (rt_rq->highest_prio < curr->prio) + resched_task(curr); } } -- cgit v1.2.3 From 5a52dd50091b6a6e710a1293db741028f8cc5aac Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:32 +0100 Subject: sched: rt-watchdog: fix .rlim_max = RLIM_INFINITY Remove the curious logic to set it_sched_expires in the future. It useless because rt.timeout wouldn't be incremented anyway. Explicity check for RLIM_INFINITY as a test programm that had a 1s soft limit and a inf hard limit would SIGKILL at 1s. This is because RLIM_INFINITY+d-1 is d-2. Signed-off-by: Peter Zijlsta CC: Michal Schmidt Signed-off-by: Ingo Molnar --- kernel/posix-cpu-timers.c | 3 ++- kernel/sched_rt.c | 8 +------- 2 files changed, 3 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index 2c076b36c4f..0b7c82ac467 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c @@ -1020,7 +1020,8 @@ static void check_thread_timers(struct task_struct *tsk, unsigned long hard = sig->rlim[RLIMIT_RTTIME].rlim_max; unsigned long *soft = &sig->rlim[RLIMIT_RTTIME].rlim_cur; - if (tsk->rt.timeout > DIV_ROUND_UP(hard, USEC_PER_SEC/HZ)) { + if (hard != RLIM_INFINITY && + tsk->rt.timeout > DIV_ROUND_UP(hard, USEC_PER_SEC/HZ)) { /* * At the hard limit, we just die. * No need to calculate anything else now. diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index f1f215db3bd..2dac5ebb8bc 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -1125,13 +1125,7 @@ static void watchdog(struct rq *rq, struct task_struct *p) p->rt.timeout++; next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ); - if (next > p->rt.timeout) { - u64 next_time = p->se.sum_exec_runtime; - - next_time += next * (NSEC_PER_SEC/HZ); - if (p->it_sched_expires > next_time) - p->it_sched_expires = next_time; - } else + if (p->rt.timeout > next) p->it_sched_expires = p->se.sum_exec_runtime; } } -- cgit v1.2.3 From 21aa9280b9f4e9e68d3fa8990df6c9d7fd71f994 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Fri, 25 Jan 2008 21:08:33 +0100 Subject: debug: show being-loaded/being-unloaded indicator for modules It's rather common that an oops/WARN_ON/BUG happens during the load or unload of a module. Unfortunatly, it's not always easy to see directly which module is being loaded/unloaded from the oops itself. Worse, it's not even always possible to ask the bug reporter, since there are so many components (udev etc) that auto-load modules that there's a good chance that even the reporter doesn't know which module this is. This patch extends the existing "show if it's tainting" print code, which is used as part of printing the modules in the oops/BUG/WARN_ON to include a "+" for "being loaded" and a "-" for "being unloaded". As a result this extension, the "taint_flags()" function gets renamed to "module_flags()" (and takes a module struct as argument, not a taint flags int). Signed-off-by: Arjan van de Ven Signed-off-by: Ingo Molnar --- kernel/module.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index dcb8a2cbf75..5bbf225ca07 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -2357,21 +2357,30 @@ static void m_stop(struct seq_file *m, void *p) mutex_unlock(&module_mutex); } -static char *taint_flags(unsigned int taints, char *buf) +static char *module_flags(struct module *mod, char *buf) { int bx = 0; - if (taints) { + if (mod->taints || + mod->state == MODULE_STATE_GOING || + mod->state == MODULE_STATE_COMING) { buf[bx++] = '('; - if (taints & TAINT_PROPRIETARY_MODULE) + if (mod->taints & TAINT_PROPRIETARY_MODULE) buf[bx++] = 'P'; - if (taints & TAINT_FORCED_MODULE) + if (mod->taints & TAINT_FORCED_MODULE) buf[bx++] = 'F'; /* * TAINT_FORCED_RMMOD: could be added. * TAINT_UNSAFE_SMP, TAINT_MACHINE_CHECK, TAINT_BAD_PAGE don't * apply to modules. */ + + /* Show a - for module-is-being-unloaded */ + if (mod->state == MODULE_STATE_GOING) + buf[bx++] = '-'; + /* Show a + for module-is-being-loaded */ + if (mod->state == MODULE_STATE_COMING) + buf[bx++] = '+'; buf[bx++] = ')'; } buf[bx] = '\0'; @@ -2398,7 +2407,7 @@ static int m_show(struct seq_file *m, void *p) /* Taints info */ if (mod->taints) - seq_printf(m, " %s", taint_flags(mod->taints, buf)); + seq_printf(m, " %s", module_flags(mod, buf)); seq_printf(m, "\n"); return 0; @@ -2493,7 +2502,7 @@ void print_modules(void) printk("Modules linked in:"); list_for_each_entry(mod, &modules, list) - printk(" %s%s", mod->name, taint_flags(mod->taints, buf)); + printk(" %s%s", mod->name, module_flags(mod, buf)); printk("\n"); } -- cgit v1.2.3 From e14af7eeb47ea96c52741c5e5fa010d33daf6973 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Fri, 25 Jan 2008 21:08:33 +0100 Subject: debug: track and print last unloaded module in the oops trace Based on a suggestion from Andi: In various cases, the unload of a module may leave some bad state around that causes a kernel crash AFTER a module is unloaded; and it's then hard to find which module caused that. This patch tracks the last unloaded module, and prints this as part of the module list in the oops trace. Right now, only the last 1 module is tracked; I expect that this is enough for the vast majority of cases where this information matters; if it turns out that tracking more is important, we can always extend it to that. [ mingo@elte.hu: build fix ] Signed-off-by: Arjan van de Ven Signed-off-by: Ingo Molnar --- kernel/module.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index 5bbf225ca07..1bb4c5e0d56 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -496,6 +496,8 @@ static struct module_attribute modinfo_##field = { \ MODINFO_ATTR(version); MODINFO_ATTR(srcversion); +static char last_unloaded_module[MODULE_NAME_LEN+1]; + #ifdef CONFIG_MODULE_UNLOAD /* Init the unload section of the module. */ static void module_unload_init(struct module *mod) @@ -719,6 +721,8 @@ sys_delete_module(const char __user *name_user, unsigned int flags) mod->exit(); mutex_lock(&module_mutex); } + /* Store the name of the last unloaded module for diagnostic purposes */ + sprintf(last_unloaded_module, mod->name); free_module(mod); out: @@ -2503,6 +2507,8 @@ void print_modules(void) printk("Modules linked in:"); list_for_each_entry(mod, &modules, list) printk(" %s%s", mod->name, module_flags(mod, buf)); + if (last_unloaded_module[0]) + printk(" [last unloaded: %s]", last_unloaded_module); printk("\n"); } -- cgit v1.2.3 From 58b8a73ab8becfcaea84abc2a06038281efa4c8a Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:33 +0100 Subject: sched: make PREEMPT_BKL the default make PREEMPT_BKL the default. precursor to removal of the !PREEMPT_BKL code. Signed-off-by: Ingo Molnar --- kernel/Kconfig.preempt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 61fa116efcd..4420ef427f8 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -53,15 +53,8 @@ config PREEMPT endchoice config PREEMPT_BKL - bool "Preempt The Big Kernel Lock" + def_bool y depends on SMP || PREEMPT - default y - help - This option reduces the latency of the kernel by making the - big kernel lock preemptible. - - Say Y here if you are building a kernel for a desktop system. - Say N if you are unsure. config RCU_TRACE bool "Enable tracing for RCU - currently stats in debugfs" -- cgit v1.2.3 From 6478d8800b75253b2a934ddcb734e13ade023ad0 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:33 +0100 Subject: sched: remove the !PREEMPT_BKL code remove the !PREEMPT_BKL code. this removes 160 lines of legacy code. Signed-off-by: Ingo Molnar --- kernel/Kconfig.preempt | 4 ---- kernel/sched.c | 19 +++---------------- 2 files changed, 3 insertions(+), 20 deletions(-) (limited to 'kernel') diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 4420ef427f8..0669b70fa6a 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -52,10 +52,6 @@ config PREEMPT endchoice -config PREEMPT_BKL - def_bool y - depends on SMP || PREEMPT - config RCU_TRACE bool "Enable tracing for RCU - currently stats in debugfs" select DEBUG_FS diff --git a/kernel/sched.c b/kernel/sched.c index 22712b2e058..629614ad035 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -3955,10 +3955,9 @@ EXPORT_SYMBOL(schedule); asmlinkage void __sched preempt_schedule(void) { struct thread_info *ti = current_thread_info(); -#ifdef CONFIG_PREEMPT_BKL struct task_struct *task = current; int saved_lock_depth; -#endif + /* * If there is a non-zero preempt_count or interrupts are disabled, * we do not want to preempt the current task. Just return.. @@ -3974,14 +3973,10 @@ asmlinkage void __sched preempt_schedule(void) * clear ->lock_depth so that schedule() doesnt * auto-release the semaphore: */ -#ifdef CONFIG_PREEMPT_BKL saved_lock_depth = task->lock_depth; task->lock_depth = -1; -#endif schedule(); -#ifdef CONFIG_PREEMPT_BKL task->lock_depth = saved_lock_depth; -#endif sub_preempt_count(PREEMPT_ACTIVE); /* @@ -4002,10 +3997,9 @@ EXPORT_SYMBOL(preempt_schedule); asmlinkage void __sched preempt_schedule_irq(void) { struct thread_info *ti = current_thread_info(); -#ifdef CONFIG_PREEMPT_BKL struct task_struct *task = current; int saved_lock_depth; -#endif + /* Catch callers which need to be fixed */ BUG_ON(ti->preempt_count || !irqs_disabled()); @@ -4017,16 +4011,12 @@ asmlinkage void __sched preempt_schedule_irq(void) * clear ->lock_depth so that schedule() doesnt * auto-release the semaphore: */ -#ifdef CONFIG_PREEMPT_BKL saved_lock_depth = task->lock_depth; task->lock_depth = -1; -#endif local_irq_enable(); schedule(); local_irq_disable(); -#ifdef CONFIG_PREEMPT_BKL task->lock_depth = saved_lock_depth; -#endif sub_preempt_count(PREEMPT_ACTIVE); /* @@ -5241,11 +5231,8 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) spin_unlock_irqrestore(&rq->lock, flags); /* Set the preempt count _outside_ the spinlocks! */ -#if defined(CONFIG_PREEMPT) && !defined(CONFIG_PREEMPT_BKL) - task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0); -#else task_thread_info(idle)->preempt_count = 0; -#endif + /* * The idle tasks have their own, simple scheduling class: */ -- cgit v1.2.3 From 1ad82fd547c716f96e544b477e0bdbfa2d647529 Mon Sep 17 00:00:00 2001 From: Paolo Ciarrocchi Date: Fri, 25 Jan 2008 21:08:33 +0100 Subject: debug: clean up kernel/profile.c Before: total: 25 errors, 13 warnings, 602 lines checked After: total: 0 errors, 2 warnings, 601 lines checked No code changed: kernel/profile.o: text data bss dec hex filename 3048 236 24 3308 cec profile.o.before 3048 236 24 3308 cec profile.o.after md5: 2501d64748a4d350dffb11203e2a5182 profile.o.before.asm 2501d64748a4d350dffb11203e2a5182 profile.o.after.asm Signed-off-by: Paolo Ciarrocchi Signed-off-by: Ingo Molnar --- kernel/profile.c | 99 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 49 insertions(+), 50 deletions(-) (limited to 'kernel') diff --git a/kernel/profile.c b/kernel/profile.c index 5e95330e512..e64c2da11c0 100644 --- a/kernel/profile.c +++ b/kernel/profile.c @@ -52,7 +52,7 @@ static DEFINE_PER_CPU(int, cpu_profile_flip); static DEFINE_MUTEX(profile_flip_mutex); #endif /* CONFIG_SMP */ -static int __init profile_setup(char * str) +static int __init profile_setup(char *str) { static char __initdata schedstr[] = "schedule"; static char __initdata sleepstr[] = "sleep"; @@ -104,28 +104,28 @@ __setup("profile=", profile_setup); void __init profile_init(void) { - if (!prof_on) + if (!prof_on) return; - + /* only text is profiled */ prof_len = (_etext - _stext) >> prof_shift; prof_buffer = alloc_bootmem(prof_len*sizeof(atomic_t)); } /* Profile event notifications */ - + #ifdef CONFIG_PROFILING - + static BLOCKING_NOTIFIER_HEAD(task_exit_notifier); static ATOMIC_NOTIFIER_HEAD(task_free_notifier); static BLOCKING_NOTIFIER_HEAD(munmap_notifier); - -void profile_task_exit(struct task_struct * task) + +void profile_task_exit(struct task_struct *task) { blocking_notifier_call_chain(&task_exit_notifier, 0, task); } - -int profile_handoff_task(struct task_struct * task) + +int profile_handoff_task(struct task_struct *task) { int ret; ret = atomic_notifier_call_chain(&task_free_notifier, 0, task); @@ -137,52 +137,55 @@ void profile_munmap(unsigned long addr) blocking_notifier_call_chain(&munmap_notifier, 0, (void *)addr); } -int task_handoff_register(struct notifier_block * n) +int task_handoff_register(struct notifier_block *n) { return atomic_notifier_chain_register(&task_free_notifier, n); } +EXPORT_SYMBOL_GPL(task_handoff_register); -int task_handoff_unregister(struct notifier_block * n) +int task_handoff_unregister(struct notifier_block *n) { return atomic_notifier_chain_unregister(&task_free_notifier, n); } +EXPORT_SYMBOL_GPL(task_handoff_unregister); -int profile_event_register(enum profile_type type, struct notifier_block * n) +int profile_event_register(enum profile_type type, struct notifier_block *n) { int err = -EINVAL; - + switch (type) { - case PROFILE_TASK_EXIT: - err = blocking_notifier_chain_register( - &task_exit_notifier, n); - break; - case PROFILE_MUNMAP: - err = blocking_notifier_chain_register( - &munmap_notifier, n); - break; + case PROFILE_TASK_EXIT: + err = blocking_notifier_chain_register( + &task_exit_notifier, n); + break; + case PROFILE_MUNMAP: + err = blocking_notifier_chain_register( + &munmap_notifier, n); + break; } - + return err; } +EXPORT_SYMBOL_GPL(profile_event_register); - -int profile_event_unregister(enum profile_type type, struct notifier_block * n) +int profile_event_unregister(enum profile_type type, struct notifier_block *n) { int err = -EINVAL; - + switch (type) { - case PROFILE_TASK_EXIT: - err = blocking_notifier_chain_unregister( - &task_exit_notifier, n); - break; - case PROFILE_MUNMAP: - err = blocking_notifier_chain_unregister( - &munmap_notifier, n); - break; + case PROFILE_TASK_EXIT: + err = blocking_notifier_chain_unregister( + &task_exit_notifier, n); + break; + case PROFILE_MUNMAP: + err = blocking_notifier_chain_unregister( + &munmap_notifier, n); + break; } return err; } +EXPORT_SYMBOL_GPL(profile_event_unregister); int register_timer_hook(int (*hook)(struct pt_regs *)) { @@ -191,6 +194,7 @@ int register_timer_hook(int (*hook)(struct pt_regs *)) timer_hook = hook; return 0; } +EXPORT_SYMBOL_GPL(register_timer_hook); void unregister_timer_hook(int (*hook)(struct pt_regs *)) { @@ -199,13 +203,7 @@ void unregister_timer_hook(int (*hook)(struct pt_regs *)) /* make sure all CPUs see the NULL hook */ synchronize_sched(); /* Allow ongoing interrupts to complete. */ } - -EXPORT_SYMBOL_GPL(register_timer_hook); EXPORT_SYMBOL_GPL(unregister_timer_hook); -EXPORT_SYMBOL_GPL(task_handoff_register); -EXPORT_SYMBOL_GPL(task_handoff_unregister); -EXPORT_SYMBOL_GPL(profile_event_register); -EXPORT_SYMBOL_GPL(profile_event_unregister); #endif /* CONFIG_PROFILING */ @@ -366,7 +364,7 @@ static int __devinit profile_cpu_callback(struct notifier_block *info, per_cpu(cpu_profile_hits, cpu)[0] = page_address(page); } break; - out_free: +out_free: page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[1]); per_cpu(cpu_profile_hits, cpu)[1] = NULL; __free_page(page); @@ -409,7 +407,6 @@ void profile_hits(int type, void *__pc, unsigned int nr_hits) atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]); } #endif /* !CONFIG_SMP */ - EXPORT_SYMBOL_GPL(profile_hits); void profile_tick(int type) @@ -427,7 +424,7 @@ void profile_tick(int type) #include #include -static int prof_cpu_mask_read_proc (char *page, char **start, off_t off, +static int prof_cpu_mask_read_proc(char *page, char **start, off_t off, int count, int *eof, void *data) { int len = cpumask_scnprintf(page, count, *(cpumask_t *)data); @@ -437,8 +434,8 @@ static int prof_cpu_mask_read_proc (char *page, char **start, off_t off, return len; } -static int prof_cpu_mask_write_proc (struct file *file, const char __user *buffer, - unsigned long count, void *data) +static int prof_cpu_mask_write_proc(struct file *file, + const char __user *buffer, unsigned long count, void *data) { cpumask_t *mask = (cpumask_t *)data; unsigned long full_count = count, err; @@ -457,7 +454,8 @@ void create_prof_cpu_mask(struct proc_dir_entry *root_irq_dir) struct proc_dir_entry *entry; /* create /proc/irq/prof_cpu_mask */ - if (!(entry = create_proc_entry("prof_cpu_mask", 0600, root_irq_dir))) + entry = create_proc_entry("prof_cpu_mask", 0600, root_irq_dir); + if (!entry) return; entry->data = (void *)&prof_cpu_mask; entry->read_proc = prof_cpu_mask_read_proc; @@ -475,7 +473,7 @@ read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos) { unsigned long p = *ppos; ssize_t read; - char * pnt; + char *pnt; unsigned int sample_step = 1 << prof_shift; profile_flip_buffers(); @@ -486,12 +484,12 @@ read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos) read = 0; while (p < sizeof(unsigned int) && count > 0) { - if (put_user(*((char *)(&sample_step)+p),buf)) + if (put_user(*((char *)(&sample_step)+p), buf)) return -EFAULT; buf++; p++; count--; read++; } pnt = (char *)prof_buffer + p - sizeof(atomic_t); - if (copy_to_user(buf,(void *)pnt,count)) + if (copy_to_user(buf, (void *)pnt, count)) return -EFAULT; read += count; *ppos += read; @@ -508,7 +506,7 @@ static ssize_t write_profile(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { #ifdef CONFIG_SMP - extern int setup_profiling_timer (unsigned int multiplier); + extern int setup_profiling_timer(unsigned int multiplier); if (count == sizeof(int)) { unsigned int multiplier; @@ -591,7 +589,8 @@ static int __init create_proc_profile(void) return 0; if (create_hash_tables()) return -1; - if (!(entry = create_proc_entry("profile", S_IWUSR | S_IRUGO, NULL))) + entry = create_proc_entry("profile", S_IWUSR | S_IRUGO, NULL); + if (!entry) return 0; entry->proc_fops = &proc_profile_operations; entry->size = (1+prof_len) * sizeof(atomic_t); -- cgit v1.2.3 From 4f05b98d54b140ed3c5851d5d5156e9918c6305d Mon Sep 17 00:00:00 2001 From: Michal Schmidt Date: Fri, 25 Jan 2008 21:08:33 +0100 Subject: sched: fix, always create kernel threads with normal priority Ensure that the kernel threads are created with the usual nice level and affinity even if kthreadd's properties were changed from the default by root. Signed-off-by: Michal Schmidt Signed-off-by: Ingo Molnar --- kernel/kthread.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/kthread.c b/kernel/kthread.c index dcfe724300e..0ac887882f9 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -15,6 +15,8 @@ #include #include +#define KTHREAD_NICE_LEVEL (-5) + static DEFINE_SPINLOCK(kthread_create_lock); static LIST_HEAD(kthread_create_list); struct task_struct *kthreadd_task; @@ -94,10 +96,18 @@ static void create_kthread(struct kthread_create_info *create) if (pid < 0) { create->result = ERR_PTR(pid); } else { + struct sched_param param = { .sched_priority = 0 }; wait_for_completion(&create->started); read_lock(&tasklist_lock); create->result = find_task_by_pid(pid); read_unlock(&tasklist_lock); + /* + * root may have changed our (kthreadd's) priority or CPU mask. + * The kernel thread should not inherit these properties. + */ + sched_setscheduler(create->result, SCHED_NORMAL, ¶m); + set_user_nice(create->result, KTHREAD_NICE_LEVEL); + set_cpus_allowed(create->result, CPU_MASK_ALL); } complete(&create->done); } @@ -221,7 +231,7 @@ int kthreadd(void *unused) /* Setup a clean context for our children to inherit. */ set_task_comm(tsk, "kthreadd"); ignore_signals(tsk); - set_user_nice(tsk, -5); + set_user_nice(tsk, KTHREAD_NICE_LEVEL); set_cpus_allowed(tsk, CPU_MASK_ALL); current->flags |= PF_NOFREEZE; -- cgit v1.2.3 From 782daeee3d596282bfee4cd9e976c86be0e194a8 Mon Sep 17 00:00:00 2001 From: Guillaume Chazarain Date: Fri, 25 Jan 2008 21:08:33 +0100 Subject: sched: fix rq->clock warps on frequency changes sched: fix rq->clock warps on frequency changes Fix 2bacec8c318ca0418c0ee9ac662ee44207765dd4 (sched: touch softlockup watchdog after idling) that reintroduced warps on frequency changes. touch_softlockup_watchdog() calls __update_rq_clock that checks rq->clock for warps, so call it after adjusting rq->clock. Signed-off-by: Guillaume Chazarain Signed-off-by: Ingo Molnar --- kernel/sched.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 629614ad035..3995d167985 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -858,7 +858,6 @@ void sched_clock_idle_wakeup_event(u64 delta_ns) struct rq *rq = cpu_rq(smp_processor_id()); u64 now = sched_clock(); - touch_softlockup_watchdog(); rq->idle_clock += delta_ns; /* * Override the previous timestamp and ignore all @@ -870,6 +869,7 @@ void sched_clock_idle_wakeup_event(u64 delta_ns) rq->prev_clock_raw = now; rq->clock += delta_ns; spin_unlock(&rq->lock); + touch_softlockup_watchdog(); } EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); -- cgit v1.2.3 From cc203d2422004498909c4886d1b94a2e388d973e Mon Sep 17 00:00:00 2001 From: Guillaume Chazarain Date: Fri, 25 Jan 2008 21:08:34 +0100 Subject: sched: monitor clock underflows in /proc/sched_debug We monitor clock overflows, let's also monitor clock underflows. Signed-off-by: Guillaume Chazarain Signed-off-by: Ingo Molnar --- kernel/sched.c | 6 ++++-- kernel/sched_debug.c | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 3995d167985..4d3a5a70086 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -465,7 +465,7 @@ struct rq { u64 clock, prev_clock_raw; s64 clock_max_delta; - unsigned int clock_warps, clock_overflows; + unsigned int clock_warps, clock_overflows, clock_underflows; u64 idle_clock; unsigned int clock_deep_idle_events; u64 tick_timestamp; @@ -3736,8 +3736,10 @@ void scheduler_tick(void) /* * Let rq->clock advance by at least TICK_NSEC: */ - if (unlikely(rq->clock < next_tick)) + if (unlikely(rq->clock < next_tick)) { rq->clock = next_tick; + rq->clock_underflows++; + } rq->tick_timestamp = rq->clock; update_cpu_load(rq); curr->sched_class->task_tick(rq, curr, 0); diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index 80fbbfc0429..9e5de098d47 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c @@ -179,6 +179,7 @@ static void print_cpu(struct seq_file *m, int cpu) PN(prev_clock_raw); P(clock_warps); P(clock_overflows); + P(clock_underflows); P(clock_deep_idle_events); PN(clock_max_delta); P(cpu_load[0]); -- cgit v1.2.3 From 326587b840785c60f5dc18557235a23bafefd620 Mon Sep 17 00:00:00 2001 From: Dmitry Adamushko Date: Fri, 25 Jan 2008 21:08:34 +0100 Subject: sched: fix goto retry in pick_next_task_rt() looking at it one more time: (1) it looks to me that there is no need to call sched_rt_ratio_exceeded() from pick_next_rt_entity() - [ for CONFIG_FAIR_GROUP_SCHED ] queues with rt_rq->rt_throttled are not within this 'tree-like hierarchy' (or whatever we should call it :-) - there is also no need to re-check 'rt_rq->rt_time > ratio' at this point as 'rt_rq->rt_time' couldn't have been increased since the last call to update_curr_rt() (which obviously calls sched_rt_ratio_esceeded()) well, it might be that 'ratio' for this rt_rq has been re-configured (and the period over which this rt_rq was active has not yet been finished)... but I don't think we should really take this into account. (2) now pick_next_rt_entity() must never return NULL, so let's change pick_next_task_rt() accordingly. Signed-off-by: Dmitry Adamushko Signed-off-by: Ingo Molnar --- kernel/sched_rt.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 2dac5ebb8bc..274b40d7bef 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -476,15 +476,12 @@ static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq, struct list_head *queue; int idx; - if (sched_rt_ratio_exceeded(rt_rq)) - goto out; - idx = sched_find_first_bit(array->bitmap); BUG_ON(idx >= MAX_RT_PRIO); queue = array->queue + idx; next = list_entry(queue->next, struct sched_rt_entity, run_list); - out: + return next; } @@ -494,7 +491,6 @@ static struct task_struct *pick_next_task_rt(struct rq *rq) struct task_struct *p; struct rt_rq *rt_rq; - retry: rt_rq = &rq->rt; if (unlikely(!rt_rq->rt_nr_running)) @@ -505,8 +501,7 @@ static struct task_struct *pick_next_task_rt(struct rq *rq) do { rt_se = pick_next_rt_entity(rq, rt_rq); - if (unlikely(!rt_se)) - goto retry; + BUG_ON(!rt_se); rt_rq = group_rt_rq(rt_se); } while (rt_rq); -- cgit v1.2.3 From 9745512ce79de686df354dc70a8d1a74d801892d Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Fri, 25 Jan 2008 21:08:34 +0100 Subject: sched: latencytop support LatencyTOP kernel infrastructure; it measures latencies in the scheduler and tracks it system wide and per process. Signed-off-by: Arjan van de Ven Signed-off-by: Ingo Molnar --- kernel/Makefile | 1 + kernel/fork.c | 1 + kernel/latencytop.c | 239 ++++++++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched_fair.c | 8 +- kernel/sysctl.c | 10 +++ 5 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 kernel/latencytop.c (limited to 'kernel') diff --git a/kernel/Makefile b/kernel/Makefile index 68755cd9a7e..390d4214626 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -62,6 +62,7 @@ obj-$(CONFIG_SYSCTL) += utsname_sysctl.o obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o obj-$(CONFIG_MARKERS) += marker.o +obj-$(CONFIG_LATENCYTOP) += latencytop.o ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y) # According to Alan Modra , the -fno-omit-frame-pointer is diff --git a/kernel/fork.c b/kernel/fork.c index 0c969f4fade..39d22b3357d 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1205,6 +1205,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, #ifdef TIF_SYSCALL_EMU clear_tsk_thread_flag(p, TIF_SYSCALL_EMU); #endif + clear_all_latency_tracing(p); /* Our parent execution domain becomes current domain These must match for thread signalling to apply */ diff --git a/kernel/latencytop.c b/kernel/latencytop.c new file mode 100644 index 00000000000..b4e3c85abe7 --- /dev/null +++ b/kernel/latencytop.c @@ -0,0 +1,239 @@ +/* + * latencytop.c: Latency display infrastructure + * + * (C) Copyright 2008 Intel Corporation + * Author: Arjan van de Ven + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static DEFINE_SPINLOCK(latency_lock); + +#define MAXLR 128 +static struct latency_record latency_record[MAXLR]; + +int latencytop_enabled; + +void clear_all_latency_tracing(struct task_struct *p) +{ + unsigned long flags; + + if (!latencytop_enabled) + return; + + spin_lock_irqsave(&latency_lock, flags); + memset(&p->latency_record, 0, sizeof(p->latency_record)); + p->latency_record_count = 0; + spin_unlock_irqrestore(&latency_lock, flags); +} + +static void clear_global_latency_tracing(void) +{ + unsigned long flags; + + spin_lock_irqsave(&latency_lock, flags); + memset(&latency_record, 0, sizeof(latency_record)); + spin_unlock_irqrestore(&latency_lock, flags); +} + +static void __sched +account_global_scheduler_latency(struct task_struct *tsk, struct latency_record *lat) +{ + int firstnonnull = MAXLR + 1; + int i; + + if (!latencytop_enabled) + return; + + /* skip kernel threads for now */ + if (!tsk->mm) + return; + + for (i = 0; i < MAXLR; i++) { + int q; + int same = 1; + /* Nothing stored: */ + if (!latency_record[i].backtrace[0]) { + if (firstnonnull > i) + firstnonnull = i; + continue; + } + for (q = 0 ; q < LT_BACKTRACEDEPTH ; q++) { + if (latency_record[i].backtrace[q] != + lat->backtrace[q]) + same = 0; + if (same && lat->backtrace[q] == 0) + break; + if (same && lat->backtrace[q] == ULONG_MAX) + break; + } + if (same) { + latency_record[i].count++; + latency_record[i].time += lat->time; + if (lat->time > latency_record[i].max) + latency_record[i].max = lat->time; + return; + } + } + + i = firstnonnull; + if (i >= MAXLR - 1) + return; + + /* Allocted a new one: */ + memcpy(&latency_record[i], lat, sizeof(struct latency_record)); +} + +static inline void store_stacktrace(struct task_struct *tsk, struct latency_record *lat) +{ + struct stack_trace trace; + + memset(&trace, 0, sizeof(trace)); + trace.max_entries = LT_BACKTRACEDEPTH; + trace.entries = &lat->backtrace[0]; + trace.skip = 0; + save_stack_trace_tsk(tsk, &trace); +} + +void __sched +account_scheduler_latency(struct task_struct *tsk, int usecs, int inter) +{ + unsigned long flags; + int i, q; + struct latency_record lat; + + if (!latencytop_enabled) + return; + + /* Long interruptible waits are generally user requested... */ + if (inter && usecs > 5000) + return; + + memset(&lat, 0, sizeof(lat)); + lat.count = 1; + lat.time = usecs; + lat.max = usecs; + store_stacktrace(tsk, &lat); + + spin_lock_irqsave(&latency_lock, flags); + + account_global_scheduler_latency(tsk, &lat); + + /* + * short term hack; if we're > 32 we stop; future we recycle: + */ + tsk->latency_record_count++; + if (tsk->latency_record_count >= LT_SAVECOUNT) + goto out_unlock; + + for (i = 0; i < LT_SAVECOUNT ; i++) { + struct latency_record *mylat; + int same = 1; + mylat = &tsk->latency_record[i]; + for (q = 0 ; q < LT_BACKTRACEDEPTH ; q++) { + if (mylat->backtrace[q] != + lat.backtrace[q]) + same = 0; + if (same && lat.backtrace[q] == 0) + break; + if (same && lat.backtrace[q] == ULONG_MAX) + break; + } + if (same) { + mylat->count++; + mylat->time += lat.time; + if (lat.time > mylat->max) + mylat->max = lat.time; + goto out_unlock; + } + } + + /* Allocated a new one: */ + i = tsk->latency_record_count; + memcpy(&tsk->latency_record[i], &lat, sizeof(struct latency_record)); + +out_unlock: + spin_unlock_irqrestore(&latency_lock, flags); +} + +static int lstats_show(struct seq_file *m, void *v) +{ + int i; + + seq_puts(m, "Latency Top version : v0.1\n"); + + for (i = 0; i < MAXLR; i++) { + if (latency_record[i].backtrace[0]) { + int q; + seq_printf(m, "%i %li %li ", + latency_record[i].count, + latency_record[i].time, + latency_record[i].max); + for (q = 0; q < LT_BACKTRACEDEPTH; q++) { + char sym[KSYM_NAME_LEN]; + char *c; + if (!latency_record[i].backtrace[q]) + break; + if (latency_record[i].backtrace[q] == ULONG_MAX) + break; + sprint_symbol(sym, latency_record[i].backtrace[q]); + c = strchr(sym, '+'); + if (c) + *c = 0; + seq_printf(m, "%s ", sym); + } + seq_printf(m, "\n"); + } + } + return 0; +} + +static ssize_t +lstats_write(struct file *file, const char __user *buf, size_t count, + loff_t *offs) +{ + clear_global_latency_tracing(); + + return count; +} + +static int lstats_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, lstats_show, NULL); +} + +static struct file_operations lstats_fops = { + .open = lstats_open, + .read = seq_read, + .write = lstats_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __init init_lstats_procfs(void) +{ + struct proc_dir_entry *pe; + + pe = create_proc_entry("latency_stats", 0644, NULL); + if (!pe) + return -ENOMEM; + + pe->proc_fops = &lstats_fops; + + return 0; +} +__initcall(init_lstats_procfs); diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 3dab1ff83c4..1b3b40ad7c5 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -20,6 +20,8 @@ * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra */ +#include + /* * Targeted preemption latency for CPU-bound tasks: * (default: 20ms * (1 + ilog(ncpus)), units: nanoseconds) @@ -434,6 +436,7 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) #ifdef CONFIG_SCHEDSTATS if (se->sleep_start) { u64 delta = rq_of(cfs_rq)->clock - se->sleep_start; + struct task_struct *tsk = task_of(se); if ((s64)delta < 0) delta = 0; @@ -443,9 +446,12 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) se->sleep_start = 0; se->sum_sleep_runtime += delta; + + account_scheduler_latency(tsk, delta >> 10, 1); } if (se->block_start) { u64 delta = rq_of(cfs_rq)->clock - se->block_start; + struct task_struct *tsk = task_of(se); if ((s64)delta < 0) delta = 0; @@ -462,11 +468,11 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) * time that the task spent sleeping: */ if (unlikely(prof_on == SLEEP_PROFILING)) { - struct task_struct *tsk = task_of(se); profile_hits(SLEEP_PROFILING, (void *)get_wchan(tsk), delta >> 20); } + account_scheduler_latency(tsk, delta >> 10, 0); } #endif } diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 3afbd25f43e..5418ef61e16 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -81,6 +81,7 @@ extern int compat_log; extern int maps_protect; extern int sysctl_stat_interval; extern int audit_argv_kb; +extern int latencytop_enabled; /* Constants used for minimum and maximum */ #ifdef CONFIG_DETECT_SOFTLOCKUP @@ -416,6 +417,15 @@ static struct ctl_table kern_table[] = { .proc_handler = &proc_dointvec_taint, }, #endif +#ifdef CONFIG_LATENCYTOP + { + .procname = "latencytop", + .data = &latencytop_enabled, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec, + }, +#endif #ifdef CONFIG_SECURITY_CAPABILITIES { .procname = "cap-bound", -- cgit v1.2.3 From 90739081ef8d5495d50abba9c5d333be9acd872a Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:34 +0100 Subject: softlockup: fix signedness fix softlockup tunables signedness. mark tunables read-mostly. Signed-off-by: Ingo Molnar --- kernel/softlockup.c | 10 +++++----- kernel/sysctl.c | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/softlockup.c b/kernel/softlockup.c index 02f0ad53444..c1d76552446 100644 --- a/kernel/softlockup.c +++ b/kernel/softlockup.c @@ -24,8 +24,8 @@ static DEFINE_PER_CPU(unsigned long, touch_timestamp); static DEFINE_PER_CPU(unsigned long, print_timestamp); static DEFINE_PER_CPU(struct task_struct *, watchdog_task); -static int did_panic; -int softlockup_thresh = 60; +static int __read_mostly did_panic; +unsigned long __read_mostly softlockup_thresh = 60; static int softlock_panic(struct notifier_block *this, unsigned long event, void *ptr) @@ -121,14 +121,14 @@ void softlockup_tick(void) /* * Have a reasonable limit on the number of tasks checked: */ -unsigned long sysctl_hung_task_check_count = 1024; +unsigned long __read_mostly sysctl_hung_task_check_count = 1024; /* * Zero means infinite timeout - no checking done: */ -unsigned long sysctl_hung_task_timeout_secs = 120; +unsigned long __read_mostly sysctl_hung_task_timeout_secs = 120; -long sysctl_hung_task_warnings = 10; +unsigned long __read_mostly sysctl_hung_task_warnings = 10; /* * Only do the hung-tasks check on one CPU: diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 5418ef61e16..8e96558cb8f 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -772,9 +772,9 @@ static struct ctl_table kern_table[] = { .ctl_name = CTL_UNNUMBERED, .procname = "softlockup_thresh", .data = &softlockup_thresh, - .maxlen = sizeof(int), + .maxlen = sizeof(unsigned long), .mode = 0644, - .proc_handler = &proc_dointvec_minmax, + .proc_handler = &proc_doulongvec_minmax, .strategy = &sysctl_intvec, .extra1 = &one, .extra2 = &sixty, @@ -783,27 +783,27 @@ static struct ctl_table kern_table[] = { .ctl_name = CTL_UNNUMBERED, .procname = "hung_task_check_count", .data = &sysctl_hung_task_check_count, - .maxlen = sizeof(int), + .maxlen = sizeof(unsigned long), .mode = 0644, - .proc_handler = &proc_dointvec_minmax, + .proc_handler = &proc_doulongvec_minmax, .strategy = &sysctl_intvec, }, { .ctl_name = CTL_UNNUMBERED, .procname = "hung_task_timeout_secs", .data = &sysctl_hung_task_timeout_secs, - .maxlen = sizeof(int), + .maxlen = sizeof(unsigned long), .mode = 0644, - .proc_handler = &proc_dointvec_minmax, + .proc_handler = &proc_doulongvec_minmax, .strategy = &sysctl_intvec, }, { .ctl_name = CTL_UNNUMBERED, .procname = "hung_task_warnings", .data = &sysctl_hung_task_warnings, - .maxlen = sizeof(int), + .maxlen = sizeof(unsigned long), .mode = 0644, - .proc_handler = &proc_dointvec_minmax, + .proc_handler = &proc_doulongvec_minmax, .strategy = &sysctl_intvec, }, #endif -- cgit v1.2.3 From 19ef9309273d26cb005cb23e6a370353dca91099 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 25 Jan 2008 21:08:34 +0100 Subject: printk: use ktime_get() printk timestamps: use ktime_get(). Some platforms have a functioning clocksource function only after they are done with early bootup, so delay this until out of SYSTEM_BOOTING state. it's also inherently safe now, as any bugs in this area will be caught by the printk recursion checks. Signed-off-by: Ingo Molnar --- kernel/printk.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/printk.c b/kernel/printk.c index 3b7c968d0ef..423a8c765a5 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -702,7 +702,9 @@ asmlinkage int vprintk(const char *fmt, va_list args) loglev_char = default_message_loglevel + '0'; } - t = cpu_clock(printk_cpu); + t = 0; + if (system_state != SYSTEM_BOOTING) + t = ktime_to_ns(ktime_get()); nanosec_rem = do_div(t, 1000000000); tlen = sprintf(tbuf, "<%c>[%5lu.%06lu] ", -- cgit v1.2.3 From 5fb5e6de55860a99c2d8fe7e0c8222d5c53d8464 Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Fri, 25 Jan 2008 21:08:34 +0100 Subject: sched: print backtrace of running tasks too The attached patch is something really simple that can sometimes help in getting more info out of a hung system. Signed-off-by: Ingo Molnar --- kernel/sched.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 4d3a5a70086..524285e46fa 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -5161,8 +5161,7 @@ void sched_show_task(struct task_struct *p) printk(KERN_CONT "%5lu %5d %6d\n", free, task_pid_nr(p), task_pid_nr(p->real_parent)); - if (state != TASK_RUNNING) - show_stack(p, NULL); + show_stack(p, NULL); } void show_state_filter(unsigned long state_filter) -- cgit v1.2.3 From 5973e5b954848c63855a357ad4ff39882e3904f7 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 25 Jan 2008 21:08:34 +0100 Subject: sched: fix: don't take a mutex from interrupt context print_cfs_stats is callable from interrupt context (sysrq), hence it should not take mutexes. Change it to use RCU since the task group data is RCU freed anyway. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 1b3b40ad7c5..45ff4e9411e 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -1434,9 +1434,9 @@ static void print_cfs_stats(struct seq_file *m, int cpu) #ifdef CONFIG_FAIR_GROUP_SCHED print_cfs_rq(m, cpu, &cpu_rq(cpu)->cfs); #endif - lock_task_group_list(); + rcu_read_lock(); for_each_leaf_cfs_rq(cpu_rq(cpu), cfs_rq) print_cfs_rq(m, cpu, cfs_rq); - unlock_task_group_list(); + rcu_read_unlock(); } #endif -- cgit v1.2.3 From 6d082592b62689fb91578d0338d04a9f50991990 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Fri, 25 Jan 2008 21:08:35 +0100 Subject: sched: keep total / count stats in addition to the max for Right now, the linux kernel (with scheduler statistics enabled) keeps track of the maximum time a process is waiting to be scheduled. While the maximum is a very useful metric, tracking average and total is equally useful (at least for latencytop) to figure out the accumulated effect of scheduler delays. The accumulated effect is important to judge the performance impact of scheduler tuning/behavior. Signed-off-by: Arjan van de Ven Signed-off-by: Ingo Molnar --- kernel/sched_debug.c | 4 ++++ kernel/sched_fair.c | 3 +++ 2 files changed, 7 insertions(+) (limited to 'kernel') diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index 9e5de098d47..4b5e24cf2f4 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c @@ -300,6 +300,8 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) PN(se.exec_max); PN(se.slice_max); PN(se.wait_max); + PN(se.wait_sum); + P(se.wait_count); P(sched_info.bkl_count); P(se.nr_migrations); P(se.nr_migrations_cold); @@ -367,6 +369,8 @@ void proc_sched_set_task(struct task_struct *p) { #ifdef CONFIG_SCHEDSTATS p->se.wait_max = 0; + p->se.wait_sum = 0; + p->se.wait_count = 0; p->se.sleep_max = 0; p->se.sum_sleep_runtime = 0; p->se.block_max = 0; diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 45ff4e9411e..72e25c7a3a1 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -385,6 +385,9 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) { schedstat_set(se->wait_max, max(se->wait_max, rq_of(cfs_rq)->clock - se->wait_start)); + schedstat_set(se->wait_count, se->wait_count + 1); + schedstat_set(se->wait_sum, se->wait_sum + + rq_of(cfs_rq)->clock - se->wait_start); schedstat_set(se->wait_start, 0); } -- cgit v1.2.3 From 81ef16e763bb899053e06f6050603a305456a085 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 26 Jan 2008 14:11:06 +0100 Subject: [S390] Remove appldata include from sysctl_check.c Forgot to remove this when removing the appldata binary sysctls. Signed-off-by: Heiko Carstens Signed-off-by: Martin Schwidefsky --- kernel/sysctl_check.c | 1 - 1 file changed, 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index a68425a5cc1..d8a5558a47b 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c @@ -1,6 +1,5 @@ #include #include -#include "../arch/s390/appldata/appldata.h" #include "../fs/xfs/linux-2.6/xfs_sysctl.h" #include #include -- cgit v1.2.3 From 326e96b92306b7af24a3608ec01156cba17a3fc1 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sun, 27 Jan 2008 08:03:54 +0100 Subject: printk: revert ktime_get() timestamps revert 19ef9309273d26cb005cb23e6a370353dca91099. Kevin Winchester reported a lockup during X startup an bisected it to this commit. Reported-by: Kevin Winchester Signed-off-by: Ingo Molnar --- kernel/printk.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/printk.c b/kernel/printk.c index 423a8c765a5..3b7c968d0ef 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -702,9 +702,7 @@ asmlinkage int vprintk(const char *fmt, va_list args) loglev_char = default_message_loglevel + '0'; } - t = 0; - if (system_state != SYSTEM_BOOTING) - t = ktime_to_ns(ktime_get()); + t = cpu_clock(printk_cpu); nanosec_rem = do_div(t, 1000000000); tlen = sprintf(tbuf, "<%c>[%5lu.%06lu] ", -- cgit v1.2.3 From fd0928df98b9578be8a786ac0cb78a47a5e17a20 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 24 Jan 2008 08:52:45 +0100 Subject: ioprio: move io priority from task_struct to io_context This is where it belongs and then it doesn't take up space for a process that doesn't do IO. Signed-off-by: Jens Axboe --- kernel/fork.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 39d22b3357d..2a86c9dff74 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include @@ -791,6 +792,26 @@ out: return error; } +static int copy_io(struct task_struct *tsk) +{ +#ifdef CONFIG_BLOCK + struct io_context *ioc = current->io_context; + + if (!ioc) + return 0; + + if (ioprio_valid(ioc->ioprio)) { + tsk->io_context = alloc_io_context(GFP_KERNEL, -1); + if (unlikely(!tsk->io_context)) + return -ENOMEM; + + tsk->io_context->task = tsk; + tsk->io_context->ioprio = ioc->ioprio; + } +#endif + return 0; +} + /* * Helper to unshare the files of the current task. * We don't want to expose copy_files internals to @@ -1156,15 +1177,17 @@ static struct task_struct *copy_process(unsigned long clone_flags, goto bad_fork_cleanup_mm; if ((retval = copy_namespaces(clone_flags, p))) goto bad_fork_cleanup_keys; + if ((retval = copy_io(p))) + goto bad_fork_cleanup_namespaces; retval = copy_thread(0, clone_flags, stack_start, stack_size, p, regs); if (retval) - goto bad_fork_cleanup_namespaces; + goto bad_fork_cleanup_io; if (pid != &init_struct_pid) { retval = -ENOMEM; pid = alloc_pid(task_active_pid_ns(p)); if (!pid) - goto bad_fork_cleanup_namespaces; + goto bad_fork_cleanup_io; if (clone_flags & CLONE_NEWPID) { retval = pid_ns_prepare_proc(task_active_pid_ns(p)); @@ -1234,9 +1257,6 @@ static struct task_struct *copy_process(unsigned long clone_flags, /* Need tasklist lock for parent etc handling! */ write_lock_irq(&tasklist_lock); - /* for sys_ioprio_set(IOPRIO_WHO_PGRP) */ - p->ioprio = current->ioprio; - /* * The task hasn't been attached yet, so its cpus_allowed mask will * not be changed, nor will its assigned CPU. @@ -1328,6 +1348,8 @@ static struct task_struct *copy_process(unsigned long clone_flags, bad_fork_free_pid: if (pid != &init_struct_pid) free_pid(pid); +bad_fork_cleanup_io: + put_io_context(p->io_context); bad_fork_cleanup_namespaces: exit_task_namespaces(p); bad_fork_cleanup_keys: -- cgit v1.2.3 From d38ecf935fcb10264a6bc190855d9595165e6eeb Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 24 Jan 2008 08:53:35 +0100 Subject: io context sharing: preliminary support Detach task state from ioc, instead keep track of how many processes are accessing the ioc. Signed-off-by: Jens Axboe --- kernel/fork.c | 1 - 1 file changed, 1 deletion(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 2a86c9dff74..1987c57abb0 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -805,7 +805,6 @@ static int copy_io(struct task_struct *tsk) if (unlikely(!tsk->io_context)) return -ENOMEM; - tsk->io_context->task = tsk; tsk->io_context->ioprio = ioc->ioprio; } #endif -- cgit v1.2.3 From fadad878cc0640cc9cd5569998bf54b693f7b38b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 24 Jan 2008 08:54:47 +0100 Subject: kernel: add CLONE_IO to specifically request sharing of IO contexts syslets (or other threads/processes that want io context sharing) can set this to enforce sharing of io context. Signed-off-by: Jens Axboe --- kernel/fork.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 1987c57abb0..314f5101d2b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -792,15 +792,21 @@ out: return error; } -static int copy_io(struct task_struct *tsk) +static int copy_io(unsigned long clone_flags, struct task_struct *tsk) { #ifdef CONFIG_BLOCK struct io_context *ioc = current->io_context; if (!ioc) return 0; - - if (ioprio_valid(ioc->ioprio)) { + /* + * Share io context with parent, if CLONE_IO is set + */ + if (clone_flags & CLONE_IO) { + tsk->io_context = ioc_task_link(ioc); + if (unlikely(!tsk->io_context)) + return -ENOMEM; + } else if (ioprio_valid(ioc->ioprio)) { tsk->io_context = alloc_io_context(GFP_KERNEL, -1); if (unlikely(!tsk->io_context)) return -ENOMEM; @@ -1176,7 +1182,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, goto bad_fork_cleanup_mm; if ((retval = copy_namespaces(clone_flags, p))) goto bad_fork_cleanup_keys; - if ((retval = copy_io(p))) + if ((retval = copy_io(clone_flags, p))) goto bad_fork_cleanup_namespaces; retval = copy_thread(0, clone_flags, stack_start, stack_size, p, regs); if (retval) -- cgit v1.2.3 From 29e796fd4de54b8f5bc30d897611210ece4fd0f2 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Fri, 30 Nov 2007 23:50:18 +1100 Subject: sysctl: Add register_sysctl_paths function There are a number of modules that register a sysctl table somewhere deeply nested in the sysctl hierarchy, such as fs/nfs, fs/xfs, dev/cdrom, etc. They all specify several dummy ctl_tables for the path name. This patch implements register_sysctl_path that takes an additional path name, and makes up dummy sysctl nodes for each component. This patch was originally written by Olaf Kirch and brought to my attention and reworked some by Olaf Hering. I have changed a few additional things so the bugs are mine. After converting all of the easy callers Olaf Hering observed allyesconfig ARCH=i386, the patch reduces the final binary size by 9369 bytes. .text +897 .data -7008 text data bss dec hex filename 26959310 4045899 4718592 35723801 2211a19 ../vmlinux-vanilla 26960207 4038891 4718592 35717690 221023a ../O-allyesconfig/vmlinux So this change is both a space savings and a code simplification. CC: Olaf Kirch CC: Olaf Hering Signed-off-by: Eric W. Biederman Cc: Serge Hallyn Cc: Daniel Lezcano Cc: Cedric Le Goater Cc: Pavel Emelyanov Signed-off-by: Andrew Morton Signed-off-by: Herbert Xu Signed-off-by: David S. Miller --- kernel/sysctl.c | 89 +++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 8e96558cb8f..f580542333e 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1561,11 +1561,12 @@ static __init int sysctl_init(void) core_initcall(sysctl_init); /** - * register_sysctl_table - register a sysctl hierarchy + * register_sysctl_paths - register a sysctl hierarchy + * @path: The path to the directory the sysctl table is in. * @table: the top-level table structure * * Register a sysctl table hierarchy. @table should be a filled in ctl_table - * array. An entry with a ctl_name of 0 terminates the table. + * array. A completely 0 filled entry terminates the table. * * The members of the &struct ctl_table structure are used as follows: * @@ -1628,25 +1629,76 @@ core_initcall(sysctl_init); * This routine returns %NULL on a failure to register, and a pointer * to the table header on success. */ -struct ctl_table_header *register_sysctl_table(struct ctl_table * table) +struct ctl_table_header *register_sysctl_paths(const struct ctl_path *path, + struct ctl_table *table) { - struct ctl_table_header *tmp; - tmp = kmalloc(sizeof(struct ctl_table_header), GFP_KERNEL); - if (!tmp) + struct ctl_table_header *header; + struct ctl_table *new, **prevp; + unsigned int n, npath; + + /* Count the path components */ + for (npath = 0; path[npath].ctl_name || path[npath].procname; ++npath) + ; + + /* + * For each path component, allocate a 2-element ctl_table array. + * The first array element will be filled with the sysctl entry + * for this, the second will be the sentinel (ctl_name == 0). + * + * We allocate everything in one go so that we don't have to + * worry about freeing additional memory in unregister_sysctl_table. + */ + header = kzalloc(sizeof(struct ctl_table_header) + + (2 * npath * sizeof(struct ctl_table)), GFP_KERNEL); + if (!header) return NULL; - tmp->ctl_table = table; - INIT_LIST_HEAD(&tmp->ctl_entry); - tmp->used = 0; - tmp->unregistering = NULL; - sysctl_set_parent(NULL, table); - if (sysctl_check_table(tmp->ctl_table)) { - kfree(tmp); + + new = (struct ctl_table *) (header + 1); + + /* Now connect the dots */ + prevp = &header->ctl_table; + for (n = 0; n < npath; ++n, ++path) { + /* Copy the procname */ + new->procname = path->procname; + new->ctl_name = path->ctl_name; + new->mode = 0555; + + *prevp = new; + prevp = &new->child; + + new += 2; + } + *prevp = table; + + INIT_LIST_HEAD(&header->ctl_entry); + header->used = 0; + header->unregistering = NULL; + sysctl_set_parent(NULL, header->ctl_table); + if (sysctl_check_table(header->ctl_table)) { + kfree(header); return NULL; } spin_lock(&sysctl_lock); - list_add_tail(&tmp->ctl_entry, &root_table_header.ctl_entry); + list_add_tail(&header->ctl_entry, &root_table_header.ctl_entry); spin_unlock(&sysctl_lock); - return tmp; + + return header; +} + +/** + * register_sysctl_table - register a sysctl table hierarchy + * @table: the top-level table structure + * + * Register a sysctl table hierarchy. @table should be a filled in ctl_table + * array. A completely 0 filled entry terminates the table. + * + * See register_sysctl_paths for more details. + */ +struct ctl_table_header *register_sysctl_table(struct ctl_table *table) +{ + static const struct ctl_path null_path[] = { {} }; + + return register_sysctl_paths(null_path, table); } /** @@ -1675,6 +1727,12 @@ struct ctl_table_header *register_sysctl_table(struct ctl_table * table) return NULL; } +struct ctl_table_header *register_sysctl_paths(const struct ctl_path *path, + struct ctl_table *table) +{ + return NULL; +} + void unregister_sysctl_table(struct ctl_table_header * table) { } @@ -2733,6 +2791,7 @@ EXPORT_SYMBOL(proc_dostring); EXPORT_SYMBOL(proc_doulongvec_minmax); EXPORT_SYMBOL(proc_doulongvec_ms_jiffies_minmax); EXPORT_SYMBOL(register_sysctl_table); +EXPORT_SYMBOL(register_sysctl_paths); EXPORT_SYMBOL(sysctl_intvec); EXPORT_SYMBOL(sysctl_jiffies); EXPORT_SYMBOL(sysctl_ms_jiffies); -- cgit v1.2.3 From 23eb06de7d2d333a0f7ebba2da663e00c9c9483e Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Fri, 30 Nov 2007 23:52:10 +1100 Subject: sysctl: Remember the ctl_table we passed to register_sysctl_paths By doing this we allow users of register_sysctl_paths that build and dynamically allocate their ctl_table to be simpler. This allows them to just remember the ctl_table_header returned from register_sysctl_paths from which they can now find the ctl_table array they need to free. Signed-off-by: Eric W. Biederman Cc: Serge Hallyn Cc: Daniel Lezcano Cc: Cedric Le Goater Cc: Pavel Emelyanov Signed-off-by: Andrew Morton Signed-off-by: Herbert Xu Signed-off-by: David S. Miller --- kernel/sysctl.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/sysctl.c b/kernel/sysctl.c index f580542333e..89b7d95ecf5 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1669,6 +1669,7 @@ struct ctl_table_header *register_sysctl_paths(const struct ctl_path *path, new += 2; } *prevp = table; + header->ctl_table_arg = table; INIT_LIST_HEAD(&header->ctl_entry); header->used = 0; -- cgit v1.2.3 From e51b6ba077791f2f8c876022b37419be7a2ceec3 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Fri, 30 Nov 2007 23:54:00 +1100 Subject: sysctl: Infrastructure for per namespace sysctls This patch implements the basic infrastructure for per namespace sysctls. A list of lists of sysctl headers is added, allowing each namespace to have it's own list of sysctl headers. Each list of sysctl headers has a lookup function to find the first sysctl header in the list, allowing the lists to have a per namespace instance. register_sysct_root is added to tell sysctl.c about additional lists of sysctl_headers. As all of the users are expected to be in kernel no unregister function is provided. sysctl_head_next is updated to walk through the list of lists. __register_sysctl_paths is added to add a new sysctl table on a non-default sysctl list. The only intrusive part of this patch is propagating the information to decided which list of sysctls to use for sysctl_check_table. Signed-off-by: Eric W. Biederman Cc: Serge Hallyn Cc: Daniel Lezcano Cc: Cedric Le Goater Cc: Pavel Emelyanov Signed-off-by: Andrew Morton Signed-off-by: Herbert Xu Signed-off-by: David S. Miller --- kernel/sysctl.c | 93 +++++++++++++++++++++++++++++++++++++++++++++------ kernel/sysctl_check.c | 25 ++++++++------ 2 files changed, 96 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 89b7d95ecf5..45e76f209dc 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -157,8 +157,16 @@ static int proc_dointvec_taint(struct ctl_table *table, int write, struct file * #endif static struct ctl_table root_table[]; -static struct ctl_table_header root_table_header = - { root_table, LIST_HEAD_INIT(root_table_header.ctl_entry) }; +static struct ctl_table_root sysctl_table_root; +static struct ctl_table_header root_table_header = { + .ctl_table = root_table, + .ctl_entry = LIST_HEAD_INIT(sysctl_table_root.header_list), + .root = &sysctl_table_root, +}; +static struct ctl_table_root sysctl_table_root = { + .root_list = LIST_HEAD_INIT(sysctl_table_root.root_list), + .header_list = LIST_HEAD_INIT(root_table_header.ctl_entry), +}; static struct ctl_table kern_table[]; static struct ctl_table vm_table[]; @@ -1371,12 +1379,27 @@ void sysctl_head_finish(struct ctl_table_header *head) spin_unlock(&sysctl_lock); } -struct ctl_table_header *sysctl_head_next(struct ctl_table_header *prev) +static struct list_head * +lookup_header_list(struct ctl_table_root *root, struct nsproxy *namespaces) { + struct list_head *header_list; + header_list = &root->header_list; + if (root->lookup) + header_list = root->lookup(root, namespaces); + return header_list; +} + +struct ctl_table_header *__sysctl_head_next(struct nsproxy *namespaces, + struct ctl_table_header *prev) +{ + struct ctl_table_root *root; + struct list_head *header_list; struct ctl_table_header *head; struct list_head *tmp; + spin_lock(&sysctl_lock); if (prev) { + head = prev; tmp = &prev->ctl_entry; unuse_table(prev); goto next; @@ -1390,14 +1413,38 @@ struct ctl_table_header *sysctl_head_next(struct ctl_table_header *prev) spin_unlock(&sysctl_lock); return head; next: + root = head->root; tmp = tmp->next; - if (tmp == &root_table_header.ctl_entry) - break; + header_list = lookup_header_list(root, namespaces); + if (tmp != header_list) + continue; + + do { + root = list_entry(root->root_list.next, + struct ctl_table_root, root_list); + if (root == &sysctl_table_root) + goto out; + header_list = lookup_header_list(root, namespaces); + } while (list_empty(header_list)); + tmp = header_list->next; } +out: spin_unlock(&sysctl_lock); return NULL; } +struct ctl_table_header *sysctl_head_next(struct ctl_table_header *prev) +{ + return __sysctl_head_next(current->nsproxy, prev); +} + +void register_sysctl_root(struct ctl_table_root *root) +{ + spin_lock(&sysctl_lock); + list_add_tail(&root->root_list, &sysctl_table_root.root_list); + spin_unlock(&sysctl_lock); +} + #ifdef CONFIG_SYSCTL_SYSCALL int do_sysctl(int __user *name, int nlen, void __user *oldval, size_t __user *oldlenp, void __user *newval, size_t newlen) @@ -1554,14 +1601,16 @@ static __init int sysctl_init(void) { int err; sysctl_set_parent(NULL, root_table); - err = sysctl_check_table(root_table); + err = sysctl_check_table(current->nsproxy, root_table); return 0; } core_initcall(sysctl_init); /** - * register_sysctl_paths - register a sysctl hierarchy + * __register_sysctl_paths - register a sysctl hierarchy + * @root: List of sysctl headers to register on + * @namespaces: Data to compute which lists of sysctl entries are visible * @path: The path to the directory the sysctl table is in. * @table: the top-level table structure * @@ -1629,9 +1678,12 @@ core_initcall(sysctl_init); * This routine returns %NULL on a failure to register, and a pointer * to the table header on success. */ -struct ctl_table_header *register_sysctl_paths(const struct ctl_path *path, - struct ctl_table *table) +struct ctl_table_header *__register_sysctl_paths( + struct ctl_table_root *root, + struct nsproxy *namespaces, + const struct ctl_path *path, struct ctl_table *table) { + struct list_head *header_list; struct ctl_table_header *header; struct ctl_table *new, **prevp; unsigned int n, npath; @@ -1674,18 +1726,37 @@ struct ctl_table_header *register_sysctl_paths(const struct ctl_path *path, INIT_LIST_HEAD(&header->ctl_entry); header->used = 0; header->unregistering = NULL; + header->root = root; sysctl_set_parent(NULL, header->ctl_table); - if (sysctl_check_table(header->ctl_table)) { + if (sysctl_check_table(namespaces, header->ctl_table)) { kfree(header); return NULL; } spin_lock(&sysctl_lock); - list_add_tail(&header->ctl_entry, &root_table_header.ctl_entry); + header_list = lookup_header_list(root, namespaces); + list_add_tail(&header->ctl_entry, header_list); spin_unlock(&sysctl_lock); return header; } +/** + * register_sysctl_table_path - register a sysctl table hierarchy + * @path: The path to the directory the sysctl table is in. + * @table: the top-level table structure + * + * Register a sysctl table hierarchy. @table should be a filled in ctl_table + * array. A completely 0 filled entry terminates the table. + * + * See __register_sysctl_paths for more details. + */ +struct ctl_table_header *register_sysctl_paths(const struct ctl_path *path, + struct ctl_table *table) +{ + return __register_sysctl_paths(&sysctl_table_root, current->nsproxy, + path, table); +} + /** * register_sysctl_table - register a sysctl table hierarchy * @table: the top-level table structure diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index d8a5558a47b..c3206fa5004 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c @@ -1342,7 +1342,8 @@ static void sysctl_repair_table(struct ctl_table *table) } } -static struct ctl_table *sysctl_check_lookup(struct ctl_table *table) +static struct ctl_table *sysctl_check_lookup(struct nsproxy *namespaces, + struct ctl_table *table) { struct ctl_table_header *head; struct ctl_table *ref, *test; @@ -1350,8 +1351,8 @@ static struct ctl_table *sysctl_check_lookup(struct ctl_table *table) depth = sysctl_depth(table); - for (head = sysctl_head_next(NULL); head; - head = sysctl_head_next(head)) { + for (head = __sysctl_head_next(namespaces, NULL); head; + head = __sysctl_head_next(namespaces, head)) { cur_depth = depth; ref = head->ctl_table; repeat: @@ -1396,13 +1397,14 @@ static void set_fail(const char **fail, struct ctl_table *table, const char *str *fail = str; } -static int sysctl_check_dir(struct ctl_table *table) +static int sysctl_check_dir(struct nsproxy *namespaces, + struct ctl_table *table) { struct ctl_table *ref; int error; error = 0; - ref = sysctl_check_lookup(table); + ref = sysctl_check_lookup(namespaces, table); if (ref) { int match = 0; if ((!table->procname && !ref->procname) || @@ -1427,11 +1429,12 @@ static int sysctl_check_dir(struct ctl_table *table) return error; } -static void sysctl_check_leaf(struct ctl_table *table, const char **fail) +static void sysctl_check_leaf(struct nsproxy *namespaces, + struct ctl_table *table, const char **fail) { struct ctl_table *ref; - ref = sysctl_check_lookup(table); + ref = sysctl_check_lookup(namespaces, table); if (ref && (ref != table)) set_fail(fail, table, "Sysctl already exists"); } @@ -1455,7 +1458,7 @@ static void sysctl_check_bin_path(struct ctl_table *table, const char **fail) } } -int sysctl_check_table(struct ctl_table *table) +int sysctl_check_table(struct nsproxy *namespaces, struct ctl_table *table) { int error = 0; for (; table->ctl_name || table->procname; table++) { @@ -1485,7 +1488,7 @@ int sysctl_check_table(struct ctl_table *table) set_fail(&fail, table, "Directory with extra1"); if (table->extra2) set_fail(&fail, table, "Directory with extra2"); - if (sysctl_check_dir(table)) + if (sysctl_check_dir(namespaces, table)) set_fail(&fail, table, "Inconsistent directory names"); } else { if ((table->strategy == sysctl_data) || @@ -1534,7 +1537,7 @@ int sysctl_check_table(struct ctl_table *table) if (!table->procname && table->proc_handler) set_fail(&fail, table, "proc_handler without procname"); #endif - sysctl_check_leaf(table, &fail); + sysctl_check_leaf(namespaces, table, &fail); } sysctl_check_bin_path(table, &fail); if (fail) { @@ -1542,7 +1545,7 @@ int sysctl_check_table(struct ctl_table *table) error = -EINVAL; } if (table->child) - error |= sysctl_check_table(table->child); + error |= sysctl_check_table(namespaces, table->child); } return error; } -- cgit v1.2.3 From 08913681e484f3f0db949dd0809012e089846216 Mon Sep 17 00:00:00 2001 From: Pavel Emelyanov Date: Wed, 5 Dec 2007 01:42:49 -0800 Subject: [NET]: Remove the empty net_table I have removed all the entries from this table (core_table, ipv4_table and tr_table), so now we can safely drop it. Signed-off-by: Pavel Emelyanov Signed-off-by: David S. Miller --- kernel/sysctl.c | 8 -------- 1 file changed, 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 45e76f209dc..4bc8e48434a 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -200,14 +200,6 @@ static struct ctl_table root_table[] = { .mode = 0555, .child = vm_table, }, -#ifdef CONFIG_NET - { - .ctl_name = CTL_NET, - .procname = "net", - .mode = 0555, - .child = net_table, - }, -#endif { .ctl_name = CTL_FS, .procname = "fs", -- cgit v1.2.3 From a2da4052f1df6bc77749f84496fe731ab8b458f7 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 29 Jan 2008 17:13:17 -0500 Subject: module: Don't report discarded init pages as kernel text. Current code could cause a bug in symbol_put_addr() if an arch used kmalloc module text: we might think the symbol belongs to the core kernel. The downside is that this might make backtraces through (discarded) init functions harder to read on some archs, but we already have that issue for modules and noone has complained. Signed-off-by: Rusty Russell --- kernel/extable.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/extable.c b/kernel/extable.c index 7fe26285531..a26cb2e1702 100644 --- a/kernel/extable.c +++ b/kernel/extable.c @@ -46,7 +46,8 @@ int core_kernel_text(unsigned long addr) addr <= (unsigned long)_etext) return 1; - if (addr >= (unsigned long)_sinittext && + if (system_state == SYSTEM_BOOTING && + addr >= (unsigned long)_sinittext && addr <= (unsigned long)_einittext) return 1; return 0; -- cgit v1.2.3 From c9a3ba55bb5da03fc7d707709a7fe078fe1aa0a0 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 29 Jan 2008 17:13:18 -0500 Subject: module: wait for dependent modules doing init. There have been reports of modules failing to load because the modules they depend on are still loading. This changes the modules to wait for a reasonable length of time in that case. We time out eventually, because there can be module loops or broken modules. Signed-off-by: Rusty Russell --- kernel/module.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index 1bb4c5e0d56..17314691d3c 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -65,6 +65,9 @@ static DEFINE_MUTEX(module_mutex); static LIST_HEAD(modules); +/* Waiting for a module to finish initializing? */ +static DECLARE_WAIT_QUEUE_HEAD(module_wq); + static BLOCKING_NOTIFIER_HEAD(module_notify_list); int register_module_notifier(struct notifier_block * nb) @@ -84,8 +87,11 @@ EXPORT_SYMBOL(unregister_module_notifier); static inline int strong_try_module_get(struct module *mod) { if (mod && mod->state == MODULE_STATE_COMING) + return -EBUSY; + if (try_module_get(mod)) return 0; - return try_module_get(mod); + else + return -ENOENT; } static inline void add_taint_module(struct module *mod, unsigned flag) @@ -539,11 +545,21 @@ static int already_uses(struct module *a, struct module *b) static int use_module(struct module *a, struct module *b) { struct module_use *use; - int no_warn; + int no_warn, err; if (b == NULL || already_uses(a, b)) return 1; - if (!strong_try_module_get(b)) + /* If we're interrupted or time out, we fail. */ + if (wait_event_interruptible_timeout( + module_wq, (err = strong_try_module_get(b)) != -EBUSY, + 30 * HZ) <= 0) { + printk("%s: gave up waiting for init of module %s.\n", + a->name, b->name); + return 0; + } + + /* If strong_try_module_get() returned a different error, we fail. */ + if (err) return 0; DEBUGP("Allocating new usage for %s.\n", a->name); @@ -816,7 +832,7 @@ static inline void module_unload_free(struct module *mod) static inline int use_module(struct module *a, struct module *b) { - return strong_try_module_get(b); + return strong_try_module_get(b) == 0; } static inline void module_unload_init(struct module *mod) @@ -1326,7 +1342,7 @@ void *__symbol_get(const char *symbol) preempt_disable(); value = __find_symbol(symbol, &owner, &crc, 1); - if (value && !strong_try_module_get(owner)) + if (value && strong_try_module_get(owner) != 0) value = 0; preempt_enable(); @@ -2132,6 +2148,7 @@ sys_init_module(void __user *umod, mutex_lock(&module_mutex); free_module(mod); mutex_unlock(&module_mutex); + wake_up(&module_wq); return ret; } @@ -2146,6 +2163,7 @@ sys_init_module(void __user *umod, mod->init_size = 0; mod->init_text_size = 0; mutex_unlock(&module_mutex); + wake_up(&module_wq); return 0; } -- cgit v1.2.3 From efa5345e39d01deef349c120f55ac6b6eabe7457 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 29 Jan 2008 17:13:20 -0500 Subject: module: Fix gratuitous sprintf in module.c Andrew sent an older version of this patch: we shouldn't use sprintf to copy a string. Signed-off-by: Rusty Russell --- kernel/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index 17314691d3c..276abd7b7ff 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -738,7 +738,7 @@ sys_delete_module(const char __user *name_user, unsigned int flags) mutex_lock(&module_mutex); } /* Store the name of the last unloaded module for diagnostic purposes */ - sprintf(last_unloaded_module, mod->name); + strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module)); free_module(mod); out: -- cgit v1.2.3 From bb9d3d56e792d2619cc0903df4ac01d86ac1261d Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 29 Jan 2008 17:13:21 -0500 Subject: module: better OOPS and lockdep coverage for loading modules If we put the module in the linked list *before* calling into to, we get the module name and functions in the OOPS (is_module_address can find the module). It also helps lockdep in a similar way. Acked-and-tested-by: Joern Engel Tested-by: Erez Zadok Signed-off-by: Rusty Russell --- kernel/module.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index 276abd7b7ff..12067ff34d0 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1293,6 +1293,17 @@ static void mod_kobject_remove(struct module *mod) kobject_put(&mod->mkobj.kobj); } +/* + * link the module with the whole machine is stopped with interrupts off + * - this defends against kallsyms not taking locks + */ +static int __link_module(void *_mod) +{ + struct module *mod = _mod; + list_add(&mod->list, &modules); + return 0; +} + /* * unlink the module with the whole machine is stopped with interrupts off * - this defends against kallsyms not taking locks @@ -2035,6 +2046,11 @@ static struct module *load_module(void __user *umod, printk(KERN_WARNING "%s: Ignoring obsolete parameters\n", mod->name); + /* Now sew it into the lists so we can get lockdep and oops + * info during argument parsing. Noone should access us, since + * strong_try_module_get() will fail. */ + stop_machine_run(__link_module, mod, NR_CPUS); + /* Size of section 0 is 0, so this works well if no params */ err = parse_args(mod->name, mod->args, (struct kernel_param *) @@ -2043,7 +2059,7 @@ static struct module *load_module(void __user *umod, / sizeof(struct kernel_param), NULL); if (err < 0) - goto arch_cleanup; + goto unlink; err = mod_sysfs_setup(mod, (struct kernel_param *) @@ -2051,7 +2067,7 @@ static struct module *load_module(void __user *umod, sechdrs[setupindex].sh_size / sizeof(struct kernel_param)); if (err < 0) - goto arch_cleanup; + goto unlink; add_sect_attrs(mod, hdr->e_shnum, secstrings, sechdrs); add_notes_attrs(mod, hdr->e_shnum, secstrings, sechdrs); @@ -2066,7 +2082,8 @@ static struct module *load_module(void __user *umod, /* Done! */ return mod; - arch_cleanup: + unlink: + stop_machine_run(__unlink_module, mod, NR_CPUS); module_arch_cleanup(mod); cleanup: kobject_del(&mod->mkobj.kobj); @@ -2091,17 +2108,6 @@ static struct module *load_module(void __user *umod, goto free_hdr; } -/* - * link the module with the whole machine is stopped with interrupts off - * - this defends against kallsyms not taking locks - */ -static int __link_module(void *_mod) -{ - struct module *mod = _mod; - list_add(&mod->list, &modules); - return 0; -} - /* This is where the real work happens */ asmlinkage long sys_init_module(void __user *umod, @@ -2126,10 +2132,6 @@ sys_init_module(void __user *umod, return PTR_ERR(mod); } - /* Now sew it into the lists. They won't access us, since - strong_try_module_get() will fail. */ - stop_machine_run(__link_module, mod, NR_CPUS); - /* Drop lock so they can recurse */ mutex_unlock(&module_mutex); -- cgit v1.2.3 From 6dd06c9fbe025f542bce4cdb91790c0f91962722 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 29 Jan 2008 17:13:22 -0500 Subject: module: make module_address_lookup safe module_address_lookup releases preemption then returns a pointer into the module space. The only user (kallsyms) copies the result, so just do that under the preempt disable. Signed-off-by: Rusty Russell --- kernel/kallsyms.c | 11 ++++------- kernel/module.c | 22 +++++++++++++--------- 2 files changed, 17 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 2fc25810509..7dadc71ce51 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -233,10 +233,11 @@ static unsigned long get_symbol_pos(unsigned long addr, int kallsyms_lookup_size_offset(unsigned long addr, unsigned long *symbolsize, unsigned long *offset) { + char namebuf[KSYM_NAME_LEN]; if (is_ksym_addr(addr)) return !!get_symbol_pos(addr, symbolsize, offset); - return !!module_address_lookup(addr, symbolsize, offset, NULL); + return !!module_address_lookup(addr, symbolsize, offset, NULL, namebuf); } /* @@ -251,8 +252,6 @@ const char *kallsyms_lookup(unsigned long addr, unsigned long *offset, char **modname, char *namebuf) { - const char *msym; - namebuf[KSYM_NAME_LEN - 1] = 0; namebuf[0] = 0; @@ -268,10 +267,8 @@ const char *kallsyms_lookup(unsigned long addr, } /* see if it's in a module */ - msym = module_address_lookup(addr, symbolsize, offset, modname); - if (msym) - return strncpy(namebuf, msym, KSYM_NAME_LEN - 1); - + return module_address_lookup(addr, symbolsize, offset, modname, + namebuf); return NULL; } diff --git a/kernel/module.c b/kernel/module.c index 12067ff34d0..e814cd7da63 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -2230,14 +2230,13 @@ static const char *get_ksymbol(struct module *mod, return mod->strtab + mod->symtab[best].st_name; } -/* For kallsyms to ask for address resolution. NULL means not found. - We don't lock, as this is used for oops resolution and races are a - lesser concern. */ -/* FIXME: Risky: returns a pointer into a module w/o lock */ -const char *module_address_lookup(unsigned long addr, - unsigned long *size, - unsigned long *offset, - char **modname) +/* For kallsyms to ask for address resolution. NULL means not found. Careful + * not to lock to avoid deadlock on oopses, simply disable preemption. */ +char *module_address_lookup(unsigned long addr, + unsigned long *size, + unsigned long *offset, + char **modname, + char *namebuf) { struct module *mod; const char *ret = NULL; @@ -2252,8 +2251,13 @@ const char *module_address_lookup(unsigned long addr, break; } } + /* Make a copy in here where it's safe */ + if (ret) { + strncpy(namebuf, ret, KSYM_NAME_LEN - 1); + ret = namebuf; + } preempt_enable(); - return ret; + return (char *)ret; } int lookup_module_symbol_name(unsigned long addr, char *symname) -- cgit v1.2.3 From 8686c99875f3849047660a5b6d02438443f22ce7 Mon Sep 17 00:00:00 2001 From: Denis Cheng Date: Mon, 21 Jan 2008 17:08:25 +0800 Subject: module: fix the module name length in param_sysfs_builtin the original code use KOBJ_NAME_LEN for built-in module name length, that's defined to 20 in linux/kobject.h, but this is not enough appearntly, many module names are longer than this; #define KOBJ_NAME_LEN 20 another macro is MODULE_NAME_LEN defined in linux/module.h, I think this is enough for module names: #define MODULE_NAME_LEN (64 - sizeof(unsigned long)) Signed-off-by: Denis Cheng Signed-off-by: Rusty Russell --- kernel/params.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/params.c b/kernel/params.c index 67f65ee7211..42fe5e6126c 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -376,8 +376,6 @@ int param_get_string(char *buffer, struct kernel_param *kp) extern struct kernel_param __start___param[], __stop___param[]; -#define MAX_KBUILD_MODNAME KOBJ_NAME_LEN - struct param_attribute { struct module_attribute mattr; @@ -587,7 +585,7 @@ static void __init param_sysfs_builtin(void) { struct kernel_param *kp, *kp_begin = NULL; unsigned int i, name_len, count = 0; - char modname[MAX_KBUILD_MODNAME + 1] = ""; + char modname[MODULE_NAME_LEN + 1] = ""; for (i=0; i < __stop___param - __start___param; i++) { char *dot; @@ -595,12 +593,12 @@ static void __init param_sysfs_builtin(void) kp = &__start___param[i]; max_name_len = - min_t(size_t, MAX_KBUILD_MODNAME, strlen(kp->name)); + min_t(size_t, MODULE_NAME_LEN, strlen(kp->name)); dot = memchr(kp->name, '.', max_name_len); if (!dot) { DEBUGP("couldn't find period in first %d characters " - "of %s\n", MAX_KBUILD_MODNAME, kp->name); + "of %s\n", MODULE_NAME_LEN, kp->name); continue; } name_len = dot - kp->name; -- cgit v1.2.3 From 0aa5bd52d0c49ca56d24584c646e6544ccbb3dc9 Mon Sep 17 00:00:00 2001 From: Jon Masters Date: Mon, 21 Jan 2008 20:43:41 +0000 Subject: module: add module taint on ndiswrapper The struct module taints member is supposed to store per-module taint data. The kernel knows about certain specific external modules that will taint the kernel, such as ndiswrapper. Use of ndiswrapper possibly should set the per-module taint in addition to the global kernel taint flag, unless we're arguing not because wrapper module itself is not what actually causes the kernel to be tainted as such? Signed-off-by: Jon Masters Signed-off-by: Rusty Russell --- kernel/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index e814cd7da63..af3f81a9474 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1916,7 +1916,7 @@ static struct module *load_module(void __user *umod, set_license(mod, get_modinfo(sechdrs, infoindex, "license")); if (strcmp(mod->name, "ndiswrapper") == 0) - add_taint(TAINT_PROPRIETARY_MODULE); + add_taint_module(mod, TAINT_PROPRIETARY_MODULE); if (strcmp(mod->name, "driverloader") == 0) add_taint_module(mod, TAINT_PROPRIETARY_MODULE); -- cgit v1.2.3 From 6494a93d55fad586238cc1940e846c6d03e1aaf6 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Sun, 27 Jan 2008 15:38:40 -0800 Subject: Module: check to see if we have a built in module with the same name When trying to load a module with the same name as a built-in one, a scary kobject backtrace comes up. Prevent that from checking for this condition and warning the user as to what exactly is going on. Cc: Rusty Russell Cc: Linus Torvalds Cc: Andrew Morton Signed-off-by: Greg Kroah-Hartman Signed-off-by: Rusty Russell --- kernel/module.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index af3f81a9474..f6a4e721fd4 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1230,6 +1230,7 @@ void module_remove_modinfo_attrs(struct module *mod) int mod_sysfs_init(struct module *mod) { int err; + struct kobject *kobj; if (!module_sysfs_initialized) { printk(KERN_ERR "%s: module sysfs not initialized\n", @@ -1237,6 +1238,15 @@ int mod_sysfs_init(struct module *mod) err = -EINVAL; goto out; } + + kobj = kset_find_obj(module_kset, mod->name); + if (kobj) { + printk(KERN_ERR "%s: module is already loaded\n", mod->name); + kobject_put(kobj); + err = -EINVAL; + goto out; + } + mod->mkobj.mod = mod; memset(&mod->mkobj.kobj, 0, sizeof(mod->mkobj.kobj)); -- cgit v1.2.3 From a6fa8e5a6172a5a5bc06ed04f34e50b36c978127 Mon Sep 17 00:00:00 2001 From: Pavel Machek Date: Wed, 30 Jan 2008 13:30:00 +0100 Subject: time: clean hungarian notation from timers Clean up hungarian notation from timer code. Signed-off-by: Pavel Machek Cc: john stultz Signed-off-by: Andrew Morton Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/timer.c | 80 ++++++++++++++++++++++++++++------------------------------ 1 file changed, 39 insertions(+), 41 deletions(-) (limited to 'kernel') diff --git a/kernel/timer.c b/kernel/timer.c index f739dfb539c..aadfbc8367f 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -58,59 +58,57 @@ EXPORT_SYMBOL(jiffies_64); #define TVN_MASK (TVN_SIZE - 1) #define TVR_MASK (TVR_SIZE - 1) -typedef struct tvec_s { +struct tvec { struct list_head vec[TVN_SIZE]; -} tvec_t; +}; -typedef struct tvec_root_s { +struct tvec_root { struct list_head vec[TVR_SIZE]; -} tvec_root_t; +}; -struct tvec_t_base_s { +struct tvec_base { spinlock_t lock; struct timer_list *running_timer; unsigned long timer_jiffies; - tvec_root_t tv1; - tvec_t tv2; - tvec_t tv3; - tvec_t tv4; - tvec_t tv5; + struct tvec_root tv1; + struct tvec tv2; + struct tvec tv3; + struct tvec tv4; + struct tvec tv5; } ____cacheline_aligned; -typedef struct tvec_t_base_s tvec_base_t; - -tvec_base_t boot_tvec_bases; +struct tvec_base boot_tvec_bases; EXPORT_SYMBOL(boot_tvec_bases); -static DEFINE_PER_CPU(tvec_base_t *, tvec_bases) = &boot_tvec_bases; +static DEFINE_PER_CPU(struct tvec_base *, tvec_bases) = &boot_tvec_bases; /* - * Note that all tvec_bases is 2 byte aligned and lower bit of + * Note that all tvec_bases are 2 byte aligned and lower bit of * base in timer_list is guaranteed to be zero. Use the LSB for * the new flag to indicate whether the timer is deferrable */ #define TBASE_DEFERRABLE_FLAG (0x1) /* Functions below help us manage 'deferrable' flag */ -static inline unsigned int tbase_get_deferrable(tvec_base_t *base) +static inline unsigned int tbase_get_deferrable(struct tvec_base *base) { return ((unsigned int)(unsigned long)base & TBASE_DEFERRABLE_FLAG); } -static inline tvec_base_t *tbase_get_base(tvec_base_t *base) +static inline struct tvec_base *tbase_get_base(struct tvec_base *base) { - return ((tvec_base_t *)((unsigned long)base & ~TBASE_DEFERRABLE_FLAG)); + return ((struct tvec_base *)((unsigned long)base & ~TBASE_DEFERRABLE_FLAG)); } static inline void timer_set_deferrable(struct timer_list *timer) { - timer->base = ((tvec_base_t *)((unsigned long)(timer->base) | + timer->base = ((struct tvec_base *)((unsigned long)(timer->base) | TBASE_DEFERRABLE_FLAG)); } static inline void -timer_set_base(struct timer_list *timer, tvec_base_t *new_base) +timer_set_base(struct timer_list *timer, struct tvec_base *new_base) { - timer->base = (tvec_base_t *)((unsigned long)(new_base) | + timer->base = (struct tvec_base *)((unsigned long)(new_base) | tbase_get_deferrable(timer->base)); } @@ -246,7 +244,7 @@ unsigned long round_jiffies_relative(unsigned long j) EXPORT_SYMBOL_GPL(round_jiffies_relative); -static inline void set_running_timer(tvec_base_t *base, +static inline void set_running_timer(struct tvec_base *base, struct timer_list *timer) { #ifdef CONFIG_SMP @@ -254,7 +252,7 @@ static inline void set_running_timer(tvec_base_t *base, #endif } -static void internal_add_timer(tvec_base_t *base, struct timer_list *timer) +static void internal_add_timer(struct tvec_base *base, struct timer_list *timer) { unsigned long expires = timer->expires; unsigned long idx = expires - base->timer_jiffies; @@ -371,14 +369,14 @@ static inline void detach_timer(struct timer_list *timer, * possible to set timer->base = NULL and drop the lock: the timer remains * locked. */ -static tvec_base_t *lock_timer_base(struct timer_list *timer, +static struct tvec_base *lock_timer_base(struct timer_list *timer, unsigned long *flags) __acquires(timer->base->lock) { - tvec_base_t *base; + struct tvec_base *base; for (;;) { - tvec_base_t *prelock_base = timer->base; + struct tvec_base *prelock_base = timer->base; base = tbase_get_base(prelock_base); if (likely(base != NULL)) { spin_lock_irqsave(&base->lock, *flags); @@ -393,7 +391,7 @@ static tvec_base_t *lock_timer_base(struct timer_list *timer, int __mod_timer(struct timer_list *timer, unsigned long expires) { - tvec_base_t *base, *new_base; + struct tvec_base *base, *new_base; unsigned long flags; int ret = 0; @@ -445,7 +443,7 @@ EXPORT_SYMBOL(__mod_timer); */ void add_timer_on(struct timer_list *timer, int cpu) { - tvec_base_t *base = per_cpu(tvec_bases, cpu); + struct tvec_base *base = per_cpu(tvec_bases, cpu); unsigned long flags; timer_stats_timer_set_start_info(timer); @@ -508,7 +506,7 @@ EXPORT_SYMBOL(mod_timer); */ int del_timer(struct timer_list *timer) { - tvec_base_t *base; + struct tvec_base *base; unsigned long flags; int ret = 0; @@ -539,7 +537,7 @@ EXPORT_SYMBOL(del_timer); */ int try_to_del_timer_sync(struct timer_list *timer) { - tvec_base_t *base; + struct tvec_base *base; unsigned long flags; int ret = -1; @@ -591,7 +589,7 @@ int del_timer_sync(struct timer_list *timer) EXPORT_SYMBOL(del_timer_sync); #endif -static int cascade(tvec_base_t *base, tvec_t *tv, int index) +static int cascade(struct tvec_base *base, struct tvec *tv, int index) { /* cascade all the timers from tv up one level */ struct timer_list *timer, *tmp; @@ -620,7 +618,7 @@ static int cascade(tvec_base_t *base, tvec_t *tv, int index) * This function cascades all vectors and executes all expired timer * vectors. */ -static inline void __run_timers(tvec_base_t *base) +static inline void __run_timers(struct tvec_base *base) { struct timer_list *timer; @@ -678,13 +676,13 @@ static inline void __run_timers(tvec_base_t *base) * is used on S/390 to stop all activity when a cpus is idle. * This functions needs to be called disabled. */ -static unsigned long __next_timer_interrupt(tvec_base_t *base) +static unsigned long __next_timer_interrupt(struct tvec_base *base) { unsigned long timer_jiffies = base->timer_jiffies; unsigned long expires = timer_jiffies + NEXT_TIMER_MAX_DELTA; int index, slot, array, found = 0; struct timer_list *nte; - tvec_t *varray[4]; + struct tvec *varray[4]; /* Look for timer events in tv1. */ index = slot = timer_jiffies & TVR_MASK; @@ -716,7 +714,7 @@ cascade: varray[3] = &base->tv5; for (array = 0; array < 4; array++) { - tvec_t *varp = varray[array]; + struct tvec *varp = varray[array]; index = slot = timer_jiffies & TVN_MASK; do { @@ -795,7 +793,7 @@ static unsigned long cmp_next_hrtimer_event(unsigned long now, */ unsigned long get_next_timer_interrupt(unsigned long now) { - tvec_base_t *base = __get_cpu_var(tvec_bases); + struct tvec_base *base = __get_cpu_var(tvec_bases); unsigned long expires; spin_lock(&base->lock); @@ -894,7 +892,7 @@ static inline void calc_load(unsigned long ticks) */ static void run_timer_softirq(struct softirq_action *h) { - tvec_base_t *base = __get_cpu_var(tvec_bases); + struct tvec_base *base = __get_cpu_var(tvec_bases); hrtimer_run_pending(); @@ -1223,7 +1221,7 @@ static struct lock_class_key base_lock_keys[NR_CPUS]; static int __cpuinit init_timers_cpu(int cpu) { int j; - tvec_base_t *base; + struct tvec_base *base; static char __cpuinitdata tvec_base_done[NR_CPUS]; if (!tvec_base_done[cpu]) { @@ -1278,7 +1276,7 @@ static int __cpuinit init_timers_cpu(int cpu) } #ifdef CONFIG_HOTPLUG_CPU -static void migrate_timer_list(tvec_base_t *new_base, struct list_head *head) +static void migrate_timer_list(struct tvec_base *new_base, struct list_head *head) { struct timer_list *timer; @@ -1292,8 +1290,8 @@ static void migrate_timer_list(tvec_base_t *new_base, struct list_head *head) static void __cpuinit migrate_timers(int cpu) { - tvec_base_t *old_base; - tvec_base_t *new_base; + struct tvec_base *old_base; + struct tvec_base *new_base; int i; BUG_ON(cpu_online(cpu)); -- cgit v1.2.3 From 4c9dc6412247abf4972080c51cd16a58c4009c19 Mon Sep 17 00:00:00 2001 From: Pavel Machek Date: Wed, 30 Jan 2008 13:30:00 +0100 Subject: time: timer cleanups Small cleanups to tick-related code. Wrong preempt count is followed by BUG(), so it is hardly KERN_WARNING. Signed-off-by: Pavel Machek Cc: john stultz Signed-off-by: Andrew Morton Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/time/tick-sched.c | 4 ++-- kernel/timer.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 1a21b6fdb67..d36ee2fd1a3 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -296,7 +296,7 @@ void tick_nohz_stop_sched_tick(void) /* Check, if the timer was already in the past */ if (hrtimer_active(&ts->sched_timer)) goto out; - } else if(!tick_program_event(expires, 0)) + } else if (!tick_program_event(expires, 0)) goto out; /* * We are past the event already. So we crossed a @@ -507,7 +507,7 @@ static inline void tick_nohz_switch_to_nohz(void) { } */ #ifdef CONFIG_HIGH_RES_TIMERS /* - * We rearm the timer until we get disabled by the idle code + * We rearm the timer until we get disabled by the idle code. * Called with interrupts disabled and timer->base->cpu_base->lock held. */ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) diff --git a/kernel/timer.c b/kernel/timer.c index aadfbc8367f..23f7ead78fa 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -655,7 +655,7 @@ static inline void __run_timers(struct tvec_base *base) int preempt_count = preempt_count(); fn(data); if (preempt_count != preempt_count()) { - printk(KERN_WARNING "huh, entered %p " + printk(KERN_ERR "huh, entered %p " "with preempt_count %08x, exited" " with %08x?\n", fn, preempt_count, -- cgit v1.2.3 From b10db7f0d2b589a7f88dc3026e150756cb437a28 Mon Sep 17 00:00:00 2001 From: Pavel Machek Date: Wed, 30 Jan 2008 13:30:00 +0100 Subject: time: more timer related cleanups I was confused by FSEC = 10^15 NSEC statement, plus small whitespace fixes. When there's copyright, there should be GPL. Signed-off-by: Pavel Machek Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/softirq.c | 4 +++- kernel/time/tick-sched.c | 2 +- kernel/time/timer_stats.c | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/softirq.c b/kernel/softirq.c index bd89bc4eb0b..8fe1ff40102 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -3,7 +3,9 @@ * * Copyright (C) 1992 Linus Torvalds * - * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903) + * Distribute under GPLv2. + * + * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903) */ #include diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index d36ee2fd1a3..49e12f6a4ba 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -9,7 +9,7 @@ * * Started by: Thomas Gleixner and Ingo Molnar * - * For licencing details see kernel-base/COPYING + * Distribute under GPLv2. */ #include #include diff --git a/kernel/time/timer_stats.c b/kernel/time/timer_stats.c index c36bb7ed030..417da8c5bc7 100644 --- a/kernel/time/timer_stats.c +++ b/kernel/time/timer_stats.c @@ -26,7 +26,7 @@ * the pid and cmdline from the owner process if applicable. * * Start/stop data collection: - * # echo 1[0] >/proc/timer_stats + * # echo [1|0] >/proc/timer_stats * * Display the information collected so far: * # cat /proc/timer_stats -- cgit v1.2.3 From 186e3cb8a465bac010ee3b020768d2fa2b505aef Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 30 Jan 2008 13:30:01 +0100 Subject: timer: clean up tick-broadcast.c clean up tick-broadcast.c Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- kernel/time/tick-broadcast.c | 7 ++----- kernel/time/tick-internal.h | 2 -- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 5b86698faa0..e1bd50cbbf5 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -126,9 +126,9 @@ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) /* * Broadcast the event to the cpus, which are set in the mask */ -int tick_do_broadcast(cpumask_t mask) +static void tick_do_broadcast(cpumask_t mask) { - int ret = 0, cpu = smp_processor_id(); + int cpu = smp_processor_id(); struct tick_device *td; /* @@ -138,7 +138,6 @@ int tick_do_broadcast(cpumask_t mask) cpu_clear(cpu, mask); td = &per_cpu(tick_cpu_device, cpu); td->evtdev->event_handler(td->evtdev); - ret = 1; } if (!cpus_empty(mask)) { @@ -151,9 +150,7 @@ int tick_do_broadcast(cpumask_t mask) cpu = first_cpu(mask); td = &per_cpu(tick_cpu_device, cpu); td->evtdev->broadcast(mask); - ret = 1; } - return ret; } /* diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index bb13f272490..f13f2b7f4fd 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h @@ -70,8 +70,6 @@ static inline int tick_resume_broadcast_oneshot(struct clock_event_device *bc) * Broadcasting support */ #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST -extern int tick_do_broadcast(cpumask_t mask); - extern int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu); extern int tick_check_broadcast_device(struct clock_event_device *dev); extern int tick_is_broadcast_device(struct clock_event_device *dev); -- cgit v1.2.3 From efd9ac8630e89b9ee7ce64008bd7783952374f37 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 30 Jan 2008 13:30:01 +0100 Subject: time: fold __get_realtime_clock_ts() into getnstimeofday() - getnstimeofday() was just a wrapper around __get_realtime_clock_ts() - Replace calls to __get_realtime_clock_ts() by calls to getnstimeofday() - Fix bogus reference to get_realtime_clock_ts(), which never existed Signed-off-by: Geert Uytterhoeven Cc: john stultz Signed-off-by: Andrew Morton Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/time/timekeeping.c | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index ab46ae8c062..77680195cf8 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -82,13 +82,12 @@ static inline s64 __get_nsec_offset(void) } /** - * __get_realtime_clock_ts - Returns the time of day in a timespec + * getnstimeofday - Returns the time of day in a timespec * @ts: pointer to the timespec to be set * - * Returns the time of day in a timespec. Used by - * do_gettimeofday() and get_realtime_clock_ts(). + * Returns the time of day in a timespec. */ -static inline void __get_realtime_clock_ts(struct timespec *ts) +void getnstimeofday(struct timespec *ts) { unsigned long seq; s64 nsecs; @@ -104,30 +103,19 @@ static inline void __get_realtime_clock_ts(struct timespec *ts) timespec_add_ns(ts, nsecs); } -/** - * getnstimeofday - Returns the time of day in a timespec - * @ts: pointer to the timespec to be set - * - * Returns the time of day in a timespec. - */ -void getnstimeofday(struct timespec *ts) -{ - __get_realtime_clock_ts(ts); -} - EXPORT_SYMBOL(getnstimeofday); /** * do_gettimeofday - Returns the time of day in a timeval * @tv: pointer to the timeval to be set * - * NOTE: Users should be converted to using get_realtime_clock_ts() + * NOTE: Users should be converted to using getnstimeofday() */ void do_gettimeofday(struct timeval *tv) { struct timespec now; - __get_realtime_clock_ts(&now); + getnstimeofday(&now); tv->tv_sec = now.tv_sec; tv->tv_usec = now.tv_nsec/1000; } -- cgit v1.2.3 From 1077f5a917b7c630231037826b344b2f7f5b903f Mon Sep 17 00:00:00 2001 From: Parag Warudkar Date: Wed, 30 Jan 2008 13:30:01 +0100 Subject: clocksource.c: use init_timer_deferrable for clocksource_watchdog clocksource_watchdog can use a deferrable timer - reduces wakeups from idle per second. Signed-off-by: Parag Warudkar Cc: john stultz Signed-off-by: Andrew Morton Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/time/clocksource.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 8d6125ad2cf..cabfa193efb 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -175,7 +175,7 @@ static void clocksource_check_watchdog(struct clocksource *cs) if (watchdog) del_timer(&watchdog_timer); watchdog = cs; - init_timer(&watchdog_timer); + init_timer_deferrable(&watchdog_timer); watchdog_timer.function = clocksource_watchdog; /* Reset watchdog cycles */ -- cgit v1.2.3 From 1ada5cba6a0318f90e45b38557e7b5206a9cba38 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Wed, 30 Jan 2008 13:30:02 +0100 Subject: clocksource: make clocksource watchdog cycle through online CPUs This way it checks if the clocks are synchronized between CPUs too. This might be able to detect slowly drifting TSCs which only go wrong over longer time. Signed-off-by: Andi Kleen Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/time/clocksource.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index cabfa193efb..edd5ef8e176 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -142,8 +142,13 @@ static void clocksource_watchdog(unsigned long data) } if (!list_empty(&watchdog_list)) { - __mod_timer(&watchdog_timer, - watchdog_timer.expires + WATCHDOG_INTERVAL); + /* Cycle through CPUs to check if the CPUs stay synchronized to + * each other. */ + int next_cpu = next_cpu(raw_smp_processor_id(), cpu_online_map); + if (next_cpu >= NR_CPUS) + next_cpu = first_cpu(cpu_online_map); + watchdog_timer.expires += WATCHDOG_INTERVAL; + add_timer_on(&watchdog_timer, next_cpu); } spin_unlock(&watchdog_lock); } @@ -165,7 +170,7 @@ static void clocksource_check_watchdog(struct clocksource *cs) if (!started && watchdog) { watchdog_last = watchdog->read(); watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL; - add_timer(&watchdog_timer); + add_timer_on(&watchdog_timer, first_cpu(cpu_online_map)); } } else { if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) @@ -186,7 +191,8 @@ static void clocksource_check_watchdog(struct clocksource *cs) watchdog_last = watchdog->read(); watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL; - add_timer(&watchdog_timer); + add_timer_on(&watchdog_timer, + first_cpu(cpu_online_map)); } } } -- cgit v1.2.3 From 4713e22ce81eb8b3353e16435362eb3d0ec95640 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 30 Jan 2008 13:30:02 +0100 Subject: clocksource: add unregister function to disable unusable clocksources On x86 the PIT might become an unusable clocksource. Add an unregister function to provide a possibilty to remove the PIT from the list of available clock sources. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- kernel/time/clocksource.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'kernel') diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index edd5ef8e176..6e9259a5d50 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -337,6 +337,21 @@ void clocksource_change_rating(struct clocksource *cs, int rating) spin_unlock_irqrestore(&clocksource_lock, flags); } +/** + * clocksource_unregister - remove a registered clocksource + */ +void clocksource_unregister(struct clocksource *cs) +{ + unsigned long flags; + + spin_lock_irqsave(&clocksource_lock, flags); + list_del(&cs->list); + if (clocksource_override == cs) + clocksource_override = NULL; + next_clocksource = select_clocksource(); + spin_unlock_irqrestore(&clocksource_lock, flags); +} + #ifdef CONFIG_SYSFS /** * sysfs_show_current_clocksources - sysfs interface for current clocksource -- cgit v1.2.3 From 45fe4fe19120a22f7339f5bb110447170c25fca9 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 30 Jan 2008 13:30:03 +0100 Subject: x86: make clockevents more robust detect zero event-device multiplicators - they then cause division-by-zero crashes if a clockevent has been initialized incorrectly. Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/time/clockevents.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'kernel') diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 5fb139fef9f..3e59fce6dd4 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -41,6 +41,11 @@ unsigned long clockevent_delta2ns(unsigned long latch, { u64 clc = ((u64) latch << evt->shift); + if (unlikely(!evt->mult)) { + evt->mult = 1; + WARN_ON(1); + } + do_div(clc, evt->mult); if (clc < 1000) clc = 1000; @@ -151,6 +156,14 @@ static void clockevents_notify_released(void) void clockevents_register_device(struct clock_event_device *dev) { BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED); + /* + * A nsec2cyc multiplicator of 0 is invalid and we'd crash + * on it, so fix it up and emit a warning: + */ + if (unlikely(!dev->mult)) { + dev->mult = 1; + WARN_ON(1); + } spin_lock(&clockevents_lock); -- cgit v1.2.3 From bbe4d18ac2e058c56adb0cd71f49d9ed3216a405 Mon Sep 17 00:00:00 2001 From: john stultz Date: Wed, 30 Jan 2008 13:30:03 +0100 Subject: NTP: correct inconsistent ntp interval/tick_length usage I recently noticed on one of my boxes that when synched with an NTP server, the drift value reported for the system was ~283ppm. While in some cases, clock hardware can be that bad, it struck me as unusual as the system was using the acpi_pm clocksource, which is one of the more trustworthy and accurate clocksources on x86 hardware. I brought up another system and let it sync to the same NTP server, and I noticed a similar 280some ppm drift. In looking at the code, I found that the acpi_pm's constant frequency was being computed correctly at boot-up, however once the system was up, even without the ntp daemon running, the clocksource's frequency was being modified by the clocksource_adjust() function. Digging deeper, I realized that in the code that keeps track of how much the clocksource is skewing from the ntp desired time, we were using different lengths to establish how long an time interval was. The clocksource was being setup with the following interval: NTP_INTERVAL_LENGTH = NSEC_PER_SEC/NTP_INTERVAL_FREQ While the ntp code was using the tick_length_base value: tick_length_base ~= (tick_usec * NSEC_PER_USEC * USER_HZ) /NTP_INTERVAL_FREQ The subtle difference is: (tick_usec * NSEC_PER_USEC * USER_HZ) != NSEC_PER_SEC This difference in calculation was causing the clocksource correction code to apply a correction factor to the clocksource so the two intervals were the same, however this results in the actual frequency of the clocksource to be made incorrect. I believe this difference would affect all clocksources, although to differing degrees depending on the clocksource resolution. The issue was introduced when my HZ free ntp patch landed in 2.6.21-rc1, so my apologies for the mistake, and for not noticing it until now. The following patch, corrects the clocksource's initialization code so it uses the same interval length as the code in ntp.c. After applying this patch, the drift value for the same system went from ~283ppm to only 2.635ppm. I believe this patch to be good, however it does affect all arches and I've only tested on x86, so some caution is advised. I do think it would be a likely candidate for a stable 2.6.24.x release. Any thoughts or feedback would be appreciated. Signed-off-by: John Stultz Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/time/timekeeping.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 77680195cf8..092a2366b5a 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -186,7 +186,8 @@ static void change_clocksource(void) clock->error = 0; clock->xtime_nsec = 0; - clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH); + clocksource_calculate_interval(clock, + (unsigned long)(current_tick_length()>>TICK_LENGTH_SHIFT)); tick_clock_notify(); @@ -243,7 +244,8 @@ void __init timekeeping_init(void) ntp_clear(); clock = clocksource_get_next(); - clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH); + clocksource_calculate_interval(clock, + (unsigned long)(current_tick_length()>>TICK_LENGTH_SHIFT)); clock->cycle_last = clocksource_read(clock); xtime.tv_sec = sec; -- cgit v1.2.3 From 6378ddb592158db4b42197f1bc8666228800e379 Mon Sep 17 00:00:00 2001 From: Venki Pallipadi Date: Wed, 30 Jan 2008 13:30:04 +0100 Subject: time: track accurate idle time with tick_sched.idle_sleeptime Current idle time in kstat is based on jiffies and is coarse grained. tick_sched.idle_sleeptime is making some attempt to keep track of idle time in a fine grained manner. But, it is not handling the time spent in interrupts fully. Make tick_sched.idle_sleeptime accurate with respect to time spent on handling interrupts and also add tick_sched.idle_lastupdate, which keeps track of last time when idle_sleeptime was updated. This statistics will be crucial for cpufreq-ondemand governor, which can shed some conservative gaurd band that is uses today while setting the frequency. The ondemand changes that uses the exact idle time is coming soon. Signed-off-by: Venkatesh Pallipadi Signed-off-by: Andrew Morton Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/softirq.c | 7 ++++- kernel/time/tick-sched.c | 70 +++++++++++++++++++++++++++++++++--------------- 2 files changed, 54 insertions(+), 23 deletions(-) (limited to 'kernel') diff --git a/kernel/softirq.c b/kernel/softirq.c index 8fe1ff40102..d7837d45419 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -280,9 +280,14 @@ asmlinkage void do_softirq(void) */ void irq_enter(void) { +#ifdef CONFIG_NO_HZ + int cpu = smp_processor_id(); + if (idle_cpu(cpu) && !in_interrupt()) + tick_nohz_stop_idle(cpu); +#endif __irq_enter(); #ifdef CONFIG_NO_HZ - if (idle_cpu(smp_processor_id())) + if (idle_cpu(cpu)) tick_nohz_update_jiffies(); #endif } diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 49e12f6a4ba..63f24b55069 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -143,6 +143,44 @@ void tick_nohz_update_jiffies(void) local_irq_restore(flags); } +void tick_nohz_stop_idle(int cpu) +{ + struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); + + if (ts->idle_active) { + ktime_t now, delta; + now = ktime_get(); + delta = ktime_sub(now, ts->idle_entrytime); + ts->idle_lastupdate = now; + ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); + ts->idle_active = 0; + } +} + +static ktime_t tick_nohz_start_idle(int cpu) +{ + struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); + ktime_t now, delta; + + now = ktime_get(); + if (ts->idle_active) { + delta = ktime_sub(now, ts->idle_entrytime); + ts->idle_lastupdate = now; + ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); + } + ts->idle_entrytime = now; + ts->idle_active = 1; + return now; +} + +u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) +{ + struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); + + *last_update_time = ktime_to_us(ts->idle_lastupdate); + return ktime_to_us(ts->idle_sleeptime); +} + /** * tick_nohz_stop_sched_tick - stop the idle tick from the idle task * @@ -155,13 +193,14 @@ void tick_nohz_stop_sched_tick(void) unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags; unsigned long rt_jiffies; struct tick_sched *ts; - ktime_t last_update, expires, now, delta; + ktime_t last_update, expires, now; struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; int cpu; local_irq_save(flags); cpu = smp_processor_id(); + now = tick_nohz_start_idle(cpu); ts = &per_cpu(tick_cpu_sched, cpu); /* @@ -193,19 +232,7 @@ void tick_nohz_stop_sched_tick(void) } } - now = ktime_get(); - /* - * When called from irq_exit we need to account the idle sleep time - * correctly. - */ - if (ts->tick_stopped) { - delta = ktime_sub(now, ts->idle_entrytime); - ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); - } - - ts->idle_entrytime = now; ts->idle_calls++; - /* Read jiffies and the time when jiffies were updated last */ do { seq = read_seqbegin(&xtime_lock); @@ -337,23 +364,22 @@ void tick_nohz_restart_sched_tick(void) int cpu = smp_processor_id(); struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); unsigned long ticks; - ktime_t now, delta; + ktime_t now; - if (!ts->tick_stopped) + local_irq_disable(); + tick_nohz_stop_idle(cpu); + + if (!ts->tick_stopped) { + local_irq_enable(); return; + } /* Update jiffies first */ - now = ktime_get(); - - local_irq_disable(); select_nohz_load_balancer(0); + now = ktime_get(); tick_do_update_jiffies64(now); cpu_clear(cpu, nohz_cpu_mask); - /* Account the idle time */ - delta = ktime_sub(now, ts->idle_entrytime); - ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); - /* * We stopped the tick in idle. Update process times would miss the * time we slept as update_process_times does only a 1 tick -- cgit v1.2.3 From 6e7c402590b75b6b45138792445ee0f0315a8473 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 30 Jan 2008 13:30:05 +0100 Subject: x86: various changes and cleanups to in_p/out_p delay details various changes to the in_p/out_p delay details: - add the io_delay=none method - make each method selectable from the kernel config - simplify the delay code a bit by getting rid of an indirect function call - add the /proc/sys/kernel/io_delay_type sysctl - change 'io_delay=standard|alternate' to io_delay=0x80 and io_delay=0xed - make the io delay config not depend on CONFIG_DEBUG_KERNEL Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner Tested-by: "David P. Reed" --- kernel/sysctl.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'kernel') diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 4bc8e48434a..357b68ba23e 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -53,6 +53,7 @@ #ifdef CONFIG_X86 #include #include +#include #endif static int deprecated_sysctl_warning(struct __sysctl_args *args); @@ -727,6 +728,14 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = &proc_dointvec, }, + { + .ctl_name = CTL_UNNUMBERED, + .procname = "io_delay_type", + .data = &io_delay_type, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec, + }, #endif #if defined(CONFIG_MMU) { -- cgit v1.2.3 From 36df29d7994180cf7f0ccc5d46495855d56d2721 Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Wed, 30 Jan 2008 13:30:51 +0100 Subject: ptrace: generic resume This makes ptrace_request handle all the ptrace requests that wake up the traced task. These do low-level ptrace implementation magic that is not arch-specific and should be kept out of arch code. The implementations on each arch usually do the same thing. The new generic code makes use of the arch_has_single_step macro and generic entry points to handle PTRACE_SINGLESTEP. Signed-off-by: Roland McGrath Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/ptrace.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) (limited to 'kernel') diff --git a/kernel/ptrace.c b/kernel/ptrace.c index c719bb9d79a..fad5f1e8511 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -366,6 +366,50 @@ static int ptrace_setsiginfo(struct task_struct *child, siginfo_t __user * data) return error; } + +#ifdef PTRACE_SINGLESTEP +#define is_singlestep(request) ((request) == PTRACE_SINGLESTEP) +#else +#define is_singlestep(request) 0 +#endif + +#ifdef PTRACE_SYSEMU +#define is_sysemu_singlestep(request) ((request) == PTRACE_SYSEMU_SINGLESTEP) +#else +#define is_sysemu_singlestep(request) 0 +#endif + +static int ptrace_resume(struct task_struct *child, long request, long data) +{ + if (!valid_signal(data)) + return -EIO; + + if (request == PTRACE_SYSCALL) + set_tsk_thread_flag(child, TIF_SYSCALL_TRACE); + else + clear_tsk_thread_flag(child, TIF_SYSCALL_TRACE); + +#ifdef TIF_SYSCALL_EMU + if (request == PTRACE_SYSEMU || request == PTRACE_SYSEMU_SINGLESTEP) + set_tsk_thread_flag(child, TIF_SYSCALL_EMU); + else + clear_tsk_thread_flag(child, TIF_SYSCALL_EMU); +#endif + + if (is_singlestep(request) || is_sysemu_singlestep(request)) { + if (unlikely(!arch_has_single_step())) + return -EIO; + user_enable_single_step(child); + } + else + user_disable_single_step(child); + + child->exit_code = data; + wake_up_process(child); + + return 0; +} + int ptrace_request(struct task_struct *child, long request, long addr, long data) { @@ -390,6 +434,23 @@ int ptrace_request(struct task_struct *child, long request, case PTRACE_DETACH: /* detach a process that was attached. */ ret = ptrace_detach(child, data); break; + +#ifdef PTRACE_SINGLESTEP + case PTRACE_SINGLESTEP: +#endif +#ifdef PTRACE_SYSEMU + case PTRACE_SYSEMU: + case PTRACE_SYSEMU_SINGLESTEP: +#endif + case PTRACE_SYSCALL: + case PTRACE_CONT: + return ptrace_resume(child, request, data); + + case PTRACE_KILL: + if (child->exit_state) /* already dead */ + return 0; + return ptrace_resume(child, request, SIGKILL); + default: break; } -- cgit v1.2.3 From 5b88abbf770a0e1975c668743100f42934f385e8 Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Wed, 30 Jan 2008 13:30:53 +0100 Subject: ptrace: generic PTRACE_SINGLEBLOCK This makes ptrace_request handle PTRACE_SINGLEBLOCK along with PTRACE_CONT et al. The new generic code makes use of the arch_has_block_step macro and generic entry points on machines that define them. [ mingo@elte.hu: bugfix ] Signed-off-by: Roland McGrath Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/ptrace.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/ptrace.c b/kernel/ptrace.c index fad5f1e8511..973d727f5e8 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -373,6 +373,12 @@ static int ptrace_setsiginfo(struct task_struct *child, siginfo_t __user * data) #define is_singlestep(request) 0 #endif +#ifdef PTRACE_SINGLEBLOCK +#define is_singleblock(request) ((request) == PTRACE_SINGLEBLOCK) +#else +#define is_singleblock(request) 0 +#endif + #ifdef PTRACE_SYSEMU #define is_sysemu_singlestep(request) ((request) == PTRACE_SYSEMU_SINGLESTEP) #else @@ -396,7 +402,11 @@ static int ptrace_resume(struct task_struct *child, long request, long data) clear_tsk_thread_flag(child, TIF_SYSCALL_EMU); #endif - if (is_singlestep(request) || is_sysemu_singlestep(request)) { + if (is_singleblock(request)) { + if (unlikely(!arch_has_block_step())) + return -EIO; + user_enable_block_step(child); + } else if (is_singlestep(request) || is_sysemu_singlestep(request)) { if (unlikely(!arch_has_single_step())) return -EIO; user_enable_single_step(child); @@ -438,6 +448,9 @@ int ptrace_request(struct task_struct *child, long request, #ifdef PTRACE_SINGLESTEP case PTRACE_SINGLESTEP: #endif +#ifdef PTRACE_SINGLEBLOCK + case PTRACE_SINGLEBLOCK: +#endif #ifdef PTRACE_SYSEMU case PTRACE_SYSEMU: case PTRACE_SYSEMU_SINGLESTEP: -- cgit v1.2.3 From 65ea5b0349903585bfed9720fa06f5edb4f1cd25 Mon Sep 17 00:00:00 2001 From: "H. Peter Anvin" Date: Wed, 30 Jan 2008 13:30:56 +0100 Subject: x86: rename the struct pt_regs members for 32/64-bit consistency We have a lot of code which differs only by the naming of specific members of structures that contain registers. In order to enable additional unifications, this patch drops the e- or r- size prefix from the register names in struct pt_regs, and drops the x- prefixes for segment registers on the 32-bit side. This patch also performs the equivalent renames in some additional places that might be candidates for unification in the future. Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/signal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/signal.c b/kernel/signal.c index afa4f781f92..bf49ce6f016 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -733,13 +733,13 @@ static void print_fatal_signal(struct pt_regs *regs, int signr) current->comm, task_pid_nr(current), signr); #if defined(__i386__) && !defined(__arch_um__) - printk("code at %08lx: ", regs->eip); + printk("code at %08lx: ", regs->ip); { int i; for (i = 0; i < 16; i++) { unsigned char insn; - __get_user(insn, (unsigned char *)(regs->eip + i)); + __get_user(insn, (unsigned char *)(regs->ip + i)); printk("%02x ", insn); } } -- cgit v1.2.3 From 95c354fe9f7d6decc08a92aa26eb233ecc2155bf Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Wed, 30 Jan 2008 13:31:20 +0100 Subject: spinlock: lockbreak cleanup The break_lock data structure and code for spinlocks is quite nasty. Not only does it double the size of a spinlock but it changes locking to a potentially less optimal trylock. Put all of that under CONFIG_GENERIC_LOCKBREAK, and introduce a __raw_spin_is_contended that uses the lock data itself to determine whether there are waiters on the lock, to be used if CONFIG_GENERIC_LOCKBREAK is not set. Rename need_lockbreak to spin_needbreak, make it use spin_is_contended to decouple it from the spinlock implementation, and make it typesafe (rwlocks do not have any need_lockbreak sites -- why do they even get bloated up with that break_lock then?). Signed-off-by: Nick Piggin Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/sched.c | 16 ++++++---------- kernel/spinlock.c | 3 +-- 2 files changed, 7 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index 524285e46fa..ba4c88088f6 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -4945,19 +4945,15 @@ EXPORT_SYMBOL(_cond_resched); */ int cond_resched_lock(spinlock_t *lock) { + int resched = need_resched() && system_state == SYSTEM_RUNNING; int ret = 0; - if (need_lockbreak(lock)) { + if (spin_needbreak(lock) || resched) { spin_unlock(lock); - cpu_relax(); - ret = 1; - spin_lock(lock); - } - if (need_resched() && system_state == SYSTEM_RUNNING) { - spin_release(&lock->dep_map, 1, _THIS_IP_); - _raw_spin_unlock(lock); - preempt_enable_no_resched(); - __cond_resched(); + if (resched && need_resched()) + __cond_resched(); + else + cpu_relax(); ret = 1; spin_lock(lock); } diff --git a/kernel/spinlock.c b/kernel/spinlock.c index cd72424c266..ae28c824512 100644 --- a/kernel/spinlock.c +++ b/kernel/spinlock.c @@ -65,8 +65,7 @@ EXPORT_SYMBOL(_write_trylock); * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are * not re-enabled during lock-acquire (which the preempt-spin-ops do): */ -#if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \ - defined(CONFIG_DEBUG_LOCK_ALLOC) +#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) void __lockfunc _read_lock(rwlock_t *lock) { -- cgit v1.2.3 From 16c3e389e7a7254ff8dc7029ac4fbe996c3c75bf Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Wed, 30 Jan 2008 13:31:47 +0100 Subject: x86: ptrace_request peekdata/pokedata This makes ptrace_request handle {PEEK,POKE}{TEXT,DATA} directly. Every arch_ptrace that could call generic_ptrace_peekdata already has a default case calling ptrace_request, so this keeps things simpler for the arch code. Signed-off-by: Roland McGrath Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/ptrace.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'kernel') diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 973d727f5e8..e6a99d2793b 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -426,6 +426,13 @@ int ptrace_request(struct task_struct *child, long request, int ret = -EIO; switch (request) { + case PTRACE_PEEKTEXT: + case PTRACE_PEEKDATA: + return generic_ptrace_peekdata(child, addr, data); + case PTRACE_POKETEXT: + case PTRACE_POKEDATA: + return generic_ptrace_pokedata(child, addr, data); + #ifdef PTRACE_OLDSETOPTIONS case PTRACE_OLDSETOPTIONS: #endif -- cgit v1.2.3 From 032d82d9065dec0e26718eca376c2029e4bd0595 Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Wed, 30 Jan 2008 13:31:47 +0100 Subject: x86: compat_ptrace_request This adds a compat_ptrace_request that is the analogue of ptrace_request for the things that 32-on-64 ptrace implementations can share in common. So far there are just a couple of requests handled generically. Signed-off-by: Roland McGrath Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/ptrace.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'kernel') diff --git a/kernel/ptrace.c b/kernel/ptrace.c index e6a99d2793b..ed1c3d56c2c 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -607,3 +607,41 @@ int generic_ptrace_pokedata(struct task_struct *tsk, long addr, long data) copied = access_process_vm(tsk, addr, &data, sizeof(data), 1); return (copied == sizeof(data)) ? 0 : -EIO; } + +#ifdef CONFIG_COMPAT +#include + +int compat_ptrace_request(struct task_struct *child, compat_long_t request, + compat_ulong_t addr, compat_ulong_t data) +{ + compat_ulong_t __user *datap = compat_ptr(data); + compat_ulong_t word; + int ret; + + switch (request) { + case PTRACE_PEEKTEXT: + case PTRACE_PEEKDATA: + ret = access_process_vm(child, addr, &word, sizeof(word), 0); + if (ret != sizeof(word)) + ret = -EIO; + else + ret = put_user(word, datap); + break; + + case PTRACE_POKETEXT: + case PTRACE_POKEDATA: + ret = access_process_vm(child, addr, &data, sizeof(data), 1); + ret = (ret != sizeof(data) ? -EIO : 0); + break; + + case PTRACE_GETEVENTMSG: + ret = put_user((compat_ulong_t) child->ptrace_message, datap); + break; + + default: + ret = ptrace_request(child, request, addr, data); + } + + return ret; +} +#endif /* CONFIG_COMPAT */ -- cgit v1.2.3 From c269f19617f508cc5c29c0b064c1a437d7011a46 Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Wed, 30 Jan 2008 13:31:48 +0100 Subject: x86: compat_sys_ptrace This adds a generic definition of compat_sys_ptrace that calls compat_arch_ptrace, parallel to sys_ptrace/arch_ptrace. Some machines needing this already define a function by that name. The new generic function is defined only on machines that put #define __ARCH_WANT_COMPAT_SYS_PTRACE into asm/ptrace.h. Signed-off-by: Roland McGrath Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/ptrace.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) (limited to 'kernel') diff --git a/kernel/ptrace.c b/kernel/ptrace.c index ed1c3d56c2c..e6e9b8be4b0 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -644,4 +644,50 @@ int compat_ptrace_request(struct task_struct *child, compat_long_t request, return ret; } + +#ifdef __ARCH_WANT_COMPAT_SYS_PTRACE +asmlinkage long compat_sys_ptrace(compat_long_t request, compat_long_t pid, + compat_long_t addr, compat_long_t data) +{ + struct task_struct *child; + long ret; + + /* + * This lock_kernel fixes a subtle race with suid exec + */ + lock_kernel(); + if (request == PTRACE_TRACEME) { + ret = ptrace_traceme(); + goto out; + } + + child = ptrace_get_task_struct(pid); + if (IS_ERR(child)) { + ret = PTR_ERR(child); + goto out; + } + + if (request == PTRACE_ATTACH) { + ret = ptrace_attach(child); + /* + * Some architectures need to do book-keeping after + * a ptrace attach. + */ + if (!ret) + arch_ptrace_attach(child); + goto out_put_task_struct; + } + + ret = ptrace_check_attach(child, request == PTRACE_KILL); + if (!ret) + ret = compat_arch_ptrace(child, request, addr, data); + + out_put_task_struct: + put_task_struct(child); + out: + unlock_kernel(); + return ret; +} +#endif /* __ARCH_WANT_COMPAT_SYS_PTRACE */ + #endif /* CONFIG_COMPAT */ -- cgit v1.2.3 From 9e094c17ee2b85130ab7b2ea37456f6867eb687a Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Wed, 30 Jan 2008 13:32:48 +0100 Subject: genirq: turn irq debugging options into module params This allows to change them at runtime using sysfs. No need to reboot to set them. I only added aliases (kernel.noirqdebug etc.) so the old options still work. Signed-off-by: Andi Kleen Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/irq/spurious.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/irq/spurious.c b/kernel/irq/spurious.c index 32b161972fa..a6b2bc831dd 100644 --- a/kernel/irq/spurious.c +++ b/kernel/irq/spurious.c @@ -10,6 +10,7 @@ #include #include #include +#include static int irqfixup __read_mostly; @@ -225,6 +226,8 @@ int noirqdebug_setup(char *str) } __setup("noirqdebug", noirqdebug_setup); +module_param(noirqdebug, bool, 0644); +MODULE_PARM_DESC(noirqdebug, "Disable irq lockup detection when true"); static int __init irqfixup_setup(char *str) { @@ -236,6 +239,8 @@ static int __init irqfixup_setup(char *str) } __setup("irqfixup", irqfixup_setup); +module_param(irqfixup, int, 0644); +MODULE_PARM_DESC("irqfixup", "0: No fixup, 1: irqfixup mode 2: irqpoll mode"); static int __init irqpoll_setup(char *str) { -- cgit v1.2.3 From 96d97cf03b3d68e6c857623da93acd522b2b7e1a Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Wed, 30 Jan 2008 13:32:48 +0100 Subject: x86: add /proc/irq/*/spurious to dump the spurious irq debugging state This is useful to debug problems with interrupt handlers that return sometimes IRQ_NONE. Signed-off-by: Andi Kleen Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/irq/proc.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 50b81b98046..c2f2ccb0549 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -75,6 +75,18 @@ static int irq_affinity_write_proc(struct file *file, const char __user *buffer, #endif +static int irq_spurious_read(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct irq_desc *d = &irq_desc[(long) data]; + return sprintf(page, "count %u\n" + "unhandled %u\n" + "last_unhandled %u ms\n", + d->irq_count, + d->irqs_unhandled, + jiffies_to_msecs(d->last_unhandled)); +} + #define MAX_NAMELEN 128 static int name_unique(unsigned int irq, struct irqaction *new_action) @@ -118,6 +130,7 @@ void register_handler_proc(unsigned int irq, struct irqaction *action) void register_irq_proc(unsigned int irq) { char name [MAX_NAMELEN]; + struct proc_dir_entry *entry; if (!root_irq_dir || (irq_desc[irq].chip == &no_irq_chip) || @@ -132,8 +145,6 @@ void register_irq_proc(unsigned int irq) #ifdef CONFIG_SMP { - struct proc_dir_entry *entry; - /* create /proc/irq//smp_affinity */ entry = create_proc_entry("smp_affinity", 0600, irq_desc[irq].dir); @@ -144,6 +155,12 @@ void register_irq_proc(unsigned int irq) } } #endif + + entry = create_proc_entry("spurious", 0444, irq_desc[irq].dir); + if (entry) { + entry->data = (void *)(long)irq; + entry->read_proc = irq_spurious_read; + } } #undef MAX_NAMELEN -- cgit v1.2.3 From 79b4cc5ee7a8086ac2c9c0afa52e6d687ce1ffef Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Wed, 30 Jan 2008 13:32:50 +0100 Subject: debug: move WARN_ON() out of line A quick grep shows that there are currently 1145 instances of WARN_ON in the kernel. Currently, WARN_ON is pretty much entirely inlined, which makes it hard to enhance it without growing the size of the kernel (and getting Andrew unhappy). This patch build on top of Olof's patch that introduces __WARN, and places the slowpath out of line. It also uses Ingo's suggestion to not use __FUNCTION__ but to use kallsyms to do the lookup; this saves a ton of extra space since gcc doesn't need to store the function string twice now: 3936367 833603 624736 5394706 525112 vmlinux.before 3917508 833603 624736 5375847 520767 vmlinux-slowpath 15Kb savings... Signed-off-by: Arjan van de Ven CC: Andrew Morton CC: Olof Johansson Acked-by: Matt Meckall Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/panic.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'kernel') diff --git a/kernel/panic.c b/kernel/panic.c index da4d6bac270..0ebea438278 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -20,6 +20,7 @@ #include #include #include +#include int panic_on_oops; int tainted; @@ -292,6 +293,20 @@ void oops_exit(void) (unsigned long long)oops_id); } +#ifdef WANT_WARN_ON_SLOWPATH +void warn_on_slowpath(const char *file, int line) +{ + char function[KSYM_SYMBOL_LEN]; + unsigned long caller = (unsigned long) __builtin_return_address(0); + + sprint_symbol(function, caller); + printk(KERN_WARNING "WARNING: at %s:%d %s()\n", file, + line, function); + dump_stack(); +} +EXPORT_SYMBOL(warn_on_slowpath); +#endif + #ifdef CONFIG_CC_STACKPROTECTOR /* * Called when gcc's -fstack-protector feature is used, and -- cgit v1.2.3 From 71c339116a216b181fc5e203ef51a033fe5e38cf Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Wed, 30 Jan 2008 13:32:50 +0100 Subject: debug: add the end-of-trace marker and the module list to Unlike oopses, WARN_ON() currently does't print the loaded modules list. This makes it harder to take action on certain bug reports. For example, recently there were a set of WARN_ON()s reported in the mac80211 stack, which were just signalling a driver bug. It takes then anther round trip to the bug reporter (if he responds at all) to find out which driver is at fault. Another issue is that, unlike oopses, WARN_ON() doesn't currently printk the helpful "cut here" line, nor the "end of trace" marker. Now that WARN_ON() is out of line, the size increase due to this is minimal and it's worth adding. Signed-off-by: Arjan van de Ven Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/panic.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/panic.c b/kernel/panic.c index 0ebea438278..d9e90cfe329 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -281,6 +281,13 @@ static int init_oops_id(void) } late_initcall(init_oops_id); +static void print_oops_end_marker(void) +{ + init_oops_id(); + printk(KERN_WARNING "---[ end trace %016llx ]---\n", + (unsigned long long)oops_id); +} + /* * Called when the architecture exits its oops handler, after printing * everything. @@ -288,9 +295,7 @@ late_initcall(init_oops_id); void oops_exit(void) { do_oops_enter_exit(); - init_oops_id(); - printk(KERN_WARNING "---[ end trace %016llx ]---\n", - (unsigned long long)oops_id); + print_oops_end_marker(); } #ifdef WANT_WARN_ON_SLOWPATH @@ -298,11 +303,14 @@ void warn_on_slowpath(const char *file, int line) { char function[KSYM_SYMBOL_LEN]; unsigned long caller = (unsigned long) __builtin_return_address(0); - sprint_symbol(function, caller); + + printk(KERN_WARNING "------------[ cut here ]------------\n"); printk(KERN_WARNING "WARNING: at %s:%d %s()\n", file, line, function); + print_modules(); dump_stack(); + print_oops_end_marker(); } EXPORT_SYMBOL(warn_on_slowpath); #endif -- cgit v1.2.3 From 8c1c9356429741a82ff176d0f3400fb9e06b2a30 Mon Sep 17 00:00:00 2001 From: Ananth N Mavinakayanahalli Date: Wed, 30 Jan 2008 13:32:53 +0100 Subject: x86: kprobes: add kprobes smoke tests that run on boot Here is a quick and naive smoke test for kprobes. This is intended to just verify if some unrelated change broke the *probes subsystem. It is self contained, architecture agnostic and isn't of any great use by itself. This needs to be built in the kernel and runs a basic set of tests to verify if kprobes, jprobes and kretprobes run fine on the kernel. In case of an error, it'll print out a message with a "BUG" prefix. This is a start; we intend to add more tests to this bucket over time. Thanks to Jim Keniston and Masami Hiramatsu for comments and suggestions. Tested on x86 (32/64) and powerpc. Signed-off-by: Ananth N Mavinakayanahalli Acked-by: Masami Hiramatsu Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- kernel/Makefile | 1 + kernel/kprobes.c | 2 + kernel/test_kprobes.c | 216 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 219 insertions(+) create mode 100644 kernel/test_kprobes.c (limited to 'kernel') diff --git a/kernel/Makefile b/kernel/Makefile index 390d4214626..62015c3d8d9 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -43,6 +43,7 @@ obj-$(CONFIG_CPUSETS) += cpuset.o obj-$(CONFIG_CGROUP_NS) += ns_cgroup.o obj-$(CONFIG_IKCONFIG) += configs.o obj-$(CONFIG_STOP_MACHINE) += stop_machine.o +obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o obj-$(CONFIG_AUDIT) += audit.o auditfilter.o obj-$(CONFIG_AUDITSYSCALL) += auditsc.o obj-$(CONFIG_AUDIT_TREE) += audit_tree.o diff --git a/kernel/kprobes.c b/kernel/kprobes.c index e3a5d817ac9..d0493eafea3 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -824,6 +824,8 @@ static int __init init_kprobes(void) if (!err) err = register_die_notifier(&kprobe_exceptions_nb); + if (!err) + init_test_probes(); return err; } diff --git a/kernel/test_kprobes.c b/kernel/test_kprobes.c new file mode 100644 index 00000000000..88cdb109e13 --- /dev/null +++ b/kernel/test_kprobes.c @@ -0,0 +1,216 @@ +/* + * test_kprobes.c - simple sanity test for *probes + * + * Copyright IBM Corp. 2008 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it would be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See + * the GNU General Public License for more details. + */ + +#include +#include +#include + +#define div_factor 3 + +static u32 rand1, preh_val, posth_val, jph_val; +static int errors, handler_errors, num_tests; + +static noinline u32 kprobe_target(u32 value) +{ + /* + * gcc ignores noinline on some architectures unless we stuff + * sufficient lard into the function. The get_kprobe() here is + * just for that. + * + * NOTE: We aren't concerned about the correctness of get_kprobe() + * here; hence, this call is neither under !preempt nor with the + * kprobe_mutex held. This is fine(tm) + */ + if (get_kprobe((void *)0xdeadbeef)) + printk(KERN_INFO "Kprobe smoke test: probe on 0xdeadbeef!\n"); + + return (value / div_factor); +} + +static int kp_pre_handler(struct kprobe *p, struct pt_regs *regs) +{ + preh_val = (rand1 / div_factor); + return 0; +} + +static void kp_post_handler(struct kprobe *p, struct pt_regs *regs, + unsigned long flags) +{ + if (preh_val != (rand1 / div_factor)) { + handler_errors++; + printk(KERN_ERR "Kprobe smoke test failed: " + "incorrect value in post_handler\n"); + } + posth_val = preh_val + div_factor; +} + +static struct kprobe kp = { + .symbol_name = "kprobe_target", + .pre_handler = kp_pre_handler, + .post_handler = kp_post_handler +}; + +static int test_kprobe(void) +{ + int ret; + + ret = register_kprobe(&kp); + if (ret < 0) { + printk(KERN_ERR "Kprobe smoke test failed: " + "register_kprobe returned %d\n", ret); + return ret; + } + + ret = kprobe_target(rand1); + unregister_kprobe(&kp); + + if (preh_val == 0) { + printk(KERN_ERR "Kprobe smoke test failed: " + "kprobe pre_handler not called\n"); + handler_errors++; + } + + if (posth_val == 0) { + printk(KERN_ERR "Kprobe smoke test failed: " + "kprobe post_handler not called\n"); + handler_errors++; + } + + return 0; +} + +static u32 j_kprobe_target(u32 value) +{ + if (value != rand1) { + handler_errors++; + printk(KERN_ERR "Kprobe smoke test failed: " + "incorrect value in jprobe handler\n"); + } + + jph_val = rand1; + jprobe_return(); + return 0; +} + +static struct jprobe jp = { + .entry = j_kprobe_target, + .kp.symbol_name = "kprobe_target" +}; + +static int test_jprobe(void) +{ + int ret; + + ret = register_jprobe(&jp); + if (ret < 0) { + printk(KERN_ERR "Kprobe smoke test failed: " + "register_jprobe returned %d\n", ret); + return ret; + } + + ret = kprobe_target(rand1); + unregister_jprobe(&jp); + if (jph_val == 0) { + printk(KERN_ERR "Kprobe smoke test failed: " + "jprobe handler not called\n"); + handler_errors++; + } + + return 0; +} + +#ifdef CONFIG_KRETPROBES +static u32 krph_val; + +static int return_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + unsigned long ret = regs_return_value(regs); + + if (ret != (rand1 / div_factor)) { + handler_errors++; + printk(KERN_ERR "Kprobe smoke test failed: " + "incorrect value in kretprobe handler\n"); + } + + krph_val = (rand1 / div_factor); + return 0; +} + +static struct kretprobe rp = { + .handler = return_handler, + .kp.symbol_name = "kprobe_target" +}; + +static int test_kretprobe(void) +{ + int ret; + + ret = register_kretprobe(&rp); + if (ret < 0) { + printk(KERN_ERR "Kprobe smoke test failed: " + "register_kretprobe returned %d\n", ret); + return ret; + } + + ret = kprobe_target(rand1); + unregister_kretprobe(&rp); + if (krph_val == 0) { + printk(KERN_ERR "Kprobe smoke test failed: " + "kretprobe handler not called\n"); + handler_errors++; + } + + return 0; +} +#endif /* CONFIG_KRETPROBES */ + +int init_test_probes(void) +{ + int ret; + + do { + rand1 = random32(); + } while (rand1 <= div_factor); + + printk(KERN_INFO "Kprobe smoke test started\n"); + num_tests++; + ret = test_kprobe(); + if (ret < 0) + errors++; + + num_tests++; + ret = test_jprobe(); + if (ret < 0) + errors++; + +#ifdef CONFIG_KRETPROBES + num_tests++; + ret = test_kretprobe(); + if (ret < 0) + errors++; +#endif /* CONFIG_KRETPROBES */ + + if (errors) + printk(KERN_ERR "BUG: Kprobe smoke test: %d out of " + "%d tests failed\n", errors, num_tests); + else if (handler_errors) + printk(KERN_ERR "BUG: Kprobe smoke test: %d error(s) " + "running handlers\n", handler_errors); + else + printk(KERN_INFO "Kprobe smoke test passed successfully\n"); + + return 0; +} -- cgit v1.2.3 From 076f9776f5d8d131b36955db8641aba3893c2c1b Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 30 Jan 2008 13:33:06 +0100 Subject: x86: make early printk selectable on 64-bit as well Enable CONFIG_EMBEDDED to select CONFIG_EARLY_PRINTK on 64-bit as well. saves ~2K: text data bss dec hex filename 7290283 3672091 1907848 12870222 c4624e vmlinux.before 7288373 3671795 1907848 12868016 c459b0 vmlinux.after Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/printk.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'kernel') diff --git a/kernel/printk.c b/kernel/printk.c index 3b7c968d0ef..58bbec68411 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -36,6 +36,13 @@ #include +/* + * Architectures can override it: + */ +void __attribute__((weak)) early_printk(const char *fmt, ...) +{ +} + #define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT) /* printk's without a loglevel use this.. */ -- cgit v1.2.3 From 6dab27784b2a97823b522e1cb88e40be40a93d45 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Wed, 30 Jan 2008 13:33:08 +0100 Subject: x86: add a simple backtrace test module During the work on the x86 32 and 64 bit backtrace code I found it useful to have a simple test module to test a process and irq context backtrace. Since the existing backtrace code was buggy, I figure it might be useful to have such a test module in the kernel so that maybe we can even detect such bugs earlier.. [ mingo@elte.hu: build fix ] Signed-off-by: Arjan van de Ven Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/Makefile | 1 + kernel/backtracetest.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 kernel/backtracetest.c (limited to 'kernel') diff --git a/kernel/Makefile b/kernel/Makefile index 62015c3d8d9..8885627ea02 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -36,6 +36,7 @@ obj-$(CONFIG_KALLSYMS) += kallsyms.o obj-$(CONFIG_PM) += power/ obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o obj-$(CONFIG_KEXEC) += kexec.o +obj-$(CONFIG_BACKTRACE_SELF_TEST) += backtracetest.o obj-$(CONFIG_COMPAT) += compat.o obj-$(CONFIG_CGROUPS) += cgroup.o obj-$(CONFIG_CGROUP_DEBUG) += cgroup_debug.o diff --git a/kernel/backtracetest.c b/kernel/backtracetest.c new file mode 100644 index 00000000000..d1a7605c5b8 --- /dev/null +++ b/kernel/backtracetest.c @@ -0,0 +1,48 @@ +/* + * Simple stack backtrace regression test module + * + * (C) Copyright 2008 Intel Corporation + * Author: Arjan van de Ven + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + */ + +#include +#include +#include + +static struct timer_list backtrace_timer; + +static void backtrace_test_timer(unsigned long data) +{ + printk("Testing a backtrace from irq context.\n"); + printk("The following trace is a kernel self test and not a bug!\n"); + dump_stack(); +} +static int backtrace_regression_test(void) +{ + printk("====[ backtrace testing ]===========\n"); + printk("Testing a backtrace from process context.\n"); + printk("The following trace is a kernel self test and not a bug!\n"); + dump_stack(); + + init_timer(&backtrace_timer); + backtrace_timer.function = backtrace_test_timer; + mod_timer(&backtrace_timer, jiffies + 10); + + msleep(10); + printk("====[ end of backtrace testing ]====\n"); + return 0; +} + +static void exitf(void) +{ +} + +module_init(backtrace_regression_test); +module_exit(exitf); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Arjan van de Ven "); -- cgit v1.2.3 From 70edcd77a0d6d0f8731c826764f5eb6732f521e9 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 30 Jan 2008 13:33:24 +0100 Subject: genirq: stackdump after the "Trying to free already-free IRQ" message these bugs are harder to find than they seem, a stackdump helps. make it dependent on CONFIG_DEBUG_SHIRQ so that people can turn it off if it annoys them. Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/irq/manage.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 1f314221d53..438a0146428 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -479,6 +479,9 @@ void free_irq(unsigned int irq, void *dev_id) return; } printk(KERN_ERR "Trying to free already-free IRQ %d\n", irq); +#ifdef CONFIG_DEBUG_SHIRQ + dump_stack(); +#endif spin_unlock_irqrestore(&desc->lock, flags); return; } -- cgit v1.2.3 From dd5af90a7f3d79e04b7eace9a98644dbf2038f4d Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Wed, 30 Jan 2008 13:33:32 +0100 Subject: x86/non-x86: percpu, node ids, apic ids x86.git fixup Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- kernel/module.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'kernel') diff --git a/kernel/module.c b/kernel/module.c index f6a4e721fd4..bd60278ee70 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -430,6 +430,14 @@ static unsigned int find_pcpusec(Elf_Ehdr *hdr, return find_sec(hdr, sechdrs, secstrings, ".data.percpu"); } +static void percpu_modcopy(void *pcpudest, const void *from, unsigned long size) +{ + int cpu; + + for_each_possible_cpu(cpu) + memcpy(pcpudest + per_cpu_offset(cpu), from, size); +} + static int percpu_modinit(void) { pcpu_num_used = 2; -- cgit v1.2.3 From 6d4e4c4fca5be806b888d606894d914847e82d78 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 16:41:05 +0200 Subject: KVM: Disallow fork() and similar games when using a VM We don't want the meaning of guest userspace changing under our feet. Signed-off-by: Avi Kivity --- kernel/fork.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 314f5101d2b..05e0b6f4365 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -393,6 +393,7 @@ void fastcall __mmdrop(struct mm_struct *mm) destroy_context(mm); free_mm(mm); } +EXPORT_SYMBOL_GPL(__mmdrop); /* * Decrement the use count and release all resources for an mm. -- cgit v1.2.3 From 296825cbe14d4c95ee9c41ca5824f7487bfb4d9d Mon Sep 17 00:00:00 2001 From: Srivatsa Vaddagiri Date: Thu, 31 Jan 2008 22:45:22 +0100 Subject: sched: fix high wake up latencies with FAIR_USER_SCHED The reason why we are getting better wakeup latencies for !FAIR_USER_SCHED is because of this snippet of code in place_entity(): if (!initial) { /* sleeps upto a single latency don't count. */ if (sched_feat(NEW_FAIR_SLEEPERS) && entity_is_task(se)) ^^^^^^^^^^^^^^^^^^ vruntime -= sysctl_sched_latency; /* ensure we never gain time by being placed backwards. */ vruntime = max_vruntime(se->vruntime, vruntime); } NEW_FAIR_SLEEPERS feature gives credit for sleeping only to tasks and not group-level entities. With the patch attached, I could see that wakeup latencies with FAIR_USER_SCHED are restored to the same level as !FAIR_USER_SCHED. Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 72e25c7a3a1..cf958aefac3 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -520,7 +520,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) if (!initial) { /* sleeps upto a single latency don't count. */ - if (sched_feat(NEW_FAIR_SLEEPERS) && entity_is_task(se)) + if (sched_feat(NEW_FAIR_SLEEPERS)) vruntime -= sysctl_sched_latency; /* ensure we never gain time by being placed backwards. */ -- cgit v1.2.3 From ef9884e6f29bbe1075204f962a00f7533bf7e8f3 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 31 Jan 2008 22:45:22 +0100 Subject: sched: let +nice tasks have smaller impact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Michel Dänzr has bisected an interactivity problem with plus-reniced tasks back to this commit: 810e95ccd58d91369191aa4ecc9e6d4a10d8d0c8 is first bad commit commit 810e95ccd58d91369191aa4ecc9e6d4a10d8d0c8 Author: Peter Zijlstra Date: Mon Oct 15 17:00:14 2007 +0200 sched: another wakeup_granularity fix unit mis-match: wakeup_gran was used against a vruntime fix this by assymetrically scaling the vtime of positive reniced tasks. Bisected-by: Michel Dänzer Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index cf958aefac3..6c091d6e159 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -1106,7 +1106,11 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) } gran = sysctl_sched_wakeup_granularity; - if (unlikely(se->load.weight != NICE_0_LOAD)) + /* + * More easily preempt - nice tasks, while not making + * it harder for + nice tasks. + */ + if (unlikely(se->load.weight > NICE_0_LOAD)) gran = calc_delta_fair(gran, &se->load); if (pse->vruntime + gran < se->vruntime) -- cgit v1.2.3 From 5aff0531ee2403b319e89bd04055fc41173e95fd Mon Sep 17 00:00:00 2001 From: Gerald Stralko Date: Thu, 31 Jan 2008 22:45:23 +0100 Subject: sched: remove unused params This removes the extra struct task_struct *p parameter in inc_nr_running and dec_nr_running functions. Signed-off by: Jerry Stralko Signed-off-by: Ingo Molnar --- kernel/sched.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched.c b/kernel/sched.c index ba4c88088f6..8355e007e02 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -1255,12 +1255,12 @@ static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd); #define sched_class_highest (&rt_sched_class) -static void inc_nr_running(struct task_struct *p, struct rq *rq) +static void inc_nr_running(struct rq *rq) { rq->nr_running++; } -static void dec_nr_running(struct task_struct *p, struct rq *rq) +static void dec_nr_running(struct rq *rq) { rq->nr_running--; } @@ -1354,7 +1354,7 @@ static void activate_task(struct rq *rq, struct task_struct *p, int wakeup) rq->nr_uninterruptible--; enqueue_task(rq, p, wakeup); - inc_nr_running(p, rq); + inc_nr_running(rq); } /* @@ -1366,7 +1366,7 @@ static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep) rq->nr_uninterruptible++; dequeue_task(rq, p, sleep); - dec_nr_running(p, rq); + dec_nr_running(rq); } /** @@ -2006,7 +2006,7 @@ void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags) * management (if any): */ p->sched_class->task_new(rq, p); - inc_nr_running(p, rq); + inc_nr_running(rq); } check_preempt_curr(rq, p); #ifdef CONFIG_SMP -- cgit v1.2.3 From c4772d99300a9fc13c86aaa370e630c5973664f6 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Thu, 31 Jan 2008 22:45:23 +0100 Subject: debug: turn ignore_loglevel into an early param i was debugging early crashes and wondered where all the printks went. The reason: ignore_loglevel_setup() was not called yet ... Signed-off-by: Ingo Molnar --- kernel/printk.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/printk.c b/kernel/printk.c index 58bbec68411..29ae1e99cde 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -455,10 +455,10 @@ static int __init ignore_loglevel_setup(char *str) ignore_loglevel = 1; printk(KERN_INFO "debug: ignoring loglevel setting.\n"); - return 1; + return 0; } -__setup("ignore_loglevel", ignore_loglevel_setup); +early_param("ignore_loglevel", ignore_loglevel_setup); /* * Write out chars from start to end - 1 inclusive -- cgit v1.2.3 From 13f09b95a82c46ed608d057b22e0dd18ebfff22a Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 31 Jan 2008 20:40:29 -0500 Subject: Ensure that we export __fatal_signal_pending() It may be used by the modules nfs.ko and sunrpc.ko Signed-off-by: Trond Myklebust [ Made it a regular export rather than GPL-only - Linus ] Signed-off-by: Linus Torvalds --- kernel/signal.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/signal.c b/kernel/signal.c index 8054dd4e2d7..4333b6dbb42 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -998,6 +998,7 @@ int fastcall __fatal_signal_pending(struct task_struct *tsk) { return sigismember(&tsk->pending.signal, SIGKILL); } +EXPORT_SYMBOL(__fatal_signal_pending); /* * Must be called under rcu_read_lock() or with tasklist_lock read-held. -- cgit v1.2.3 From 406a1d868001423c85a3165288e566e65f424fe6 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 28 Jan 2008 20:47:09 -0800 Subject: [AUDIT]: Increase skb->truesize in audit_expand The recent UDP patch exposed this bug in the audit code. It was calling pskb_expand_head without increasing skb->truesize. The caller of pskb_expand_head needs to do so because that function is designed to be called in places where truesize is already fixed and therefore it doesn't update its value. Because the audit system is using it in a place where the truesize has not yet been fixed, it needs to update its value manually. Signed-off-by: Herbert Xu Acked-by: James Morris Signed-off-by: David S. Miller --- kernel/audit.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/audit.c b/kernel/audit.c index f93c2713017..801c946dd24 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -1200,13 +1200,17 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask, static inline int audit_expand(struct audit_buffer *ab, int extra) { struct sk_buff *skb = ab->skb; - int ret = pskb_expand_head(skb, skb_headroom(skb), extra, - ab->gfp_mask); + int oldtail = skb_tailroom(skb); + int ret = pskb_expand_head(skb, 0, extra, ab->gfp_mask); + int newtail = skb_tailroom(skb); + if (ret < 0) { audit_log_lost("out of memory in audit_expand"); return 0; } - return skb_tailroom(skb); + + skb->truesize += newtail - oldtail; + return newtail; } /* -- cgit v1.2.3 From 3588a085cd52ef080bf72df772378e1ba6bb292f Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 1 Feb 2008 17:45:13 +0100 Subject: hrtimer: fix hrtimer_init_sleeper() users this patch: commit 37bb6cb4097e29ffee970065b74499cbf10603a3 Author: Peter Zijlstra Date: Fri Jan 25 21:08:32 2008 +0100 hrtimer: unlock hrtimer_wakeup Broke hrtimer_init_sleeper() users. It forgot to fix up the futex caller of this function to detect the failed queueing and messed up the do_nanosleep() caller in that it could leak a TASK_INTERRUPTIBLE state. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- kernel/futex.c | 2 ++ kernel/hrtimer.c | 2 ++ 2 files changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/futex.c b/kernel/futex.c index db9824de8bf..0edd314798c 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -1252,6 +1252,8 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared, t.timer.expires = *abs_time; hrtimer_start(&t.timer, t.timer.expires, HRTIMER_MODE_ABS); + if (!hrtimer_active(&t.timer)) + t.task = NULL; /* * the timer could have already expired, in which diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index bd5d6b5060b..1069998fe25 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -1315,6 +1315,8 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod } while (t->task && !signal_pending(current)); + __set_current_state(TASK_RUNNING); + return t->task == NULL; } -- cgit v1.2.3 From 1001d0a9ee74a468077dfd4da0565174e88de26b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 1 Feb 2008 17:45:13 +0100 Subject: timekeeping: update xtime_cache when time(zone) changes xtime_cache needs to be updated whenever xtime and or wall_to_monotic are changed. Otherwise users of xtime_cache might see a stale (and in the case of timezone changes utterly wrong) value until the next update happens. Fixup the obvious places, which miss this update. Signed-off-by: Thomas Gleixner Acked-by: John Stultz Tested-by: Dhaval Giani Signed-off-by: Ingo Molnar --- kernel/time.c | 1 + kernel/time/timekeeping.c | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/time.c b/kernel/time.c index 09d3c45c4da..4064c0566e7 100644 --- a/kernel/time.c +++ b/kernel/time.c @@ -129,6 +129,7 @@ static inline void warp_clock(void) write_seqlock_irq(&xtime_lock); wall_to_monotonic.tv_sec -= sys_tz.tz_minuteswest * 60; xtime.tv_sec += sys_tz.tz_minuteswest * 60; + update_xtime_cache(0); write_sequnlock_irq(&xtime_lock); clock_was_set(); } diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 092a2366b5a..cd5dbc4579c 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -47,7 +47,7 @@ struct timespec wall_to_monotonic __attribute__ ((aligned (16))); static unsigned long total_sleep_time; /* seconds */ static struct timespec xtime_cache __attribute__ ((aligned (16))); -static inline void update_xtime_cache(u64 nsec) +void update_xtime_cache(u64 nsec) { xtime_cache = xtime; timespec_add_ns(&xtime_cache, nsec); @@ -145,6 +145,7 @@ int do_settimeofday(struct timespec *tv) set_normalized_timespec(&xtime, sec, nsec); set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec); + update_xtime_cache(0); clock->error = 0; ntp_clear(); @@ -252,8 +253,8 @@ void __init timekeeping_init(void) xtime.tv_nsec = 0; set_normalized_timespec(&wall_to_monotonic, -xtime.tv_sec, -xtime.tv_nsec); + update_xtime_cache(0); total_sleep_time = 0; - write_sequnlock_irqrestore(&xtime_lock, flags); } @@ -290,6 +291,7 @@ static int timekeeping_resume(struct sys_device *dev) } /* Make sure that we have the correct xtime reference */ timespec_add_ns(&xtime, timekeeping_suspend_nsecs); + update_xtime_cache(0); /* re-base the last cycle value */ clock->cycle_last = clocksource_read(clock); clock->error = 0; -- cgit v1.2.3 From 5df7fa1c62146a0933767d040d400013310dbcc7 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 1 Feb 2008 17:45:14 +0100 Subject: tick-sched: add more debug information To allow better diagnosis of tick-sched related, especially NOHZ related problems, we need to know when the last wakeup via an irq happened and when the CPU left the idle state. Add two fields (idle_waketime, idle_exittime) to the tick_sched structure and add them to the timer_list output. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- kernel/time/tick-sched.c | 2 ++ kernel/time/timer_list.c | 2 ++ 2 files changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 63f24b55069..88267f0a847 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -137,6 +137,7 @@ void tick_nohz_update_jiffies(void) cpu_clear(cpu, nohz_cpu_mask); now = ktime_get(); + ts->idle_waketime = now; local_irq_save(flags); tick_do_update_jiffies64(now); @@ -400,6 +401,7 @@ void tick_nohz_restart_sched_tick(void) * Cancel the scheduled timer and restore the tick */ ts->tick_stopped = 0; + ts->idle_exittime = now; hrtimer_cancel(&ts->sched_timer); ts->sched_timer.expires = ts->idle_tick; diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index 12c5f4cb6b8..d3d94c1a0fd 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c @@ -166,6 +166,8 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now) P(idle_calls); P(idle_sleeps); P_ns(idle_entrytime); + P_ns(idle_waketime); + P_ns(idle_exittime); P_ns(idle_sleeptime); P(last_jiffies); P(next_jiffies); -- cgit v1.2.3 From 83e96c604e781098a2f61b8a294919bcf3abfab4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 1 Feb 2008 17:45:14 +0100 Subject: futex: Remove warn on in return fixup path The WARN_ON() in the fixup return path of futex_lock_pi() can trigger with false positives. The following scenario happens: t1 holds the futex and t2 and t3 are blocked on the kernel side rt_mutex. t1 releases the futex (and the rt_mutex) and assigned t2 to be the next owner of the futex. t2 is interrupted and returns w/o acquiring the rt_mutex, before t1 can release the rtmutex. t1 releases the rtmutex and t3 becomes the pending owner of the rtmutex. t2 notices that it is the designated owner (user space variable) and fails to acquire the rt_mutex via trylock, because it is not allowed to steal the rt_mutex from t3. Now it looks at the rt_mutex pending owner (t3) and assigns the futex and the pi_state to it. During the fixup t4 steals the rtmutex from t3. t2 returns from the fixup and the owner of the rt_mutex has changed from t3 to t4. There is no need to do another round of fixups from t2. The important part (t2 is not returning as the user space visible owner) is done. The further fixups are done, before either t3 or t4 return to user space. For the user space it is not relevant which task (t3 or t4) is the real owner, as long as those are both in the kernel, which is guaranteed by the serialization of the hash bucket lock. Both tasks (which ever returns first to userspace - t4 because it locked the rt_mutex or t3 due to a signal) are going through the lock_futex_pi() return path where the ownership is fixed before the return to user space. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- kernel/futex.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'kernel') diff --git a/kernel/futex.c b/kernel/futex.c index 0edd314798c..0006d64c448 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -1537,9 +1537,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared, owner = rt_mutex_owner(&q.pi_state->pi_mutex); res = fixup_pi_state_owner(uaddr, &q, owner); - WARN_ON(rt_mutex_owner(&q.pi_state->pi_mutex) != - owner); - /* propagate -EFAULT, if the fixup failed */ if (res) ret = res; -- cgit v1.2.3 From cd689985cf49f6ff5c8eddc48d98b9d581d9475d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 1 Feb 2008 17:45:14 +0100 Subject: futex: Add bitset conditional wait/wakeup functionality To allow the implementation of optimized rw-locks in user space, glibc needs a possibility to select waiters for wakeup depending on a bitset mask. This requires two new futex OPs: FUTEX_WAIT_BITS and FUTEX_WAKE_BITS These OPs are basically the same as FUTEX_WAIT and FUTEX_WAKE plus an additional argument - a bitset. Further the FUTEX_WAIT_BITS OP is expecting an absolute timeout value instead of the relative one, which is used for the FUTEX_WAIT OP. FUTEX_WAIT_BITS calls into the kernel with a bitset. The bitset is stored in the futex_q structure, which is used to enqueue the waiter into the hashed futex waitqueue. FUTEX_WAKE_BITS also calls into the kernel with a bitset. The wakeup function logically ANDs the bitset with the bitset stored in each waiters futex_q structure. If the result is zero (i.e. none of the set bits in the bitsets is matching), then the waiter is not woken up. If the result is not zero (i.e. one of the set bits in the bitsets is matching), then the waiter is woken. The bitset provided by the caller must be non zero. In case the provided bitset is zero the kernel returns EINVAL. Internaly the new OPs are only extensions to the existing FUTEX_WAIT and FUTEX_WAKE functions. The existing OPs hand a bitset with all bits set into the futex_wait() and futex_wake() functions. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- kernel/futex.c | 37 ++++++++++++++++++++++++++++++------- kernel/futex_compat.c | 3 ++- 2 files changed, 32 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/futex.c b/kernel/futex.c index 0006d64c448..a6baaec44b8 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -109,6 +109,9 @@ struct futex_q { /* Optional priority inheritance state: */ struct futex_pi_state *pi_state; struct task_struct *task; + + /* Bitset for the optional bitmasked wakeup */ + u32 bitset; }; /* @@ -722,7 +725,7 @@ double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) * to this virtual address: */ static int futex_wake(u32 __user *uaddr, struct rw_semaphore *fshared, - int nr_wake) + int nr_wake, u32 bitset) { struct futex_hash_bucket *hb; struct futex_q *this, *next; @@ -730,6 +733,9 @@ static int futex_wake(u32 __user *uaddr, struct rw_semaphore *fshared, union futex_key key; int ret; + if (!bitset) + return -EINVAL; + futex_lock_mm(fshared); ret = get_futex_key(uaddr, fshared, &key); @@ -746,6 +752,11 @@ static int futex_wake(u32 __user *uaddr, struct rw_semaphore *fshared, ret = -EINVAL; break; } + + /* Check if one of the bits is set in both bitsets */ + if (!(this->bitset & bitset)) + continue; + wake_futex(this); if (++ret >= nr_wake) break; @@ -1156,7 +1167,7 @@ static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q, static long futex_wait_restart(struct restart_block *restart); static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared, - u32 val, ktime_t *abs_time) + u32 val, ktime_t *abs_time, u32 bitset) { struct task_struct *curr = current; DECLARE_WAITQUEUE(wait, curr); @@ -1167,7 +1178,11 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared, struct hrtimer_sleeper t; int rem = 0; + if (!bitset) + return -EINVAL; + q.pi_state = NULL; + q.bitset = bitset; retry: futex_lock_mm(fshared); @@ -1295,6 +1310,7 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared, restart->futex.uaddr = (u32 *)uaddr; restart->futex.val = val; restart->futex.time = abs_time->tv64; + restart->futex.bitset = bitset; restart->futex.flags = 0; if (fshared) @@ -1321,7 +1337,8 @@ static long futex_wait_restart(struct restart_block *restart) restart->fn = do_no_restart_syscall; if (restart->futex.flags & FLAGS_SHARED) fshared = ¤t->mm->mmap_sem; - return (long)futex_wait(uaddr, fshared, restart->futex.val, &t); + return (long)futex_wait(uaddr, fshared, restart->futex.val, &t, + restart->futex.bitset); } @@ -1942,7 +1959,8 @@ retry: * PI futexes happens in exit_pi_state(): */ if (!pi && (uval & FUTEX_WAITERS)) - futex_wake(uaddr, &curr->mm->mmap_sem, 1); + futex_wake(uaddr, &curr->mm->mmap_sem, 1, + FUTEX_BITSET_MATCH_ANY); } return 0; } @@ -2042,10 +2060,14 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, switch (cmd) { case FUTEX_WAIT: - ret = futex_wait(uaddr, fshared, val, timeout); + val3 = FUTEX_BITSET_MATCH_ANY; + case FUTEX_WAIT_BITSET: + ret = futex_wait(uaddr, fshared, val, timeout, val3); break; case FUTEX_WAKE: - ret = futex_wake(uaddr, fshared, val); + val3 = FUTEX_BITSET_MATCH_ANY; + case FUTEX_WAKE_BITSET: + ret = futex_wake(uaddr, fshared, val, val3); break; case FUTEX_FD: /* non-zero val means F_SETOWN(getpid()) & F_SETSIG(val) */ @@ -2085,7 +2107,8 @@ asmlinkage long sys_futex(u32 __user *uaddr, int op, u32 val, u32 val2 = 0; int cmd = op & FUTEX_CMD_MASK; - if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI)) { + if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI || + cmd == FUTEX_WAIT_BITSET)) { if (copy_from_user(&ts, utime, sizeof(ts)) != 0) return -EFAULT; if (!timespec_valid(&ts)) diff --git a/kernel/futex_compat.c b/kernel/futex_compat.c index 0a43def6fee..133d558db45 100644 --- a/kernel/futex_compat.c +++ b/kernel/futex_compat.c @@ -167,7 +167,8 @@ asmlinkage long compat_sys_futex(u32 __user *uaddr, int op, u32 val, int val2 = 0; int cmd = op & FUTEX_CMD_MASK; - if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI)) { + if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI || + cmd == FUTEX_WAIT_BITSET)) { if (get_compat_timespec(&ts, utime)) return -EFAULT; if (!timespec_valid(&ts)) -- cgit v1.2.3 From 0c11b9428f619ab377c92eff2f160a834a6585dd Mon Sep 17 00:00:00 2001 From: Al Viro Date: Thu, 10 Jan 2008 04:20:52 -0500 Subject: [PATCH] switch audit_get_loginuid() to task_struct * all callers pass something->audit_context Signed-off-by: Al Viro --- kernel/auditsc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/auditsc.c b/kernel/auditsc.c index bce9ecdb771..bd4e0a2443f 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -1804,8 +1804,9 @@ int audit_set_loginuid(struct task_struct *task, uid_t loginuid) * * Returns the context's loginuid or -1 if @ctx is NULL. */ -uid_t audit_get_loginuid(struct audit_context *ctx) +uid_t audit_get_loginuid(struct task_struct *task) { + struct audit_context *ctx = task->audit_context; return ctx ? ctx->loginuid : -1; } @@ -2273,7 +2274,7 @@ void audit_core_dumps(long signr) ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_ANOM_ABEND); audit_log_format(ab, "auid=%u uid=%u gid=%u", - audit_get_loginuid(current->audit_context), + audit_get_loginuid(current), current->uid, current->gid); selinux_get_task_sid(current, &sid); if (sid) { -- cgit v1.2.3 From bfef93a5d1fb5654fe2025276c55e202d10b5255 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Thu, 10 Jan 2008 04:53:18 -0500 Subject: [PATCH] get rid of loginuid races Keeping loginuid in audit_context is racy and results in messier code. Taken to task_struct, out of the way of ->audit_context changes. Signed-off-by: Al Viro --- kernel/auditsc.c | 56 +++++++++++++++----------------------------------------- 1 file changed, 15 insertions(+), 41 deletions(-) (limited to 'kernel') diff --git a/kernel/auditsc.c b/kernel/auditsc.c index bd4e0a2443f..c95173a194b 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -192,7 +192,6 @@ struct audit_context { enum audit_state state; unsigned int serial; /* serial number for record */ struct timespec ctime; /* time of syscall entry */ - uid_t loginuid; /* login uid (identity) */ int major; /* syscall number */ unsigned long argv[4]; /* syscall arguments */ int return_valid; /* return code is valid */ @@ -506,7 +505,7 @@ static int audit_filter_rules(struct task_struct *tsk, case AUDIT_LOGINUID: result = 0; if (ctx) - result = audit_comparator(ctx->loginuid, f->op, f->val); + result = audit_comparator(tsk->loginuid, f->op, f->val); break; case AUDIT_SUBJ_USER: case AUDIT_SUBJ_ROLE: @@ -783,11 +782,8 @@ static inline void audit_free_aux(struct audit_context *context) static inline void audit_zero_context(struct audit_context *context, enum audit_state state) { - uid_t loginuid = context->loginuid; - memset(context, 0, sizeof(*context)); context->state = state; - context->loginuid = loginuid; } static inline struct audit_context *audit_alloc_context(enum audit_state state) @@ -826,11 +822,6 @@ int audit_alloc(struct task_struct *tsk) return -ENOMEM; } - /* Preserve login uid */ - context->loginuid = -1; - if (current->audit_context) - context->loginuid = current->audit_context->loginuid; - tsk->audit_context = context; set_tsk_thread_flag(tsk, TIF_SYSCALL_AUDIT); return 0; @@ -1047,7 +1038,7 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts context->name_count, context->ppid, context->pid, - context->loginuid, + tsk->loginuid, context->uid, context->gid, context->euid, context->suid, context->fsuid, @@ -1779,39 +1770,22 @@ int audit_set_loginuid(struct task_struct *task, uid_t loginuid) { struct audit_context *context = task->audit_context; - if (context) { - /* Only log if audit is enabled */ - if (context->in_syscall) { - struct audit_buffer *ab; - - ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_LOGIN); - if (ab) { - audit_log_format(ab, "login pid=%d uid=%u " - "old auid=%u new auid=%u", - task->pid, task->uid, - context->loginuid, loginuid); - audit_log_end(ab); - } + if (context && context->in_syscall) { + struct audit_buffer *ab; + + ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_LOGIN); + if (ab) { + audit_log_format(ab, "login pid=%d uid=%u " + "old auid=%u new auid=%u", + task->pid, task->uid, + task->loginuid, loginuid); + audit_log_end(ab); } - context->loginuid = loginuid; } + task->loginuid = loginuid; return 0; } -/** - * audit_get_loginuid - get the loginuid for an audit_context - * @ctx: the audit_context - * - * Returns the context's loginuid or -1 if @ctx is NULL. - */ -uid_t audit_get_loginuid(struct task_struct *task) -{ - struct audit_context *ctx = task->audit_context; - return ctx ? ctx->loginuid : -1; -} - -EXPORT_SYMBOL(audit_get_loginuid); - /** * __audit_mq_open - record audit data for a POSIX MQ open * @oflag: open flag @@ -2217,8 +2191,8 @@ int __audit_signal_info(int sig, struct task_struct *t) if (audit_pid && t->tgid == audit_pid) { if (sig == SIGTERM || sig == SIGHUP || sig == SIGUSR1) { audit_sig_pid = tsk->pid; - if (ctx) - audit_sig_uid = ctx->loginuid; + if (tsk->loginuid != -1) + audit_sig_uid = tsk->loginuid; else audit_sig_uid = tsk->uid; selinux_get_task_sid(tsk, &audit_sig_sid); -- cgit v1.2.3 From f701b75ed5ffb6820efe530d1a3abcc6fc4678ad Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 7 Jan 2008 13:34:51 -0500 Subject: [AUDIT] return EINTR not ERESTART* The syscall exit code will change ERESTART* kernel internal return codes to EINTR if it does not restart the syscall. Since we collect the audit info before that point we should fix those in the audit log as well. Signed-off-by: Eric Paris --- kernel/auditsc.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/auditsc.c b/kernel/auditsc.c index c95173a194b..ce8c957201e 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -701,7 +701,24 @@ static inline struct audit_context *audit_get_context(struct task_struct *tsk, if (likely(!context)) return NULL; context->return_valid = return_valid; - context->return_code = return_code; + + /* + * we need to fix up the return code in the audit logs if the actual + * return codes are later going to be fixed up by the arch specific + * signal handlers + * + * This is actually a test for: + * (rc == ERESTARTSYS ) || (rc == ERESTARTNOINTR) || + * (rc == ERESTARTNOHAND) || (rc == ERESTART_RESTARTBLOCK) + * + * but is faster than a bunch of || + */ + if (unlikely(return_code <= -ERESTARTSYS) && + (return_code >= -ERESTART_RESTARTBLOCK) && + (return_code != -ENOIOCTLCMD)) + context->return_code = -EINTR; + else + context->return_code = return_code; if (context->in_syscall && !context->dummy && !context->auditable) { enum audit_state state; -- cgit v1.2.3 From c2a7780efe37d01bdb3facc85a94663e6d67d4a8 Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 7 Jan 2008 13:40:17 -0500 Subject: [AUDIT] collect uid, loginuid, and comm in OBJ_PID records Add uid, loginuid, and comm collection to OBJ_PID records. This just gives users a little more information about the task that received a signal. pid is rather meaningless after the fact, and even though comm isn't great we can't collect exe reasonably on this code path for performance reasons. Signed-off-by: Eric Paris --- kernel/auditsc.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/auditsc.c b/kernel/auditsc.c index ce8c957201e..a222e73fec7 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -176,7 +176,10 @@ struct audit_aux_data_fd_pair { struct audit_aux_data_pids { struct audit_aux_data d; pid_t target_pid[AUDIT_AUX_PIDS]; + uid_t target_auid[AUDIT_AUX_PIDS]; + uid_t target_uid[AUDIT_AUX_PIDS]; u32 target_sid[AUDIT_AUX_PIDS]; + char target_comm[AUDIT_AUX_PIDS][TASK_COMM_LEN]; int pid_count; }; @@ -214,7 +217,10 @@ struct audit_context { int arch; pid_t target_pid; + uid_t target_auid; + uid_t target_uid; u32 target_sid; + char target_comm[TASK_COMM_LEN]; struct audit_tree_refs *trees, *first_trees; int tree_count; @@ -930,7 +936,7 @@ static void audit_log_task_info(struct audit_buffer *ab, struct task_struct *tsk } static int audit_log_pid_context(struct audit_context *context, pid_t pid, - u32 sid) + uid_t auid, uid_t uid, u32 sid, char *comm) { struct audit_buffer *ab; char *s = NULL; @@ -941,11 +947,14 @@ static int audit_log_pid_context(struct audit_context *context, pid_t pid, if (!ab) return 1; + audit_log_format(ab, "opid=%d oauid=%d ouid=%d", pid, auid, uid); if (selinux_sid_to_string(sid, &s, &len)) { - audit_log_format(ab, "opid=%d obj=(none)", pid); + audit_log_format(ab, " obj=(none)"); rc = 1; } else - audit_log_format(ab, "opid=%d obj=%s", pid, s); + audit_log_format(ab, " obj=%s", s); + audit_log_format(ab, " ocomm="); + audit_log_untrustedstring(ab, comm); audit_log_end(ab); kfree(s); @@ -1176,13 +1185,17 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts for (i = 0; i < axs->pid_count; i++) if (audit_log_pid_context(context, axs->target_pid[i], - axs->target_sid[i])) + axs->target_auid[i], + axs->target_uid[i], + axs->target_sid[i], + axs->target_comm[i])) call_panic = 1; } if (context->target_pid && audit_log_pid_context(context, context->target_pid, - context->target_sid)) + context->target_auid, context->target_uid, + context->target_sid, context->target_comm)) call_panic = 1; if (context->pwd && context->pwdmnt) { @@ -2185,7 +2198,10 @@ void __audit_ptrace(struct task_struct *t) struct audit_context *context = current->audit_context; context->target_pid = t->pid; + context->target_auid = audit_get_loginuid(t); + context->target_uid = t->uid; selinux_get_task_sid(t, &context->target_sid); + memcpy(context->target_comm, t->comm, TASK_COMM_LEN); } /** @@ -2222,7 +2238,10 @@ int __audit_signal_info(int sig, struct task_struct *t) * in audit_context */ if (!ctx->target_pid) { ctx->target_pid = t->tgid; + ctx->target_auid = audit_get_loginuid(t); + ctx->target_uid = t->uid; selinux_get_task_sid(t, &ctx->target_sid); + memcpy(ctx->target_comm, t->comm, TASK_COMM_LEN); return 0; } @@ -2239,7 +2258,10 @@ int __audit_signal_info(int sig, struct task_struct *t) BUG_ON(axp->pid_count >= AUDIT_AUX_PIDS); axp->target_pid[axp->pid_count] = t->tgid; + axp->target_auid[axp->pid_count] = audit_get_loginuid(t); + axp->target_uid[axp->pid_count] = t->uid; selinux_get_task_sid(t, &axp->target_sid[axp->pid_count]); + memcpy(axp->target_comm[axp->pid_count], t->comm, TASK_COMM_LEN); axp->pid_count++; return 0; -- cgit v1.2.3 From 4746ec5b01ed07205a91e4f7ed9de9d70f371407 Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Tue, 8 Jan 2008 10:06:53 -0500 Subject: [AUDIT] add session id to audit messages In order to correlate audit records to an individual login add a session id. This is incremented every time a user logs in and is included in almost all messages which currently output the auid. The field is labeled ses= or oses= Signed-off-by: Eric Paris --- kernel/auditsc.c | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/auditsc.c b/kernel/auditsc.c index a222e73fec7..4e67abb0290 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -178,6 +178,7 @@ struct audit_aux_data_pids { pid_t target_pid[AUDIT_AUX_PIDS]; uid_t target_auid[AUDIT_AUX_PIDS]; uid_t target_uid[AUDIT_AUX_PIDS]; + unsigned int target_sessionid[AUDIT_AUX_PIDS]; u32 target_sid[AUDIT_AUX_PIDS]; char target_comm[AUDIT_AUX_PIDS][TASK_COMM_LEN]; int pid_count; @@ -219,6 +220,7 @@ struct audit_context { pid_t target_pid; uid_t target_auid; uid_t target_uid; + unsigned int target_sessionid; u32 target_sid; char target_comm[TASK_COMM_LEN]; @@ -936,7 +938,8 @@ static void audit_log_task_info(struct audit_buffer *ab, struct task_struct *tsk } static int audit_log_pid_context(struct audit_context *context, pid_t pid, - uid_t auid, uid_t uid, u32 sid, char *comm) + uid_t auid, uid_t uid, unsigned int sessionid, + u32 sid, char *comm) { struct audit_buffer *ab; char *s = NULL; @@ -947,7 +950,8 @@ static int audit_log_pid_context(struct audit_context *context, pid_t pid, if (!ab) return 1; - audit_log_format(ab, "opid=%d oauid=%d ouid=%d", pid, auid, uid); + audit_log_format(ab, "opid=%d oauid=%d ouid=%d oses=%d", pid, auid, + uid, sessionid); if (selinux_sid_to_string(sid, &s, &len)) { audit_log_format(ab, " obj=(none)"); rc = 1; @@ -1056,7 +1060,7 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts " a0=%lx a1=%lx a2=%lx a3=%lx items=%d" " ppid=%d pid=%d auid=%u uid=%u gid=%u" " euid=%u suid=%u fsuid=%u" - " egid=%u sgid=%u fsgid=%u tty=%s", + " egid=%u sgid=%u fsgid=%u tty=%s ses=%u", context->argv[0], context->argv[1], context->argv[2], @@ -1068,7 +1072,8 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts context->uid, context->gid, context->euid, context->suid, context->fsuid, - context->egid, context->sgid, context->fsgid, tty); + context->egid, context->sgid, context->fsgid, tty, + tsk->sessionid); mutex_unlock(&tty_mutex); @@ -1187,6 +1192,7 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts if (audit_log_pid_context(context, axs->target_pid[i], axs->target_auid[i], axs->target_uid[i], + axs->target_sessionid[i], axs->target_sid[i], axs->target_comm[i])) call_panic = 1; @@ -1195,6 +1201,7 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts if (context->target_pid && audit_log_pid_context(context, context->target_pid, context->target_auid, context->target_uid, + context->target_sessionid, context->target_sid, context->target_comm)) call_panic = 1; @@ -1787,6 +1794,9 @@ void auditsc_get_stamp(struct audit_context *ctx, ctx->auditable = 1; } +/* global counter which is incremented every time something logs in */ +static atomic_t session_id = ATOMIC_INIT(0); + /** * audit_set_loginuid - set a task's audit_context loginuid * @task: task whose audit context is being modified @@ -1798,6 +1808,7 @@ void auditsc_get_stamp(struct audit_context *ctx, */ int audit_set_loginuid(struct task_struct *task, uid_t loginuid) { + unsigned int sessionid = atomic_inc_return(&session_id); struct audit_context *context = task->audit_context; if (context && context->in_syscall) { @@ -1806,12 +1817,15 @@ int audit_set_loginuid(struct task_struct *task, uid_t loginuid) ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_LOGIN); if (ab) { audit_log_format(ab, "login pid=%d uid=%u " - "old auid=%u new auid=%u", + "old auid=%u new auid=%u" + " old ses=%u new ses=%u", task->pid, task->uid, - task->loginuid, loginuid); + task->loginuid, loginuid, + task->sessionid, sessionid); audit_log_end(ab); } } + task->sessionid = sessionid; task->loginuid = loginuid; return 0; } @@ -2200,6 +2214,7 @@ void __audit_ptrace(struct task_struct *t) context->target_pid = t->pid; context->target_auid = audit_get_loginuid(t); context->target_uid = t->uid; + context->target_sessionid = audit_get_sessionid(t); selinux_get_task_sid(t, &context->target_sid); memcpy(context->target_comm, t->comm, TASK_COMM_LEN); } @@ -2240,6 +2255,7 @@ int __audit_signal_info(int sig, struct task_struct *t) ctx->target_pid = t->tgid; ctx->target_auid = audit_get_loginuid(t); ctx->target_uid = t->uid; + ctx->target_sessionid = audit_get_sessionid(t); selinux_get_task_sid(t, &ctx->target_sid); memcpy(ctx->target_comm, t->comm, TASK_COMM_LEN); return 0; @@ -2260,6 +2276,7 @@ int __audit_signal_info(int sig, struct task_struct *t) axp->target_pid[axp->pid_count] = t->tgid; axp->target_auid[axp->pid_count] = audit_get_loginuid(t); axp->target_uid[axp->pid_count] = t->uid; + axp->target_sessionid[axp->pid_count] = audit_get_sessionid(t); selinux_get_task_sid(t, &axp->target_sid[axp->pid_count]); memcpy(axp->target_comm[axp->pid_count], t->comm, TASK_COMM_LEN); axp->pid_count++; @@ -2278,6 +2295,8 @@ void audit_core_dumps(long signr) { struct audit_buffer *ab; u32 sid; + uid_t auid = audit_get_loginuid(current); + unsigned int sessionid = audit_get_sessionid(current); if (!audit_enabled) return; @@ -2286,9 +2305,8 @@ void audit_core_dumps(long signr) return; ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_ANOM_ABEND); - audit_log_format(ab, "auid=%u uid=%u gid=%u", - audit_get_loginuid(current), - current->uid, current->gid); + audit_log_format(ab, "auid=%u uid=%u gid=%u ses=%u", + auid, current->uid, current->gid, sessionid); selinux_get_task_sid(current, &sid); if (sid) { char *ctx = NULL; -- cgit v1.2.3 From c0641f28dcbecb6dc34a4fd003a9947fcd080696 Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 7 Jan 2008 13:49:15 -0500 Subject: [AUDIT] Add End of Event record This patch adds an end of event record type. It will be sent by the kernel as the last record when a multi-record event is triggered. This will aid realtime analysis programs since they will now reliably know they have the last record to complete an event. The audit daemon filters this and will not write it to disk. Signed-off-by: Steve Grubb Signed-off-by: Eric Paris --- kernel/auditsc.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 4e67abb0290..6e5de767bad 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -1270,6 +1270,11 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts audit_log_end(ab); } + + /* Send end of event record to help user space know we are finished */ + ab = audit_log_start(context, GFP_KERNEL, AUDIT_EOE); + if (ab) + audit_log_end(ab); if (call_panic) audit_panic("error converting sid to string"); } -- cgit v1.2.3 From 6246ccab99093a562044596dd868213caa0b2b4c Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 7 Jan 2008 14:01:18 -0500 Subject: [AUDIT] do not panic on exclude messages in audit_log_pid_context() If we fail to get an ab in audit_log_pid_context this may be due to an exclude rule rather than a memory allocation failure. If it was due to a memory allocation failue we would have already paniced and no need to do it again. Signed-off-by: Eric Paris --- kernel/auditsc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 6e5de767bad..aaaca8a13bb 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -948,7 +948,7 @@ static int audit_log_pid_context(struct audit_context *context, pid_t pid, ab = audit_log_start(context, GFP_KERNEL, AUDIT_OBJ_PID); if (!ab) - return 1; + return rc; audit_log_format(ab, "opid=%d oauid=%d ouid=%d oses=%d", pid, auid, uid, sessionid); -- cgit v1.2.3 From e445deb593d67c8ed13bd357c780a93d78bc84cf Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 7 Jan 2008 14:19:15 -0500 Subject: [AUDIT] include audit type in audit message when using printk Currently audit drops the audit type when an audit message goes through printk instead of the audit deamon. This is a minor annoyance in that the audit type is no longer part of the message and the information the audit type conveys needs to be carried in, or derived from the message data. The attached patch includes the type number as part of the printk. Admittedly it isn't the type name that the audit deamon provides but I think this is better than dropping the type completely. Signed-pff-by: John Johansen Signed-off-by: Eric Paris --- kernel/audit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/audit.c b/kernel/audit.c index 801c946dd24..cf669828942 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -1438,7 +1438,8 @@ void audit_log_end(struct audit_buffer *ab) ab->skb = NULL; wake_up_interruptible(&kauditd_wait); } else { - printk(KERN_NOTICE "%s\n", ab->skb->data + NLMSG_SPACE(0)); + struct nlmsghdr *nlh = nlmsg_hdr(ab->skb); + printk(KERN_NOTICE "type=%d %s\n", nlh->nlmsg_type, ab->skb->data + NLMSG_SPACE(0)); } } audit_buffer_free(ab); -- cgit v1.2.3 From de6bbd1d30e5912620d25dd15e3f180ac7f9fcef Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 7 Jan 2008 14:31:58 -0500 Subject: [AUDIT] break large execve argument logging into smaller messages execve arguments can be quite large. There is no limit on the number of arguments and a 4G limit on the size of an argument. this patch prints those aruguments in bite sized pieces. a userspace size limitation of 8k was discovered so this keeps messages around 7.5k single arguments larger than 7.5k in length are split into multiple records and can be identified as aX[Y]= Signed-off-by: Eric Paris --- kernel/audit.c | 37 ++++++---- kernel/auditsc.c | 205 ++++++++++++++++++++++++++++++++++++++++++++----------- kernel/sysctl.c | 11 --- 3 files changed, 188 insertions(+), 65 deletions(-) (limited to 'kernel') diff --git a/kernel/audit.c b/kernel/audit.c index cf669828942..26ff925e13f 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -1349,6 +1349,21 @@ static void audit_log_n_string(struct audit_buffer *ab, size_t slen, skb_put(skb, slen + 2); /* don't include null terminator */ } +/** + * audit_string_contains_control - does a string need to be logged in hex + * @string - string to be checked + * @len - max length of the string to check + */ +int audit_string_contains_control(const char *string, size_t len) +{ + const unsigned char *p; + for (p = string; p < (const unsigned char *)string + len && *p; p++) { + if (*p == '"' || *p < 0x21 || *p > 0x7f) + return 1; + } + return 0; +} + /** * audit_log_n_untrustedstring - log a string that may contain random characters * @ab: audit_buffer @@ -1363,19 +1378,13 @@ static void audit_log_n_string(struct audit_buffer *ab, size_t slen, * The caller specifies the number of characters in the string to log, which may * or may not be the entire string. */ -const char *audit_log_n_untrustedstring(struct audit_buffer *ab, size_t len, - const char *string) +void audit_log_n_untrustedstring(struct audit_buffer *ab, size_t len, + const char *string) { - const unsigned char *p; - - for (p = string; p < (const unsigned char *)string + len && *p; p++) { - if (*p == '"' || *p < 0x21 || *p > 0x7f) { - audit_log_hex(ab, string, len); - return string + len + 1; - } - } - audit_log_n_string(ab, len, string); - return p + 1; + if (audit_string_contains_control(string, len)) + audit_log_hex(ab, string, len); + else + audit_log_n_string(ab, len, string); } /** @@ -1386,9 +1395,9 @@ const char *audit_log_n_untrustedstring(struct audit_buffer *ab, size_t len, * Same as audit_log_n_untrustedstring(), except that strlen is used to * determine string length. */ -const char *audit_log_untrustedstring(struct audit_buffer *ab, const char *string) +void audit_log_untrustedstring(struct audit_buffer *ab, const char *string) { - return audit_log_n_untrustedstring(ab, strlen(string), string); + audit_log_n_untrustedstring(ab, strlen(string), string); } /* This is a helper-function to print the escaped d_path */ diff --git a/kernel/auditsc.c b/kernel/auditsc.c index aaaca8a13bb..6e03322e155 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -78,6 +78,9 @@ extern struct list_head audit_filter_list[]; /* Indicates that audit should log the full pathname. */ #define AUDIT_NAME_FULL -1 +/* no execve audit message should be longer than this (userspace limits) */ +#define MAX_EXECVE_AUDIT_LEN 7500 + /* number of audit rules */ int audit_n_rules; @@ -965,55 +968,187 @@ static int audit_log_pid_context(struct audit_context *context, pid_t pid, return rc; } -static void audit_log_execve_info(struct audit_buffer *ab, - struct audit_aux_data_execve *axi) +/* + * to_send and len_sent accounting are very loose estimates. We aren't + * really worried about a hard cap to MAX_EXECVE_AUDIT_LEN so much as being + * within about 500 bytes (next page boundry) + * + * why snprintf? an int is up to 12 digits long. if we just assumed when + * logging that a[%d]= was going to be 16 characters long we would be wasting + * space in every audit message. In one 7500 byte message we can log up to + * about 1000 min size arguments. That comes down to about 50% waste of space + * if we didn't do the snprintf to find out how long arg_num_len was. + */ +static int audit_log_single_execve_arg(struct audit_context *context, + struct audit_buffer **ab, + int arg_num, + size_t *len_sent, + const char __user *p, + char *buf) { - int i; - long len, ret; - const char __user *p; - char *buf; + char arg_num_len_buf[12]; + const char __user *tmp_p = p; + /* how many digits are in arg_num? 3 is the length of a=\n */ + size_t arg_num_len = snprintf(arg_num_len_buf, 12, "%d", arg_num) + 3; + size_t len, len_left, to_send; + size_t max_execve_audit_len = MAX_EXECVE_AUDIT_LEN; + unsigned int i, has_cntl = 0, too_long = 0; + int ret; + + /* strnlen_user includes the null we don't want to send */ + len_left = len = strnlen_user(p, MAX_ARG_STRLEN) - 1; - if (axi->mm != current->mm) - return; /* execve failed, no additional info */ - - p = (const char __user *)axi->mm->arg_start; + /* + * We just created this mm, if we can't find the strings + * we just copied into it something is _very_ wrong. Similar + * for strings that are too long, we should not have created + * any. + */ + if (unlikely((len = -1) || len > MAX_ARG_STRLEN - 1)) { + WARN_ON(1); + send_sig(SIGKILL, current, 0); + } - for (i = 0; i < axi->argc; i++, p += len) { - len = strnlen_user(p, MAX_ARG_STRLEN); + /* walk the whole argument looking for non-ascii chars */ + do { + if (len_left > MAX_EXECVE_AUDIT_LEN) + to_send = MAX_EXECVE_AUDIT_LEN; + else + to_send = len_left; + ret = copy_from_user(buf, tmp_p, to_send); /* - * We just created this mm, if we can't find the strings - * we just copied into it something is _very_ wrong. Similar - * for strings that are too long, we should not have created - * any. + * There is no reason for this copy to be short. We just + * copied them here, and the mm hasn't been exposed to user- + * space yet. */ - if (!len || len > MAX_ARG_STRLEN) { + if (ret) { WARN_ON(1); send_sig(SIGKILL, current, 0); } - - buf = kmalloc(len, GFP_KERNEL); - if (!buf) { - audit_panic("out of memory for argv string\n"); + buf[to_send] = '\0'; + has_cntl = audit_string_contains_control(buf, to_send); + if (has_cntl) { + /* + * hex messages get logged as 2 bytes, so we can only + * send half as much in each message + */ + max_execve_audit_len = MAX_EXECVE_AUDIT_LEN / 2; break; } + len_left -= to_send; + tmp_p += to_send; + } while (len_left > 0); + + len_left = len; + + if (len > max_execve_audit_len) + too_long = 1; + + /* rewalk the argument actually logging the message */ + for (i = 0; len_left > 0; i++) { + int room_left; + + if (len_left > max_execve_audit_len) + to_send = max_execve_audit_len; + else + to_send = len_left; + + /* do we have space left to send this argument in this ab? */ + room_left = MAX_EXECVE_AUDIT_LEN - arg_num_len - *len_sent; + if (has_cntl) + room_left -= (to_send * 2); + else + room_left -= to_send; + if (room_left < 0) { + *len_sent = 0; + audit_log_end(*ab); + *ab = audit_log_start(context, GFP_KERNEL, AUDIT_EXECVE); + if (!*ab) + return 0; + } - ret = copy_from_user(buf, p, len); /* - * There is no reason for this copy to be short. We just - * copied them here, and the mm hasn't been exposed to user- - * space yet. + * first record needs to say how long the original string was + * so we can be sure nothing was lost. + */ + if ((i == 0) && (too_long)) + audit_log_format(*ab, "a%d_len=%ld ", arg_num, + has_cntl ? 2*len : len); + + /* + * normally arguments are small enough to fit and we already + * filled buf above when we checked for control characters + * so don't bother with another copy_from_user */ + if (len >= max_execve_audit_len) + ret = copy_from_user(buf, p, to_send); + else + ret = 0; if (ret) { WARN_ON(1); send_sig(SIGKILL, current, 0); } + buf[to_send] = '\0'; + + /* actually log it */ + audit_log_format(*ab, "a%d", arg_num); + if (too_long) + audit_log_format(*ab, "[%d]", i); + audit_log_format(*ab, "="); + if (has_cntl) + audit_log_hex(*ab, buf, to_send); + else + audit_log_format(*ab, "\"%s\"", buf); + audit_log_format(*ab, "\n"); + + p += to_send; + len_left -= to_send; + *len_sent += arg_num_len; + if (has_cntl) + *len_sent += to_send * 2; + else + *len_sent += to_send; + } + /* include the null we didn't log */ + return len + 1; +} + +static void audit_log_execve_info(struct audit_context *context, + struct audit_buffer **ab, + struct audit_aux_data_execve *axi) +{ + int i; + size_t len, len_sent = 0; + const char __user *p; + char *buf; - audit_log_format(ab, "a%d=", i); - audit_log_untrustedstring(ab, buf); - audit_log_format(ab, "\n"); + if (axi->mm != current->mm) + return; /* execve failed, no additional info */ + + p = (const char __user *)axi->mm->arg_start; - kfree(buf); + audit_log_format(*ab, "argc=%d ", axi->argc); + + /* + * we need some kernel buffer to hold the userspace args. Just + * allocate one big one rather than allocating one of the right size + * for every single argument inside audit_log_single_execve_arg() + * should be <8k allocation so should be pretty safe. + */ + buf = kmalloc(MAX_EXECVE_AUDIT_LEN + 1, GFP_KERNEL); + if (!buf) { + audit_panic("out of memory for argv string\n"); + return; } + + for (i = 0; i < axi->argc; i++) { + len = audit_log_single_execve_arg(context, ab, i, + &len_sent, p, buf); + if (len <= 0) + break; + p += len; + } + kfree(buf); } static void audit_log_exit(struct audit_context *context, struct task_struct *tsk) @@ -1157,7 +1292,7 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts case AUDIT_EXECVE: { struct audit_aux_data_execve *axi = (void *)aux; - audit_log_execve_info(ab, axi); + audit_log_execve_info(context, &ab, axi); break; } case AUDIT_SOCKETCALL: { @@ -2094,8 +2229,6 @@ int __audit_ipc_set_perm(unsigned long qbytes, uid_t uid, gid_t gid, mode_t mode return 0; } -int audit_argv_kb = 32; - int audit_bprm(struct linux_binprm *bprm) { struct audit_aux_data_execve *ax; @@ -2104,14 +2237,6 @@ int audit_bprm(struct linux_binprm *bprm) if (likely(!audit_enabled || !context || context->dummy)) return 0; - /* - * Even though the stack code doesn't limit the arg+env size any more, - * the audit code requires that _all_ arguments be logged in a single - * netlink skb. Hence cap it :-( - */ - if (bprm->argv_len > (audit_argv_kb << 10)) - return -E2BIG; - ax = kmalloc(sizeof(*ax), GFP_KERNEL); if (!ax) return -ENOMEM; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 357b68ba23e..7cb1ac3e6ff 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -81,7 +81,6 @@ extern int percpu_pagelist_fraction; extern int compat_log; extern int maps_protect; extern int sysctl_stat_interval; -extern int audit_argv_kb; extern int latencytop_enabled; /* Constants used for minimum and maximum */ @@ -390,16 +389,6 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = &proc_dointvec, }, -#ifdef CONFIG_AUDITSYSCALL - { - .ctl_name = CTL_UNNUMBERED, - .procname = "audit_argv_kb", - .data = &audit_argv_kb, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = &proc_dointvec, - }, -#endif { .ctl_name = KERN_CORE_PATTERN, .procname = "core_pattern", -- cgit v1.2.3 From 1a6b9f2317f18db768010252c957d99daf40678f Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 7 Jan 2008 17:09:31 -0500 Subject: [AUDIT] make audit=0 really stop audit messages Some audit messages (namely configuration changes) are still emitted even if the audit subsystem has been explicitly disabled. This patch turns those messages off as well. Signed-off-by: Eric Paris --- kernel/audit.c | 191 ++++++++++++++++----------------------------------- kernel/auditfilter.c | 54 +++++++++------ 2 files changed, 93 insertions(+), 152 deletions(-) (limited to 'kernel') diff --git a/kernel/audit.c b/kernel/audit.c index 26ff925e13f..7e29372da28 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -66,9 +66,9 @@ * (Initialization happens after skb_init is called.) */ static int audit_initialized; -/* 0 - no auditing - * 1 - auditing enabled - * 2 - auditing enabled and configuration is locked/unchangeable. */ +#define AUDIT_OFF 0 +#define AUDIT_ON 1 +#define AUDIT_LOCKED 2 int audit_enabled; /* Default state when kernel boots without any parameters. */ @@ -240,152 +240,90 @@ void audit_log_lost(const char *message) } } -static int audit_set_rate_limit(int limit, uid_t loginuid, u32 sid) +static int audit_log_config_change(char *function_name, int new, int old, + uid_t loginuid, u32 sid, int allow_changes) { - int res, rc = 0, old = audit_rate_limit; - - /* check if we are locked */ - if (audit_enabled == 2) - res = 0; - else - res = 1; + struct audit_buffer *ab; + int rc = 0; + ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE); + audit_log_format(ab, "%s=%d old=%d by auid=%u", function_name, new, + old, loginuid); if (sid) { char *ctx = NULL; u32 len; - if ((rc = selinux_sid_to_string(sid, &ctx, &len)) == 0) { - audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE, - "audit_rate_limit=%d old=%d by auid=%u" - " subj=%s res=%d", - limit, old, loginuid, ctx, res); + + rc = selinux_sid_to_string(sid, &ctx, &len); + if (rc) { + audit_log_format(ab, " sid=%u", sid); + allow_changes = 0; /* Something weird, deny request */ + } else { + audit_log_format(ab, " subj=%s", ctx); kfree(ctx); - } else - res = 0; /* Something weird, deny request */ + } } - audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE, - "audit_rate_limit=%d old=%d by auid=%u res=%d", - limit, old, loginuid, res); - - /* If we are allowed, make the change */ - if (res == 1) - audit_rate_limit = limit; - /* Not allowed, update reason */ - else if (rc == 0) - rc = -EPERM; + audit_log_format(ab, " res=%d", allow_changes); + audit_log_end(ab); return rc; } -static int audit_set_backlog_limit(int limit, uid_t loginuid, u32 sid) +static int audit_do_config_change(char *function_name, int *to_change, + int new, uid_t loginuid, u32 sid) { - int res, rc = 0, old = audit_backlog_limit; + int allow_changes, rc = 0, old = *to_change; /* check if we are locked */ - if (audit_enabled == 2) - res = 0; + if (audit_enabled == AUDIT_LOCKED) + allow_changes = 0; else - res = 1; + allow_changes = 1; - if (sid) { - char *ctx = NULL; - u32 len; - if ((rc = selinux_sid_to_string(sid, &ctx, &len)) == 0) { - audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE, - "audit_backlog_limit=%d old=%d by auid=%u" - " subj=%s res=%d", - limit, old, loginuid, ctx, res); - kfree(ctx); - } else - res = 0; /* Something weird, deny request */ + if (audit_enabled != AUDIT_OFF) { + rc = audit_log_config_change(function_name, new, old, + loginuid, sid, allow_changes); + if (rc) + allow_changes = 0; } - audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE, - "audit_backlog_limit=%d old=%d by auid=%u res=%d", - limit, old, loginuid, res); /* If we are allowed, make the change */ - if (res == 1) - audit_backlog_limit = limit; + if (allow_changes == 1) + *to_change = new; /* Not allowed, update reason */ else if (rc == 0) rc = -EPERM; return rc; } -static int audit_set_enabled(int state, uid_t loginuid, u32 sid) +static int audit_set_rate_limit(int limit, uid_t loginuid, u32 sid) { - int res, rc = 0, old = audit_enabled; - - if (state < 0 || state > 2) - return -EINVAL; + return audit_do_config_change("audit_rate_limit", &audit_rate_limit, + limit, loginuid, sid); +} - /* check if we are locked */ - if (audit_enabled == 2) - res = 0; - else - res = 1; +static int audit_set_backlog_limit(int limit, uid_t loginuid, u32 sid) +{ + return audit_do_config_change("audit_backlog_limit", &audit_backlog_limit, + limit, loginuid, sid); +} - if (sid) { - char *ctx = NULL; - u32 len; - if ((rc = selinux_sid_to_string(sid, &ctx, &len)) == 0) { - audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE, - "audit_enabled=%d old=%d by auid=%u" - " subj=%s res=%d", - state, old, loginuid, ctx, res); - kfree(ctx); - } else - res = 0; /* Something weird, deny request */ - } - audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE, - "audit_enabled=%d old=%d by auid=%u res=%d", - state, old, loginuid, res); +static int audit_set_enabled(int state, uid_t loginuid, u32 sid) +{ + if (state < AUDIT_OFF || state > AUDIT_LOCKED) + return -EINVAL; - /* If we are allowed, make the change */ - if (res == 1) - audit_enabled = state; - /* Not allowed, update reason */ - else if (rc == 0) - rc = -EPERM; - return rc; + return audit_do_config_change("audit_enabled", &audit_enabled, state, + loginuid, sid); } static int audit_set_failure(int state, uid_t loginuid, u32 sid) { - int res, rc = 0, old = audit_failure; - if (state != AUDIT_FAIL_SILENT && state != AUDIT_FAIL_PRINTK && state != AUDIT_FAIL_PANIC) return -EINVAL; - /* check if we are locked */ - if (audit_enabled == 2) - res = 0; - else - res = 1; - - if (sid) { - char *ctx = NULL; - u32 len; - if ((rc = selinux_sid_to_string(sid, &ctx, &len)) == 0) { - audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE, - "audit_failure=%d old=%d by auid=%u" - " subj=%s res=%d", - state, old, loginuid, ctx, res); - kfree(ctx); - } else - res = 0; /* Something weird, deny request */ - } - audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE, - "audit_failure=%d old=%d by auid=%u res=%d", - state, old, loginuid, res); - - /* If we are allowed, make the change */ - if (res == 1) - audit_failure = state; - /* Not allowed, update reason */ - else if (rc == 0) - rc = -EPERM; - return rc; + return audit_do_config_change("audit_failure", &audit_failure, state, + loginuid, sid); } static int kauditd_thread(void *dummy) @@ -634,23 +572,14 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) if (err < 0) return err; } if (status_get->mask & AUDIT_STATUS_PID) { - int old = audit_pid; - if (sid) { - if ((err = selinux_sid_to_string( - sid, &ctx, &len))) - return err; - else - audit_log(NULL, GFP_KERNEL, - AUDIT_CONFIG_CHANGE, - "audit_pid=%d old=%d by auid=%u subj=%s", - status_get->pid, old, - loginuid, ctx); - kfree(ctx); - } else - audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE, - "audit_pid=%d old=%d by auid=%u", - status_get->pid, old, loginuid); - audit_pid = status_get->pid; + int new_pid = status_get->pid; + + if (audit_enabled != AUDIT_OFF) + audit_log_config_change("audit_pid", new_pid, + audit_pid, loginuid, + sid, 1); + + audit_pid = new_pid; } if (status_get->mask & AUDIT_STATUS_RATE_LIMIT) err = audit_set_rate_limit(status_get->rate_limit, @@ -709,7 +638,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) case AUDIT_DEL: if (nlmsg_len(nlh) < sizeof(struct audit_rule)) return -EINVAL; - if (audit_enabled == 2) { + if (audit_enabled == AUDIT_LOCKED) { ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE); if (ab) { @@ -743,7 +672,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) case AUDIT_DEL_RULE: if (nlmsg_len(nlh) < sizeof(struct audit_rule_data)) return -EINVAL; - if (audit_enabled == 2) { + if (audit_enabled == AUDIT_LOCKED) { ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE); if (ab) { diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index 5d96f2cc7be..6f19fd477aa 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c @@ -95,6 +95,8 @@ extern struct inotify_handle *audit_ih; /* Inotify events we care about. */ #define AUDIT_IN_WATCH IN_MOVE|IN_CREATE|IN_DELETE|IN_DELETE_SELF|IN_MOVE_SELF +extern int audit_enabled; + void audit_free_parent(struct inotify_watch *i_watch) { struct audit_parent *parent; @@ -974,7 +976,6 @@ static void audit_update_watch(struct audit_parent *parent, struct audit_watch *owatch, *nwatch, *nextw; struct audit_krule *r, *nextr; struct audit_entry *oentry, *nentry; - struct audit_buffer *ab; mutex_lock(&audit_filter_mutex); list_for_each_entry_safe(owatch, nextw, &parent->watches, wlist) { @@ -1014,13 +1015,18 @@ static void audit_update_watch(struct audit_parent *parent, call_rcu(&oentry->rcu, audit_free_rule_rcu); } - ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE); - audit_log_format(ab, "op=updated rules specifying path="); - audit_log_untrustedstring(ab, owatch->path); - audit_log_format(ab, " with dev=%u ino=%lu\n", dev, ino); - audit_log_format(ab, " list=%d res=1", r->listnr); - audit_log_end(ab); - + if (audit_enabled) { + struct audit_buffer *ab; + ab = audit_log_start(NULL, GFP_KERNEL, + AUDIT_CONFIG_CHANGE); + audit_log_format(ab, + "op=updated rules specifying path="); + audit_log_untrustedstring(ab, owatch->path); + audit_log_format(ab, " with dev=%u ino=%lu\n", + dev, ino); + audit_log_format(ab, " list=%d res=1", r->listnr); + audit_log_end(ab); + } audit_remove_watch(owatch); goto add_watch_to_parent; /* event applies to a single watch */ } @@ -1039,25 +1045,28 @@ static void audit_remove_parent_watches(struct audit_parent *parent) struct audit_watch *w, *nextw; struct audit_krule *r, *nextr; struct audit_entry *e; - struct audit_buffer *ab; mutex_lock(&audit_filter_mutex); parent->flags |= AUDIT_PARENT_INVALID; list_for_each_entry_safe(w, nextw, &parent->watches, wlist) { list_for_each_entry_safe(r, nextr, &w->rules, rlist) { e = container_of(r, struct audit_entry, rule); - - ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE); - audit_log_format(ab, "op=remove rule path="); - audit_log_untrustedstring(ab, w->path); - if (r->filterkey) { - audit_log_format(ab, " key="); - audit_log_untrustedstring(ab, r->filterkey); - } else - audit_log_format(ab, " key=(null)"); - audit_log_format(ab, " list=%d res=1", r->listnr); - audit_log_end(ab); - + if (audit_enabled) { + struct audit_buffer *ab; + ab = audit_log_start(NULL, GFP_KERNEL, + AUDIT_CONFIG_CHANGE); + audit_log_format(ab, "op=remove rule path="); + audit_log_untrustedstring(ab, w->path); + if (r->filterkey) { + audit_log_format(ab, " key="); + audit_log_untrustedstring(ab, + r->filterkey); + } else + audit_log_format(ab, " key=(null)"); + audit_log_format(ab, " list=%d res=1", + r->listnr); + audit_log_end(ab); + } list_del(&r->rlist); list_del_rcu(&e->list); call_rcu(&e->rcu, audit_free_rule_rcu); @@ -1495,6 +1504,9 @@ static void audit_log_rule_change(uid_t loginuid, u32 sid, char *action, { struct audit_buffer *ab; + if (!audit_enabled) + return; + ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE); if (!ab) return; -- cgit v1.2.3 From 50397bd1e471391d27f64efad9271459c913de87 Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 7 Jan 2008 18:14:19 -0500 Subject: [AUDIT] clean up audit_receive_msg() generally clean up audit_receive_msg() don't free random memory if selinux_sid_to_string fails for some reason. Move generic auditing to a helper function Signed-off-by: Eric Paris --- kernel/audit.c | 169 ++++++++++++++++++++++----------------------------------- 1 file changed, 66 insertions(+), 103 deletions(-) (limited to 'kernel') diff --git a/kernel/audit.c b/kernel/audit.c index 7e29372da28..549b2f55b64 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -152,8 +152,10 @@ struct audit_buffer { static void audit_set_pid(struct audit_buffer *ab, pid_t pid) { - struct nlmsghdr *nlh = nlmsg_hdr(ab->skb); - nlh->nlmsg_pid = pid; + if (ab) { + struct nlmsghdr *nlh = nlmsg_hdr(ab->skb); + nlh->nlmsg_pid = pid; + } } void audit_panic(const char *message) @@ -511,6 +513,33 @@ static int audit_netlink_ok(struct sk_buff *skb, u16 msg_type) return err; } +static int audit_log_common_recv_msg(struct audit_buffer **ab, u16 msg_type, + u32 pid, u32 uid, uid_t auid, u32 sid) +{ + int rc = 0; + char *ctx = NULL; + u32 len; + + if (!audit_enabled) { + *ab = NULL; + return rc; + } + + *ab = audit_log_start(NULL, GFP_KERNEL, msg_type); + audit_log_format(*ab, "user pid=%d uid=%u auid=%u", + pid, uid, auid); + if (sid) { + rc = selinux_sid_to_string(sid, &ctx, &len); + if (rc) + audit_log_format(*ab, " ssid=%u", sid); + else + audit_log_format(*ab, " subj=%s", ctx); + kfree(ctx); + } + + return rc; +} + static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) { u32 uid, pid, seq, sid; @@ -521,7 +550,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) u16 msg_type = nlh->nlmsg_type; uid_t loginuid; /* loginuid of sender */ struct audit_sig_info *sig_data; - char *ctx; + char *ctx = NULL; u32 len; err = audit_netlink_ok(skb, msg_type); @@ -602,36 +631,22 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) if (err) break; } - ab = audit_log_start(NULL, GFP_KERNEL, msg_type); - if (ab) { - audit_log_format(ab, - "user pid=%d uid=%u auid=%u", - pid, uid, loginuid); - if (sid) { - if (selinux_sid_to_string( - sid, &ctx, &len)) { - audit_log_format(ab, - " ssid=%u", sid); - /* Maybe call audit_panic? */ - } else - audit_log_format(ab, - " subj=%s", ctx); - kfree(ctx); - } - if (msg_type != AUDIT_USER_TTY) - audit_log_format(ab, " msg='%.1024s'", - (char *)data); - else { - int size; - - audit_log_format(ab, " msg="); - size = nlmsg_len(nlh); - audit_log_n_untrustedstring(ab, size, - data); - } - audit_set_pid(ab, pid); - audit_log_end(ab); + audit_log_common_recv_msg(&ab, msg_type, pid, uid, + loginuid, sid); + + if (msg_type != AUDIT_USER_TTY) + audit_log_format(ab, " msg='%.1024s'", + (char *)data); + else { + int size; + + audit_log_format(ab, " msg="); + size = nlmsg_len(nlh); + audit_log_n_untrustedstring(ab, size, + data); } + audit_set_pid(ab, pid); + audit_log_end(ab); } break; case AUDIT_ADD: @@ -639,27 +654,12 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) if (nlmsg_len(nlh) < sizeof(struct audit_rule)) return -EINVAL; if (audit_enabled == AUDIT_LOCKED) { - ab = audit_log_start(NULL, GFP_KERNEL, - AUDIT_CONFIG_CHANGE); - if (ab) { - audit_log_format(ab, - "pid=%d uid=%u auid=%u", - pid, uid, loginuid); - if (sid) { - if (selinux_sid_to_string( - sid, &ctx, &len)) { - audit_log_format(ab, - " ssid=%u", sid); - /* Maybe call audit_panic? */ - } else - audit_log_format(ab, - " subj=%s", ctx); - kfree(ctx); - } - audit_log_format(ab, " audit_enabled=%d res=0", - audit_enabled); - audit_log_end(ab); - } + audit_log_common_recv_msg(&ab, AUDIT_CONFIG_CHANGE, pid, + uid, loginuid, sid); + + audit_log_format(ab, " audit_enabled=%d res=0", + audit_enabled); + audit_log_end(ab); return -EPERM; } /* fallthrough */ @@ -673,27 +673,12 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) if (nlmsg_len(nlh) < sizeof(struct audit_rule_data)) return -EINVAL; if (audit_enabled == AUDIT_LOCKED) { - ab = audit_log_start(NULL, GFP_KERNEL, - AUDIT_CONFIG_CHANGE); - if (ab) { - audit_log_format(ab, - "pid=%d uid=%u auid=%u", - pid, uid, loginuid); - if (sid) { - if (selinux_sid_to_string( - sid, &ctx, &len)) { - audit_log_format(ab, - " ssid=%u", sid); - /* Maybe call audit_panic? */ - } else - audit_log_format(ab, - " subj=%s", ctx); - kfree(ctx); - } - audit_log_format(ab, " audit_enabled=%d res=0", - audit_enabled); - audit_log_end(ab); - } + audit_log_common_recv_msg(&ab, AUDIT_CONFIG_CHANGE, pid, + uid, loginuid, sid); + + audit_log_format(ab, " audit_enabled=%d res=0", + audit_enabled); + audit_log_end(ab); return -EPERM; } /* fallthrough */ @@ -704,19 +689,10 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) break; case AUDIT_TRIM: audit_trim_trees(); - ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE); - if (!ab) - break; - audit_log_format(ab, "auid=%u", loginuid); - if (sid) { - u32 len; - ctx = NULL; - if (selinux_sid_to_string(sid, &ctx, &len)) - audit_log_format(ab, " ssid=%u", sid); - else - audit_log_format(ab, " subj=%s", ctx); - kfree(ctx); - } + + audit_log_common_recv_msg(&ab, AUDIT_CONFIG_CHANGE, pid, + uid, loginuid, sid); + audit_log_format(ab, " op=trim res=1"); audit_log_end(ab); break; @@ -746,22 +722,9 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) /* OK, here comes... */ err = audit_tag_tree(old, new); - ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE); - if (!ab) { - kfree(old); - kfree(new); - break; - } - audit_log_format(ab, "auid=%u", loginuid); - if (sid) { - u32 len; - ctx = NULL; - if (selinux_sid_to_string(sid, &ctx, &len)) - audit_log_format(ab, " ssid=%u", sid); - else - audit_log_format(ab, " subj=%s", ctx); - kfree(ctx); - } + audit_log_common_recv_msg(&ab, AUDIT_CONFIG_CHANGE, pid, + uid, loginuid, sid); + audit_log_format(ab, " op=make_equiv old="); audit_log_untrustedstring(ab, old); audit_log_format(ab, " new="); -- cgit v1.2.3 From b593d384efcff7bdf6beb1bc1bc69927977aee26 Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Tue, 8 Jan 2008 17:38:31 -0500 Subject: [AUDIT] create context if auditing was ever enabled Disabling audit at runtime by auditctl doesn't mean that we can stop allocating contexts for new processes; we don't want to miss them when that sucker is reenabled. (based on work from Al Viro in the RHEL kernel series) Signed-off-by: Eric Paris --- kernel/audit.c | 16 +++++++++++++--- kernel/auditsc.c | 3 ++- 2 files changed, 15 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/audit.c b/kernel/audit.c index 549b2f55b64..1242021c7a6 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -70,6 +70,7 @@ static int audit_initialized; #define AUDIT_ON 1 #define AUDIT_LOCKED 2 int audit_enabled; +int audit_ever_enabled; /* Default state when kernel boots without any parameters. */ static int audit_default; @@ -310,11 +311,17 @@ static int audit_set_backlog_limit(int limit, uid_t loginuid, u32 sid) static int audit_set_enabled(int state, uid_t loginuid, u32 sid) { + int rc; if (state < AUDIT_OFF || state > AUDIT_LOCKED) return -EINVAL; - return audit_do_config_change("audit_enabled", &audit_enabled, state, - loginuid, sid); + rc = audit_do_config_change("audit_enabled", &audit_enabled, state, + loginuid, sid); + + if (!rc) + audit_ever_enabled |= !!state; + + return rc; } static int audit_set_failure(int state, uid_t loginuid, u32 sid) @@ -857,6 +864,7 @@ static int __init audit_init(void) skb_queue_head_init(&audit_skb_queue); audit_initialized = 1; audit_enabled = audit_default; + audit_ever_enabled |= !!audit_default; /* Register the callback with selinux. This callback will be invoked * when a new policy is loaded. */ @@ -884,8 +892,10 @@ static int __init audit_enable(char *str) printk(KERN_INFO "audit: %s%s\n", audit_default ? "enabled" : "disabled", audit_initialized ? "" : " (after initialization)"); - if (audit_initialized) + if (audit_initialized) { audit_enabled = audit_default; + audit_ever_enabled |= !!audit_default; + } return 1; } diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 6e03322e155..1c06ecf38d7 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -70,6 +70,7 @@ #include "audit.h" extern struct list_head audit_filter_list[]; +extern int audit_ever_enabled; /* AUDIT_NAMES is the number of slots we reserve in the audit_context * for saving names from getname(). */ @@ -838,7 +839,7 @@ int audit_alloc(struct task_struct *tsk) struct audit_context *context; enum audit_state state; - if (likely(!audit_enabled)) + if (likely(!audit_ever_enabled)) return 0; /* Return if not auditing. */ state = audit_filter_task(tsk); -- cgit v1.2.3 From ef00be0554f1af9f2b685e0e3bb9e2ec0181937e Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Thu, 10 Jan 2008 11:02:39 -0800 Subject: [patch 1/2] kernel/audit.c: warning fix kernel/audit.c: In function 'audit_log_start': kernel/audit.c:1133: warning: 'serial' may be used uninitialized in this function Cc: Al Viro Signed-off-by: Andrew Morton --- kernel/audit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/audit.c b/kernel/audit.c index 1242021c7a6..b617f69fd4e 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -1032,7 +1032,7 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask, { struct audit_buffer *ab = NULL; struct timespec t; - unsigned int serial; + unsigned int uninitialized_var(serial); int reserve; unsigned long timeout_start = jiffies; -- cgit v1.2.3 From 148b38dc9309044c8656aa36d5fd86069e2ea7cc Mon Sep 17 00:00:00 2001 From: Richard Knutsson Date: Thu, 10 Jan 2008 11:02:40 -0800 Subject: [patch 2/2] audit: complement va_copy with va_end() Complement va_copy() with va_end(). Signed-off-by: Richard Knutsson Cc: Al Viro Signed-off-by: Andrew Morton --- kernel/audit.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/audit.c b/kernel/audit.c index b617f69fd4e..9e3e457ddfb 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -1151,6 +1151,7 @@ static void audit_log_vformat(struct audit_buffer *ab, const char *fmt, goto out; len = vsnprintf(skb_tail_pointer(skb), avail, fmt, args2); } + va_end(args2); if (len > 0) skb_put(skb, len); out: -- cgit v1.2.3 From 320f1b1ed28c601cc152053a2f428a126cb608bc Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Wed, 23 Jan 2008 22:55:05 -0500 Subject: [AUDIT] ratelimit printk messages audit some printk messages from the audit system can become excessive. This patch ratelimits those messages. It was found that messages, such as the audit backlog lost printk message could flood the logs to the point that a machine could take an nmi watchdog hit or otherwise become unresponsive. Signed-off-by: Eric Paris --- kernel/audit.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/audit.c b/kernel/audit.c index 9e3e457ddfb..c8555b18021 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -166,7 +166,8 @@ void audit_panic(const char *message) case AUDIT_FAIL_SILENT: break; case AUDIT_FAIL_PRINTK: - printk(KERN_ERR "audit: %s\n", message); + if (printk_ratelimit()) + printk(KERN_ERR "audit: %s\n", message); break; case AUDIT_FAIL_PANIC: panic("audit: %s\n", message); @@ -234,11 +235,13 @@ void audit_log_lost(const char *message) } if (print) { - printk(KERN_WARNING - "audit: audit_lost=%d audit_rate_limit=%d audit_backlog_limit=%d\n", - atomic_read(&audit_lost), - audit_rate_limit, - audit_backlog_limit); + if (printk_ratelimit()) + printk(KERN_WARNING + "audit: audit_lost=%d audit_rate_limit=%d " + "audit_backlog_limit=%d\n", + atomic_read(&audit_lost), + audit_rate_limit, + audit_backlog_limit); audit_panic(message); } } @@ -352,7 +355,11 @@ static int kauditd_thread(void *dummy) audit_pid = 0; } } else { - printk(KERN_NOTICE "%s\n", skb->data + NLMSG_SPACE(0)); + if (printk_ratelimit()) + printk(KERN_NOTICE "%s\n", skb->data + + NLMSG_SPACE(0)); + else + audit_log_lost("printk limit exceeded\n"); kfree_skb(skb); } } else { @@ -1066,7 +1073,7 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask, remove_wait_queue(&audit_backlog_wait, &wait); continue; } - if (audit_rate_check()) + if (audit_rate_check() && printk_ratelimit()) printk(KERN_WARNING "audit: audit_backlog=%d > " "audit_backlog_limit=%d\n", @@ -1349,9 +1356,11 @@ void audit_log_end(struct audit_buffer *ab) skb_queue_tail(&audit_skb_queue, ab->skb); ab->skb = NULL; wake_up_interruptible(&kauditd_wait); - } else { + } else if (printk_ratelimit()) { struct nlmsghdr *nlh = nlmsg_hdr(ab->skb); printk(KERN_NOTICE "type=%d %s\n", nlh->nlmsg_type, ab->skb->data + NLMSG_SPACE(0)); + } else { + audit_log_lost("printk limit exceeded\n"); } } audit_buffer_free(ab); -- cgit v1.2.3 From af508b34d27e3341287d89e0eae6752fdb1b873f Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 26 Oct 2007 00:59:31 +0200 Subject: Hibernation: Introduce SNAPSHOT_GET_IMAGE_SIZE ioctl Add a new ioctl, SNAPSHOT_GET_IMAGE_SIZE, returning the size of the (just created) hibernation image, to the hibernation userland interface. This ioctl is necessary so that the userland utilities using the interface need not access the hibernation image header, owned by the kernel, in order to obtain the size of the image. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/power.h | 4 +++- kernel/power/snapshot.c | 7 ++++++- kernel/power/user.c | 18 ++++++++++++++---- 3 files changed, 23 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/power/power.h b/kernel/power/power.h index 2093c3a9a99..23c17031ed2 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -128,6 +128,7 @@ struct snapshot_handle { #define data_of(handle) ((handle).buffer + (handle).buf_offset) extern unsigned int snapshot_additional_pages(struct zone *zone); +extern unsigned long snapshot_get_image_size(void); extern int snapshot_read_next(struct snapshot_handle *handle, size_t count); extern int snapshot_write_next(struct snapshot_handle *handle, size_t count); extern void snapshot_write_finalize(struct snapshot_handle *handle); @@ -158,7 +159,8 @@ struct resume_swap_area { #define SNAPSHOT_PMOPS _IOW(SNAPSHOT_IOC_MAGIC, 12, unsigned int) #define SNAPSHOT_SET_SWAP_AREA _IOW(SNAPSHOT_IOC_MAGIC, 13, \ struct resume_swap_area) -#define SNAPSHOT_IOC_MAXNR 13 +#define SNAPSHOT_GET_IMAGE_SIZE _IOR(SNAPSHOT_IOC_MAGIC, 14, loff_t) +#define SNAPSHOT_IOC_MAXNR 14 #define PMOPS_PREPARE 1 #define PMOPS_ENTER 2 diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index 78039b477d2..c5ce0f34a5d 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -1264,12 +1264,17 @@ static char *check_image_kernel(struct swsusp_info *info) } #endif /* CONFIG_ARCH_HIBERNATION_HEADER */ +unsigned long snapshot_get_image_size(void) +{ + return nr_copy_pages + nr_meta_pages + 1; +} + static int init_header(struct swsusp_info *info) { memset(info, 0, sizeof(struct swsusp_info)); info->num_physpages = num_physpages; info->image_pages = nr_copy_pages; - info->pages = nr_copy_pages + nr_meta_pages + 1; + info->pages = snapshot_get_image_size(); info->size = info->pages; info->size <<= PAGE_SHIFT; return init_header_complete(info); diff --git a/kernel/power/user.c b/kernel/power/user.c index 5bd321bcbb7..88aac26e598 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c @@ -133,7 +133,7 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, { int error = 0; struct snapshot_data *data; - loff_t avail; + loff_t size; sector_t offset; if (_IOC_TYPE(cmd) != SNAPSHOT_IOC_MAGIC) @@ -210,10 +210,20 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, image_size = arg; break; + case SNAPSHOT_GET_IMAGE_SIZE: + if (!data->ready) { + error = -ENODATA; + break; + } + size = snapshot_get_image_size(); + size <<= PAGE_SHIFT; + error = put_user(size, (loff_t __user *)arg); + break; + case SNAPSHOT_AVAIL_SWAP: - avail = count_swap_pages(data->swap, 1); - avail <<= PAGE_SHIFT; - error = put_user(avail, (loff_t __user *)arg); + size = count_swap_pages(data->swap, 1); + size <<= PAGE_SHIFT; + error = put_user(size, (loff_t __user *)arg); break; case SNAPSHOT_GET_SWAP_PAGE: -- cgit v1.2.3 From eb57c1cf059630454b40fb8bb124e3f318d241f8 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 26 Oct 2007 01:01:10 +0200 Subject: Hibernation: Rework platform support ioctls (rev. 2) Modify the hibernation userland interface by adding two new ioctls to it, SNAPSHOT_PLATFORM_SUPPORT and SNAPSHOT_POWER_OFF, that can be used, respectively, to switch the hibernation platform support on/off and to make the kernel transition the system to the hibernation state (eg. ACPI S4) using the platform (eg. ACPI) driver. These ioctls are intended to replace the misdesigned SNAPSHOT_PMOPS ioctl, which from now is regarded as obsolete and will be removed in the future. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/power.h | 9 +++------ kernel/power/user.c | 39 +++++++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/power/power.h b/kernel/power/power.h index 23c17031ed2..6ca85fd4975 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -156,15 +156,12 @@ struct resume_swap_area { #define SNAPSHOT_FREE_SWAP_PAGES _IO(SNAPSHOT_IOC_MAGIC, 9) #define SNAPSHOT_SET_SWAP_FILE _IOW(SNAPSHOT_IOC_MAGIC, 10, unsigned int) #define SNAPSHOT_S2RAM _IO(SNAPSHOT_IOC_MAGIC, 11) -#define SNAPSHOT_PMOPS _IOW(SNAPSHOT_IOC_MAGIC, 12, unsigned int) #define SNAPSHOT_SET_SWAP_AREA _IOW(SNAPSHOT_IOC_MAGIC, 13, \ struct resume_swap_area) #define SNAPSHOT_GET_IMAGE_SIZE _IOR(SNAPSHOT_IOC_MAGIC, 14, loff_t) -#define SNAPSHOT_IOC_MAXNR 14 - -#define PMOPS_PREPARE 1 -#define PMOPS_ENTER 2 -#define PMOPS_FINISH 3 +#define SNAPSHOT_PLATFORM_SUPPORT _IO(SNAPSHOT_IOC_MAGIC, 15) +#define SNAPSHOT_POWER_OFF _IO(SNAPSHOT_IOC_MAGIC, 16) +#define SNAPSHOT_IOC_MAXNR 16 /* If unset, the snapshot device cannot be open. */ extern atomic_t snapshot_device_available; diff --git a/kernel/power/user.c b/kernel/power/user.c index 88aac26e598..de3fb433ae3 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c @@ -28,6 +28,18 @@ #include "power.h" +/* + * NOTE: The SNAPSHOT_PMOPS ioctl is obsolete and will be removed in the + * future. It is only preserved here for compatibility with existing userland + * utilities. + */ +#define SNAPSHOT_PMOPS _IOW(SNAPSHOT_IOC_MAGIC, 12, unsigned int) + +#define PMOPS_PREPARE 1 +#define PMOPS_ENTER 2 +#define PMOPS_FINISH 3 + + #define SNAPSHOT_MINOR 231 static struct snapshot_data { @@ -36,7 +48,7 @@ static struct snapshot_data { int mode; char frozen; char ready; - char platform_suspend; + char platform_support; } snapshot_state; atomic_t snapshot_device_available = ATOMIC_INIT(1); @@ -70,7 +82,7 @@ static int snapshot_open(struct inode *inode, struct file *filp) } data->frozen = 0; data->ready = 0; - data->platform_suspend = 0; + data->platform_support = 0; return 0; } @@ -183,7 +195,7 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, error = -EPERM; break; } - error = hibernation_snapshot(data->platform_suspend); + error = hibernation_snapshot(data->platform_support); if (!error) error = put_user(in_suspend, (unsigned int __user *)arg); if (!error) @@ -197,7 +209,7 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, error = -EPERM; break; } - error = hibernation_restore(data->platform_suspend); + error = hibernation_restore(data->platform_support); break; case SNAPSHOT_FREE: @@ -285,26 +297,33 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, mutex_unlock(&pm_mutex); break; - case SNAPSHOT_PMOPS: + case SNAPSHOT_PLATFORM_SUPPORT: + data->platform_support = !!arg; + break; + + case SNAPSHOT_POWER_OFF: + if (data->platform_support) + error = hibernation_platform_enter(); + break; + + case SNAPSHOT_PMOPS: /* This ioctl is deprecated */ error = -EINVAL; switch (arg) { case PMOPS_PREPARE: - data->platform_suspend = 1; + data->platform_support = 1; error = 0; break; case PMOPS_ENTER: - if (data->platform_suspend) + if (data->platform_support) error = hibernation_platform_enter(); - break; case PMOPS_FINISH: - if (data->platform_suspend) + if (data->platform_support) error = 0; - break; default: -- cgit v1.2.3 From 96f737490cfc368fdafe49769f52fc8460f9349f Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 26 Oct 2007 01:02:15 +0200 Subject: Hibernation: Mark SNAPSHOT_SET_SWAP_FILE ioctl as deprecated (rev. 2) Mark the SNAPSHOT_SET_SWAP_FILE ioctl belonging to the hibernation userland interface as deprecated. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/power.h | 1 - kernel/power/user.c | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/power/power.h b/kernel/power/power.h index 6ca85fd4975..8837ea334e3 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -154,7 +154,6 @@ struct resume_swap_area { #define SNAPSHOT_AVAIL_SWAP _IOR(SNAPSHOT_IOC_MAGIC, 7, void *) #define SNAPSHOT_GET_SWAP_PAGE _IOR(SNAPSHOT_IOC_MAGIC, 8, void *) #define SNAPSHOT_FREE_SWAP_PAGES _IO(SNAPSHOT_IOC_MAGIC, 9) -#define SNAPSHOT_SET_SWAP_FILE _IOW(SNAPSHOT_IOC_MAGIC, 10, unsigned int) #define SNAPSHOT_S2RAM _IO(SNAPSHOT_IOC_MAGIC, 11) #define SNAPSHOT_SET_SWAP_AREA _IOW(SNAPSHOT_IOC_MAGIC, 13, \ struct resume_swap_area) diff --git a/kernel/power/user.c b/kernel/power/user.c index de3fb433ae3..5e866e07855 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c @@ -29,10 +29,11 @@ #include "power.h" /* - * NOTE: The SNAPSHOT_PMOPS ioctl is obsolete and will be removed in the - * future. It is only preserved here for compatibility with existing userland - * utilities. + * NOTE: The SNAPSHOT_SET_SWAP_FILE and SNAPSHOT_PMOPS ioctls are obsolete and + * will be removed in the future. They are only preserved here for + * compatibility with existing userland utilities. */ +#define SNAPSHOT_SET_SWAP_FILE _IOW(SNAPSHOT_IOC_MAGIC, 10, unsigned int) #define SNAPSHOT_PMOPS _IOW(SNAPSHOT_IOC_MAGIC, 12, unsigned int) #define PMOPS_PREPARE 1 @@ -260,7 +261,7 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, free_all_swap_pages(data->swap); break; - case SNAPSHOT_SET_SWAP_FILE: + case SNAPSHOT_SET_SWAP_FILE: /* This ioctl is deprecated */ if (!swsusp_swap_in_use()) { /* * User space encodes device types as two-byte values, -- cgit v1.2.3 From cc5d207c85b9a6fafebe2856ead0a9360978c8cd Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 26 Oct 2007 01:03:33 +0200 Subject: Hibernation: Correct definitions of some ioctls (rev. 2) Three ioctl numbers belonging to the hibernation userland interface, SNAPSHOT_ATOMIC_SNAPSHOT, SNAPSHOT_AVAIL_SWAP, SNAPSHOT_GET_SWAP_PAGE, are defined in a wrong way (eg. not portable). Provide new ioctl numbers for these ioctls and mark the existing ones as deprecated. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/power.h | 10 +++++----- kernel/power/user.c | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/power/power.h b/kernel/power/power.h index 8837ea334e3..0dd66fabd39 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -147,12 +147,8 @@ struct resume_swap_area { #define SNAPSHOT_IOC_MAGIC '3' #define SNAPSHOT_FREEZE _IO(SNAPSHOT_IOC_MAGIC, 1) #define SNAPSHOT_UNFREEZE _IO(SNAPSHOT_IOC_MAGIC, 2) -#define SNAPSHOT_ATOMIC_SNAPSHOT _IOW(SNAPSHOT_IOC_MAGIC, 3, void *) #define SNAPSHOT_ATOMIC_RESTORE _IO(SNAPSHOT_IOC_MAGIC, 4) #define SNAPSHOT_FREE _IO(SNAPSHOT_IOC_MAGIC, 5) -#define SNAPSHOT_SET_IMAGE_SIZE _IOW(SNAPSHOT_IOC_MAGIC, 6, unsigned long) -#define SNAPSHOT_AVAIL_SWAP _IOR(SNAPSHOT_IOC_MAGIC, 7, void *) -#define SNAPSHOT_GET_SWAP_PAGE _IOR(SNAPSHOT_IOC_MAGIC, 8, void *) #define SNAPSHOT_FREE_SWAP_PAGES _IO(SNAPSHOT_IOC_MAGIC, 9) #define SNAPSHOT_S2RAM _IO(SNAPSHOT_IOC_MAGIC, 11) #define SNAPSHOT_SET_SWAP_AREA _IOW(SNAPSHOT_IOC_MAGIC, 13, \ @@ -160,7 +156,11 @@ struct resume_swap_area { #define SNAPSHOT_GET_IMAGE_SIZE _IOR(SNAPSHOT_IOC_MAGIC, 14, loff_t) #define SNAPSHOT_PLATFORM_SUPPORT _IO(SNAPSHOT_IOC_MAGIC, 15) #define SNAPSHOT_POWER_OFF _IO(SNAPSHOT_IOC_MAGIC, 16) -#define SNAPSHOT_IOC_MAXNR 16 +#define SNAPSHOT_CREATE_IMAGE _IOW(SNAPSHOT_IOC_MAGIC, 17, int) +#define SNAPSHOT_PREF_IMAGE_SIZE _IO(SNAPSHOT_IOC_MAGIC, 18) +#define SNAPSHOT_AVAIL_SWAP_SIZE _IOR(SNAPSHOT_IOC_MAGIC, 19, loff_t) +#define SNAPSHOT_ALLOC_SWAP_PAGE _IOR(SNAPSHOT_IOC_MAGIC, 20, loff_t) +#define SNAPSHOT_IOC_MAXNR 20 /* If unset, the snapshot device cannot be open. */ extern atomic_t snapshot_device_available; diff --git a/kernel/power/user.c b/kernel/power/user.c index 5e866e07855..b902a7e3bd1 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c @@ -40,6 +40,16 @@ #define PMOPS_ENTER 2 #define PMOPS_FINISH 3 +/* + * NOTE: The following ioctl definitions are wrong and have been replaced with + * correct ones. They are only preserved here for compatibility with existing + * userland utilities and will be removed in the future. + */ +#define SNAPSHOT_ATOMIC_SNAPSHOT _IOW(SNAPSHOT_IOC_MAGIC, 3, void *) +#define SNAPSHOT_SET_IMAGE_SIZE _IOW(SNAPSHOT_IOC_MAGIC, 6, unsigned long) +#define SNAPSHOT_AVAIL_SWAP _IOR(SNAPSHOT_IOC_MAGIC, 7, void *) +#define SNAPSHOT_GET_SWAP_PAGE _IOR(SNAPSHOT_IOC_MAGIC, 8, void *) + #define SNAPSHOT_MINOR 231 @@ -191,6 +201,7 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, data->frozen = 0; break; + case SNAPSHOT_CREATE_IMAGE: case SNAPSHOT_ATOMIC_SNAPSHOT: if (data->mode != O_RDONLY || !data->frozen || data->ready) { error = -EPERM; @@ -198,7 +209,7 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, } error = hibernation_snapshot(data->platform_support); if (!error) - error = put_user(in_suspend, (unsigned int __user *)arg); + error = put_user(in_suspend, (int __user *)arg); if (!error) data->ready = 1; break; @@ -219,6 +230,7 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, data->ready = 0; break; + case SNAPSHOT_PREF_IMAGE_SIZE: case SNAPSHOT_SET_IMAGE_SIZE: image_size = arg; break; @@ -233,12 +245,14 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, error = put_user(size, (loff_t __user *)arg); break; + case SNAPSHOT_AVAIL_SWAP_SIZE: case SNAPSHOT_AVAIL_SWAP: size = count_swap_pages(data->swap, 1); size <<= PAGE_SHIFT; error = put_user(size, (loff_t __user *)arg); break; + case SNAPSHOT_ALLOC_SWAP_PAGE: case SNAPSHOT_GET_SWAP_PAGE: if (data->swap < 0 || data->swap >= MAX_SWAPFILES) { error = -ENODEV; @@ -247,7 +261,7 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, offset = alloc_swapdev_block(data->swap); if (offset) { offset <<= PAGE_SHIFT; - error = put_user(offset, (sector_t __user *)arg); + error = put_user(offset, (loff_t __user *)arg); } else { error = -ENOSPC; } -- cgit v1.2.3 From 3010f8caa48ed38679cc32b0d8e84b82cb8d9980 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 26 Oct 2007 01:05:05 +0200 Subject: Hibernation: Introduce exportable suspend ioctls header (rev. 2) Move the definitions of hibernation ioctls to a separate header file in include/linux, which can be exported to the user space. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/power.h | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) (limited to 'kernel') diff --git a/kernel/power/power.h b/kernel/power/power.h index 0dd66fabd39..ef9060576a4 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -1,4 +1,5 @@ #include +#include #include struct swsusp_info { @@ -134,34 +135,6 @@ extern int snapshot_write_next(struct snapshot_handle *handle, size_t count); extern void snapshot_write_finalize(struct snapshot_handle *handle); extern int snapshot_image_loaded(struct snapshot_handle *handle); -/* - * This structure is used to pass the values needed for the identification - * of the resume swap area from a user space to the kernel via the - * SNAPSHOT_SET_SWAP_AREA ioctl - */ -struct resume_swap_area { - loff_t offset; - u_int32_t dev; -} __attribute__((packed)); - -#define SNAPSHOT_IOC_MAGIC '3' -#define SNAPSHOT_FREEZE _IO(SNAPSHOT_IOC_MAGIC, 1) -#define SNAPSHOT_UNFREEZE _IO(SNAPSHOT_IOC_MAGIC, 2) -#define SNAPSHOT_ATOMIC_RESTORE _IO(SNAPSHOT_IOC_MAGIC, 4) -#define SNAPSHOT_FREE _IO(SNAPSHOT_IOC_MAGIC, 5) -#define SNAPSHOT_FREE_SWAP_PAGES _IO(SNAPSHOT_IOC_MAGIC, 9) -#define SNAPSHOT_S2RAM _IO(SNAPSHOT_IOC_MAGIC, 11) -#define SNAPSHOT_SET_SWAP_AREA _IOW(SNAPSHOT_IOC_MAGIC, 13, \ - struct resume_swap_area) -#define SNAPSHOT_GET_IMAGE_SIZE _IOR(SNAPSHOT_IOC_MAGIC, 14, loff_t) -#define SNAPSHOT_PLATFORM_SUPPORT _IO(SNAPSHOT_IOC_MAGIC, 15) -#define SNAPSHOT_POWER_OFF _IO(SNAPSHOT_IOC_MAGIC, 16) -#define SNAPSHOT_CREATE_IMAGE _IOW(SNAPSHOT_IOC_MAGIC, 17, int) -#define SNAPSHOT_PREF_IMAGE_SIZE _IO(SNAPSHOT_IOC_MAGIC, 18) -#define SNAPSHOT_AVAIL_SWAP_SIZE _IOR(SNAPSHOT_IOC_MAGIC, 19, loff_t) -#define SNAPSHOT_ALLOC_SWAP_PAGE _IOR(SNAPSHOT_IOC_MAGIC, 20, loff_t) -#define SNAPSHOT_IOC_MAXNR 20 - /* If unset, the snapshot device cannot be open. */ extern atomic_t snapshot_device_available; -- cgit v1.2.3 From 2f8ed1c60b06b797bf79a1dc540f0bed8c9d75a0 Mon Sep 17 00:00:00 2001 From: Adrian Bunk Date: Mon, 19 Nov 2007 23:36:20 +0100 Subject: Hibernation: Move function prototypes to header This patch moves the prototypes of count_highmem_pages() and restore_highmem() to kernel/power/power.h Signed-off-by: Adrian Bunk Acked-by: Pavel Machek Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/power.h | 8 ++++++++ kernel/power/snapshot.c | 1 - kernel/power/swsusp.c | 8 -------- 3 files changed, 8 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/power/power.h b/kernel/power/power.h index ef9060576a4..c5321eb1f7c 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -180,3 +180,11 @@ static inline int pm_notifier_call_chain(unsigned long val) return (blocking_notifier_call_chain(&pm_chain_head, val, NULL) == NOTIFY_BAD) ? -EINVAL : 0; } + +#ifdef CONFIG_HIGHMEM +unsigned int count_highmem_pages(void); +int restore_highmem(void); +#else +static inline unsigned int count_highmem_pages(void) { return 0; } +static inline int restore_highmem(void) { return 0; } +#endif diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index c5ce0f34a5d..1ec3eccb1a3 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -872,7 +872,6 @@ unsigned int count_highmem_pages(void) } #else static inline void *saveable_highmem_page(unsigned long pfn) { return NULL; } -static inline unsigned int count_highmem_pages(void) { return 0; } #endif /* CONFIG_HIGHMEM */ /** diff --git a/kernel/power/swsusp.c b/kernel/power/swsusp.c index e1722d3155f..605c536795b 100644 --- a/kernel/power/swsusp.c +++ b/kernel/power/swsusp.c @@ -64,14 +64,6 @@ unsigned long image_size = 500 * 1024 * 1024; int in_suspend __nosavedata = 0; -#ifdef CONFIG_HIGHMEM -unsigned int count_highmem_pages(void); -int restore_highmem(void); -#else -static inline int restore_highmem(void) { return 0; } -static inline unsigned int count_highmem_pages(void) { return 0; } -#endif - /** * The following functions are used for tracing the allocated * swap pages, so that they can be freed in case of an error. -- cgit v1.2.3 From c3e94d899c864e558f938f9845ddb8c2e5d5ccd0 Mon Sep 17 00:00:00 2001 From: Alan Stern Date: Mon, 19 Nov 2007 23:38:25 +0100 Subject: Hibernation: Add PM_RESTORE_PREPARE and PM_POST_RESTORE notifiers (rev. 2) Add PM_RESTORE_PREPARE and PM_POST_RESTORE notifiers to the PM core, to be used in analogy with the existing PM_HIBERNATION_PREPARE and PM_POST_HIBERNATION notifiers. Signed-off-by: Alan Stern Acked-by: Pavel Machek Signed-off-by: "Rafael J. Wysocki" Signed-off-by: Len Brown --- kernel/power/disk.c | 5 +++++ kernel/power/user.c | 31 +++++++++++++++++++------------ 2 files changed, 24 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/power/disk.c b/kernel/power/disk.c index b138b431e27..65973650823 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c @@ -499,6 +499,10 @@ static int software_resume(void) goto Unlock; } + error = pm_notifier_call_chain(PM_RESTORE_PREPARE); + if (error) + goto Finish; + error = create_basic_memory_bitmaps(); if (error) goto Finish; @@ -522,6 +526,7 @@ static int software_resume(void) Done: free_basic_memory_bitmaps(); Finish: + pm_notifier_call_chain(PM_POST_RESTORE); atomic_inc(&snapshot_device_available); /* For success case, the suspend path will release the lock */ Unlock: diff --git a/kernel/power/user.c b/kernel/power/user.c index b902a7e3bd1..f5512cb3aa8 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c @@ -67,6 +67,7 @@ atomic_t snapshot_device_available = ATOMIC_INIT(1); static int snapshot_open(struct inode *inode, struct file *filp) { struct snapshot_data *data; + int error; if (!atomic_add_unless(&snapshot_device_available, -1, 0)) return -EBUSY; @@ -87,9 +88,19 @@ static int snapshot_open(struct inode *inode, struct file *filp) data->swap = swsusp_resume_device ? swap_type_of(swsusp_resume_device, 0, NULL) : -1; data->mode = O_RDONLY; + error = pm_notifier_call_chain(PM_RESTORE_PREPARE); + if (error) + pm_notifier_call_chain(PM_POST_RESTORE); } else { data->swap = -1; data->mode = O_WRONLY; + error = pm_notifier_call_chain(PM_HIBERNATION_PREPARE); + if (error) + pm_notifier_call_chain(PM_POST_HIBERNATION); + } + if (error) { + atomic_inc(&snapshot_device_available); + return error; } data->frozen = 0; data->ready = 0; @@ -111,6 +122,8 @@ static int snapshot_release(struct inode *inode, struct file *filp) thaw_processes(); mutex_unlock(&pm_mutex); } + pm_notifier_call_chain(data->mode == O_WRONLY ? + PM_POST_HIBERNATION : PM_POST_RESTORE); atomic_inc(&snapshot_device_available); return 0; } @@ -174,18 +187,13 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, if (data->frozen) break; mutex_lock(&pm_mutex); - error = pm_notifier_call_chain(PM_HIBERNATION_PREPARE); - if (!error) { - printk("Syncing filesystems ... "); - sys_sync(); - printk("done.\n"); - - error = freeze_processes(); - if (error) - thaw_processes(); - } + printk("Syncing filesystems ... "); + sys_sync(); + printk("done.\n"); + + error = freeze_processes(); if (error) - pm_notifier_call_chain(PM_POST_HIBERNATION); + thaw_processes(); mutex_unlock(&pm_mutex); if (!error) data->frozen = 1; @@ -196,7 +204,6 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp, break; mutex_lock(&pm_mutex); thaw_processes(); - pm_notifier_call_chain(PM_POST_HIBERNATION); mutex_unlock(&pm_mutex); data->frozen = 0; break; -- cgit v1.2.3 From 0e7d56e3d9d7e37c79d0e05ffb3994e34beb3bbc Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 19 Nov 2007 23:41:19 +0100 Subject: Suspend: Testing facility (rev. 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce sysfs attribute /sys/power/pm_test allowing one to test the suspend core code.  Namely, writing one of the strings: freezer devices platform processors core to this file causes the suspend code to work in one of the test modes defined as follows: freezer - test the freezing of processes devices - test the freezing of processes and suspending of devices platform - test the freezing of processes, suspending of devices and platform global   control methods(*) processors - test the freezing of processes, suspending of devices, platform global   control methods and the disabling of nonboot CPUs core - test the freezing of processes, suspending of devices, platform global   control methods, the disabling of nonboot CPUs and suspending of   platform/system devices (*) These are ACPI global control methods on ACPI systems Then, if a suspend is started by normal means, the suspend core will perform its normal operations up to the point indicated by given test level.  Next, it will wait for 5 seconds and carry out the resume operations needed to transition the system back to the fully functional state. Writing "none" to /sys/power/pm_test turns the testing off. When open for reading, /sys/power/pm_test contains a space-separated list of all available tests (including "none" that represents the normal functionality) in which the current test level is indicated by square brackets. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/main.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++----- kernel/power/power.h | 18 +++++++++ 2 files changed, 117 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/power/main.c b/kernel/power/main.c index efc08360e62..84e1ae63bb8 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -31,6 +31,79 @@ DEFINE_MUTEX(pm_mutex); unsigned int pm_flags; EXPORT_SYMBOL(pm_flags); +#ifdef CONFIG_PM_DEBUG +int pm_test_level = TEST_NONE; + +static int suspend_test(int level) +{ + if (pm_test_level == level) { + printk(KERN_INFO "suspend debug: Waiting for 5 seconds.\n"); + mdelay(5000); + return 1; + } + return 0; +} + +static const char * const pm_tests[__TEST_AFTER_LAST] = { + [TEST_NONE] = "none", + [TEST_CORE] = "core", + [TEST_CPUS] = "processors", + [TEST_PLATFORM] = "platform", + [TEST_DEVICES] = "devices", + [TEST_FREEZER] = "freezer", +}; + +static ssize_t pm_test_show(struct kset *kset, char *buf) +{ + char *s = buf; + int level; + + for (level = TEST_FIRST; level <= TEST_MAX; level++) + if (pm_tests[level]) { + if (level == pm_test_level) + s += sprintf(s, "[%s] ", pm_tests[level]); + else + s += sprintf(s, "%s ", pm_tests[level]); + } + + if (s != buf) + /* convert the last space to a newline */ + *(s-1) = '\n'; + + return (s - buf); +} + +static ssize_t pm_test_store(struct kset *kset, const char *buf, size_t n) +{ + const char * const *s; + int level; + char *p; + int len; + int error = -EINVAL; + + p = memchr(buf, '\n', n); + len = p ? p - buf : n; + + mutex_lock(&pm_mutex); + + level = TEST_FIRST; + for (s = &pm_tests[level]; level <= TEST_MAX; s++, level++) + if (*s && len == strlen(*s) && !strncmp(buf, *s, len)) { + pm_test_level = level; + error = 0; + break; + } + + mutex_unlock(&pm_mutex); + + return error ? error : n; +} + +power_attr(pm_test); +#else /* !CONFIG_PM_DEBUG */ +static inline int suspend_test(int level) { return 0; } +#endif /* !CONFIG_PM_DEBUG */ + #ifdef CONFIG_SUSPEND /* This is just an arbitrary number */ @@ -136,7 +209,10 @@ static int suspend_enter(suspend_state_t state) printk(KERN_ERR "Some devices failed to power down\n"); goto Done; } - error = suspend_ops->enter(state); + + if (!suspend_test(TEST_CORE)) + error = suspend_ops->enter(state); + device_power_up(); Done: arch_suspend_enable_irqs(); @@ -167,16 +243,25 @@ int suspend_devices_and_enter(suspend_state_t state) printk(KERN_ERR "Some devices failed to suspend\n"); goto Resume_console; } + + if (suspend_test(TEST_DEVICES)) + goto Resume_devices; + if (suspend_ops->prepare) { error = suspend_ops->prepare(); if (error) goto Resume_devices; } + + if (suspend_test(TEST_PLATFORM)) + goto Finish; + error = disable_nonboot_cpus(); - if (!error) + if (!error && !suspend_test(TEST_CPUS)) suspend_enter(state); enable_nonboot_cpus(); + Finish: if (suspend_ops->finish) suspend_ops->finish(); Resume_devices: @@ -243,12 +328,17 @@ static int enter_state(suspend_state_t state) printk("done.\n"); pr_debug("PM: Preparing system for %s sleep\n", pm_states[state]); - if ((error = suspend_prepare())) + error = suspend_prepare(); + if (error) goto Unlock; + if (suspend_test(TEST_FREEZER)) + goto Finish; + pr_debug("PM: Entering %s sleep\n", pm_states[state]); error = suspend_devices_and_enter(state); + Finish: pr_debug("PM: Finishing wakeup.\n"); suspend_finish(); Unlock: @@ -369,18 +459,18 @@ pm_trace_store(struct kobject *kobj, struct kobj_attribute *attr, } power_attr(pm_trace); +#endif /* CONFIG_PM_TRACE */ static struct attribute * g[] = { &state_attr.attr, +#ifdef CONFIG_PM_TRACE &pm_trace_attr.attr, +#endif +#ifdef CONFIG_PM_DEBUG + &pm_test_attr.attr, +#endif NULL, }; -#else -static struct attribute * g[] = { - &state_attr.attr, - NULL, -}; -#endif /* CONFIG_PM_TRACE */ static struct attribute_group attr_group = { .attrs = g, diff --git a/kernel/power/power.h b/kernel/power/power.h index c5321eb1f7c..9f9e16e3396 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -188,3 +188,21 @@ int restore_highmem(void); static inline unsigned int count_highmem_pages(void) { return 0; } static inline int restore_highmem(void) { return 0; } #endif + +/* + * Suspend test levels + */ +enum { + /* keep first */ + TEST_NONE, + TEST_CORE, + TEST_CPUS, + TEST_PLATFORM, + TEST_DEVICES, + TEST_FREEZER, + /* keep last */ + __TEST_AFTER_LAST +}; + +#define TEST_FIRST TEST_NONE +#define TEST_MAX (__TEST_AFTER_LAST - 1) -- cgit v1.2.3 From 039a75c6e17ba4ff76998d6ac6ee3d508fff1930 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 29 Jan 2008 00:29:06 +0100 Subject: suspend: build fix responding to 2.6.25 kset change Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/main.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/power/main.c b/kernel/power/main.c index 84e1ae63bb8..fc717b83682 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -53,7 +53,8 @@ static const char * const pm_tests[__TEST_AFTER_LAST] = { [TEST_FREEZER] = "freezer", }; -static ssize_t pm_test_show(struct kset *kset, char *buf) +static ssize_t pm_test_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) { char *s = buf; int level; @@ -73,7 +74,8 @@ static ssize_t pm_test_show(struct kset *kset, char *buf) return (s - buf); } -static ssize_t pm_test_store(struct kset *kset, const char *buf, size_t n) +static ssize_t pm_test_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t n) { const char * const *s; int level; @@ -104,6 +106,7 @@ power_attr(pm_test); static inline int suspend_test(int level) { return 0; } #endif /* !CONFIG_PM_DEBUG */ + #ifdef CONFIG_SUSPEND /* This is just an arbitrary number */ -- cgit v1.2.3 From 4cc79776c9ea431790e04fcacbebb30d28eb1570 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 19 Nov 2007 23:42:31 +0100 Subject: Hibernation: New testing facility (rev. 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make it possible to test the hibernation core code with the help of the /sys/power/pm_test attribute introduced for suspend testing in the previous patch. Writing an appropriate string to this file causes the hibernation code to work in one of the test modes defined as follows: freezer - test the freezing of processes devices - test the freezing of processes and suspending of devices platform - test the freezing of processes, suspending of devices and platform global   control methods(*) processors - test the freezing of processes, suspending of devices, platform global   control methods(*) and the disabling of nonboot CPUs core - test the freezing of processes, suspending of devices, platform global   control methods(*), the disabling of nonboot CPUs and suspending of   platform/system devices (*) - the platform global control methods are only available on ACPI systems       and are only tested if the hibernation mode is set to "platform" Then, if a hibernation is started by normal means, the hibernation core will perform its normal operations up to the point indicated by given test level. Next, it will wait for 5 seconds and carry out the resume operations needed to transition the system back to the fully functional state. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/disk.c | 70 +++++++++++++++++++++++++++++++++++++++++----------- kernel/power/power.h | 2 ++ 2 files changed, 57 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/power/disk.c b/kernel/power/disk.c index 65973650823..0866b163c6b 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c @@ -70,6 +70,35 @@ void hibernation_set_ops(struct platform_hibernation_ops *ops) mutex_unlock(&pm_mutex); } +#ifdef CONFIG_PM_DEBUG +static void hibernation_debug_sleep(void) +{ + printk(KERN_INFO "hibernation debug: Waiting for 5 seconds.\n"); + mdelay(5000); +} + +static int hibernation_testmode(int mode) +{ + if (hibernation_mode == mode) { + hibernation_debug_sleep(); + return 1; + } + return 0; +} + +static int hibernation_test(int level) +{ + if (pm_test_level == level) { + hibernation_debug_sleep(); + return 1; + } + return 0; +} +#else /* !CONFIG_PM_DEBUG */ +static int hibernation_testmode(int mode) { return 0; } +static int hibernation_test(int level) { return 0; } +#endif /* !CONFIG_PM_DEBUG */ + /** * platform_start - tell the platform driver that we're starting * hibernation @@ -167,6 +196,10 @@ int create_image(int platform_mode) goto Enable_irqs; } + if (hibernation_test(TEST_CORE)) + goto Power_up; + + in_suspend = 1; save_processor_state(); error = swsusp_arch_suspend(); if (error) @@ -175,6 +208,7 @@ int create_image(int platform_mode) restore_processor_state(); if (!in_suspend) platform_leave(platform_mode); + Power_up: /* NOTE: device_power_up() is just a resume() for devices * that suspended with irqs off ... no overall powerup. */ @@ -211,24 +245,29 @@ int hibernation_snapshot(int platform_mode) if (error) goto Resume_console; - error = platform_pre_snapshot(platform_mode); - if (error) + if (hibernation_test(TEST_DEVICES)) goto Resume_devices; + error = platform_pre_snapshot(platform_mode); + if (error || hibernation_test(TEST_PLATFORM)) + goto Finish; + error = disable_nonboot_cpus(); if (!error) { - if (hibernation_mode != HIBERNATION_TEST) { - in_suspend = 1; - error = create_image(platform_mode); - /* Control returns here after successful restore */ - } else { - printk("swsusp debug: Waiting for 5 seconds.\n"); - mdelay(5000); - } + if (hibernation_test(TEST_CPUS)) + goto Enable_cpus; + + if (hibernation_testmode(HIBERNATION_TEST)) + goto Enable_cpus; + + error = create_image(platform_mode); + /* Control returns here after successful restore */ } + Enable_cpus: enable_nonboot_cpus(); - Resume_devices: + Finish: platform_finish(platform_mode); + Resume_devices: device_resume(); Resume_console: resume_console(); @@ -406,11 +445,12 @@ int hibernate(void) if (error) goto Finish; - if (hibernation_mode == HIBERNATION_TESTPROC) { - printk("swsusp debug: Waiting for 5 seconds.\n"); - mdelay(5000); + if (hibernation_test(TEST_FREEZER)) goto Thaw; - } + + if (hibernation_testmode(HIBERNATION_TESTPROC)) + goto Thaw; + error = hibernation_snapshot(hibernation_mode == HIBERNATION_PLATFORM); if (in_suspend && !error) { unsigned int flags = 0; diff --git a/kernel/power/power.h b/kernel/power/power.h index 9f9e16e3396..f9f0d4d26c5 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -206,3 +206,5 @@ enum { #define TEST_FIRST TEST_NONE #define TEST_MAX (__TEST_AFTER_LAST - 1) + +extern int pm_test_level; -- cgit v1.2.3 From 90dda1cb6ace6abd777f84bf051c4f86fa58986a Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Mon, 19 Nov 2007 23:46:16 +0100 Subject: PM: Make PM_TRACE more architecture independent When trying to debug a suspend failure I started implementing PM_TRACE for powerpc. I then noticed that I'm debugging a suspend failure and so PM_TRACE isn't useful at all, but thought that nonetheless this could be useful in the future. Basically, to support PM_TRACE, you add a Kconfig option that selects PM_TRACE and provides the infrastructure as per the help text of PM_TRACE. Signed-off-by: Johannes Berg Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/Kconfig | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index 8e186c67814..06a08f7cebd 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -44,9 +44,30 @@ config PM_VERBOSE ---help--- This option enables verbose messages from the Power Management code. +config CAN_PM_TRACE + def_bool y + depends on PM_DEBUG && PM_SLEEP && EXPERIMENTAL + config PM_TRACE + bool + help + This enables code to save the last PM event point across + reboot. The architecture needs to support this, x86 for + example does by saving things in the RTC, see below. + + The architecture specific code must provide the extern + functions from as well as the + header with a TRACE_RESUME() macro. + + The way the information is presented is architecture- + dependent, x86 will print the information during a + late_initcall. + +config PM_TRACE_RTC bool "Suspend/resume event tracing" - depends on PM_DEBUG && X86 && PM_SLEEP && EXPERIMENTAL + depends on CAN_PM_TRACE + depends on X86 + select PM_TRACE default n ---help--- This enables some cheesy code to save the last PM event point in the -- cgit v1.2.3 From 825257569350e913bee3bc918508c0aa6e3398cd Mon Sep 17 00:00:00 2001 From: Alan Stern Date: Mon, 19 Nov 2007 23:49:18 +0100 Subject: PM: Convert PM notifiers to out-of-line code This patch (as1008b) converts the PM notifier routines from inline calls to out-of-line code. It also prevents pm_chain_head from being created when CONFIG_PM_SLEEP isn't enabled, and EXPORTs the notifier registration and unregistration routines. Signed-off-by: Alan Stern Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/main.c | 28 ++++++++++++++++++++++++++-- kernel/power/power.h | 12 ++++-------- 2 files changed, 30 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/power/main.c b/kernel/power/main.c index fc717b83682..0a9f269075e 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -24,13 +24,37 @@ #include "power.h" -BLOCKING_NOTIFIER_HEAD(pm_chain_head); - DEFINE_MUTEX(pm_mutex); unsigned int pm_flags; EXPORT_SYMBOL(pm_flags); +#ifdef CONFIG_PM_SLEEP + +/* Routines for PM-transition notifications */ + +static BLOCKING_NOTIFIER_HEAD(pm_chain_head); + +int register_pm_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&pm_chain_head, nb); +} +EXPORT_SYMBOL_GPL(register_pm_notifier); + +int unregister_pm_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&pm_chain_head, nb); +} +EXPORT_SYMBOL_GPL(unregister_pm_notifier); + +int pm_notifier_call_chain(unsigned long val) +{ + return (blocking_notifier_call_chain(&pm_chain_head, val, NULL) + == NOTIFY_BAD) ? -EINVAL : 0; +} + +#endif /* CONFIG_PM_SLEEP */ + #ifdef CONFIG_PM_DEBUG int pm_test_level = TEST_NONE; diff --git a/kernel/power/power.h b/kernel/power/power.h index f9f0d4d26c5..a9732fd1223 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -172,14 +172,10 @@ static inline int suspend_devices_and_enter(suspend_state_t state) } #endif /* !CONFIG_SUSPEND */ -/* kernel/power/common.c */ -extern struct blocking_notifier_head pm_chain_head; - -static inline int pm_notifier_call_chain(unsigned long val) -{ - return (blocking_notifier_call_chain(&pm_chain_head, val, NULL) - == NOTIFY_BAD) ? -EINVAL : 0; -} +#ifdef CONFIG_PM_SLEEP +/* kernel/power/main.c */ +extern int pm_notifier_call_chain(unsigned long val); +#endif #ifdef CONFIG_HIGHMEM unsigned int count_highmem_pages(void); -- cgit v1.2.3 From 2ed43b63285c394cb5e1829c199cc94c7b8233b9 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Sat, 8 Dec 2007 02:03:26 +0100 Subject: Suspend: Fix compilation warning for CONFIG_SUSPEND unset Suspend: Make debug facility depend on CONFIG_SUSPEND Make the new suspend debug facility code depend on CONFIG_SUSPEND, as appropriate, to remove the compiler warning printed when CONFIG_PM is set and CONFIG_SUSPEND is not set. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/main.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/power/main.c b/kernel/power/main.c index 0a9f269075e..7fb805de19a 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -55,6 +55,8 @@ int pm_notifier_call_chain(unsigned long val) #endif /* CONFIG_PM_SLEEP */ +#ifdef CONFIG_SUSPEND + #ifdef CONFIG_PM_DEBUG int pm_test_level = TEST_NONE; -- cgit v1.2.3 From 72df68ca8e006a0107933c4fb13c741a0a48163f Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Sat, 8 Dec 2007 02:04:21 +0100 Subject: Hibernation: Move low level resume to disk.c Move the low level restore code to kernel/power/disk.c , since the corresponding low level hibernation code is already there. Make restore fail if device_power_down(PMSG_PRETHAW) returns an error. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/disk.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- kernel/power/power.h | 1 - kernel/power/swsusp.c | 35 ----------------------------------- 3 files changed, 48 insertions(+), 37 deletions(-) (limited to 'kernel') diff --git a/kernel/power/disk.c b/kernel/power/disk.c index 0866b163c6b..2a4bada184e 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c @@ -274,6 +274,53 @@ int hibernation_snapshot(int platform_mode) return error; } +/** + * resume_target_kernel - prepare devices that need to be suspended with + * interrupts off, restore the contents of highmem that have not been + * restored yet from the image and run the low level code that will restore + * the remaining contents of memory and switch to the just restored target + * kernel. + */ + +static int resume_target_kernel(void) +{ + int error; + + local_irq_disable(); + error = device_power_down(PMSG_PRETHAW); + if (error) { + printk(KERN_ERR "Some devices failed to power down, " + "aborting resume\n"); + goto Enable_irqs; + } + /* We'll ignore saved state, but this gets preempt count (etc) right */ + save_processor_state(); + error = restore_highmem(); + if (!error) { + error = swsusp_arch_resume(); + /* + * The code below is only ever reached in case of a failure. + * Otherwise execution continues at place where + * swsusp_arch_suspend() was called + */ + BUG_ON(!error); + /* This call to restore_highmem() undos the previous one */ + restore_highmem(); + } + /* + * The only reason why swsusp_arch_resume() can fail is memory being + * very tight, so we have to free it as soon as we can to avoid + * subsequent failures + */ + swsusp_free(); + restore_processor_state(); + touch_softlockup_watchdog(); + device_power_up(); + Enable_irqs: + local_irq_enable(); + return error; +} + /** * hibernation_restore - quiesce devices and restore the hibernation * snapshot image. If successful, control returns in hibernation_snaphot() @@ -297,7 +344,7 @@ int hibernation_restore(int platform_mode) if (!error) { error = disable_nonboot_cpus(); if (!error) - error = swsusp_resume(); + error = resume_target_kernel(); enable_nonboot_cpus(); } platform_restore_cleanup(platform_mode); diff --git a/kernel/power/power.h b/kernel/power/power.h index a9732fd1223..8ec5499c5ce 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -152,7 +152,6 @@ extern int swsusp_swap_in_use(void); extern int swsusp_check(void); extern int swsusp_shrink_memory(void); extern void swsusp_free(void); -extern int swsusp_resume(void); extern int swsusp_read(unsigned int *flags_p); extern int swsusp_write(unsigned int flags); extern void swsusp_close(void); diff --git a/kernel/power/swsusp.c b/kernel/power/swsusp.c index 605c536795b..dc29a20aff4 100644 --- a/kernel/power/swsusp.c +++ b/kernel/power/swsusp.c @@ -261,38 +261,3 @@ int swsusp_shrink_memory(void) return 0; } - -int swsusp_resume(void) -{ - int error; - - local_irq_disable(); - /* NOTE: device_power_down() is just a suspend() with irqs off; - * it has no special "power things down" semantics - */ - if (device_power_down(PMSG_PRETHAW)) - printk(KERN_ERR "Some devices failed to power down, very bad\n"); - /* We'll ignore saved state, but this gets preempt count (etc) right */ - save_processor_state(); - error = restore_highmem(); - if (!error) { - error = swsusp_arch_resume(); - /* The code below is only ever reached in case of a failure. - * Otherwise execution continues at place where - * swsusp_arch_suspend() was called - */ - BUG_ON(!error); - /* This call to restore_highmem() undos the previous one */ - restore_highmem(); - } - /* The only reason why swsusp_arch_resume() can fail is memory being - * very tight, so we have to free it as soon as we can to avoid - * subsequent failures - */ - swsusp_free(); - restore_processor_state(); - touch_softlockup_watchdog(); - device_power_up(); - local_irq_enable(); - return error; -} -- cgit v1.2.3 From 9628c0ee6a6d9ef06a77ea25932c00817f9e88a0 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Sat, 8 Dec 2007 02:06:00 +0100 Subject: Suspend: Fix comment in main.c Fix a comment in kernel/power/main.c so that it doesn't contain lines longer that 80 characters. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/power/main.c b/kernel/power/main.c index 7fb805de19a..d0cedaa7edf 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -250,8 +250,8 @@ static int suspend_enter(suspend_state_t state) } /** - * suspend_devices_and_enter - suspend devices and enter the desired system sleep - * state. + * suspend_devices_and_enter - suspend devices and enter the desired system + * sleep state. * @state: state to enter */ int suspend_devices_and_enter(suspend_state_t state) -- cgit v1.2.3 From 9575809c6fc15e7b6bb1932b6104c80a6d4ffdc9 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Sat, 8 Dec 2007 02:06:57 +0100 Subject: Hibernation: Fix comment in disk.c Fix a comment in kernel/power/disk.c so that it doesn't contain lines longer that 80 characters. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/disk.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/power/disk.c b/kernel/power/disk.c index 2a4bada184e..3e24a200f1d 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c @@ -568,8 +568,8 @@ static int software_resume(void) if (noresume) { /** - * FIXME: If noresume is specified, we need to find the partition - * and reset it back to normal swap space. + * FIXME: If noresume is specified, we need to find the + * partition and reset it back to normal swap space. */ mutex_unlock(&pm_mutex); return 0; -- cgit v1.2.3 From b6887a29441ed5f0728b31ce90c0f0a0427317a3 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Sat, 8 Dec 2007 02:07:40 +0100 Subject: Hibernation: Remove unnecessary variable declaration Remove the unnecessary extern declaration of resume_file[] from kernel/power/swap.c . Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/swap.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel') diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 917aba10057..ef41440879b 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -28,8 +28,6 @@ #include "power.h" -extern char resume_file[]; - #define SWSUSP_SIG "S1SUSPEND" struct swsusp_header { -- cgit v1.2.3 From 465d2b477f6a0ffe01242561a93e7bf81d67c776 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Sat, 8 Dec 2007 02:08:38 +0100 Subject: Suspend: Use common prefix in messages Make suspend messages start with one common prefix "PM: ". Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/power/main.c b/kernel/power/main.c index d0cedaa7edf..56881681f18 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -235,7 +235,7 @@ static int suspend_enter(suspend_state_t state) BUG_ON(!irqs_disabled()); if ((error = device_power_down(PMSG_SUSPEND))) { - printk(KERN_ERR "Some devices failed to power down\n"); + printk(KERN_ERR "PM: Some devices failed to power down\n"); goto Done; } @@ -269,7 +269,7 @@ int suspend_devices_and_enter(suspend_state_t state) suspend_console(); error = device_suspend(PMSG_SUSPEND); if (error) { - printk(KERN_ERR "Some devices failed to suspend\n"); + printk(KERN_ERR "PM: Some devices failed to suspend\n"); goto Resume_console; } @@ -352,7 +352,7 @@ static int enter_state(suspend_state_t state) if (!mutex_trylock(&pm_mutex)) return -EBUSY; - printk("Syncing filesystems ... "); + printk(KERN_INFO "PM: Syncing filesystems ... "); sys_sync(); printk("done.\n"); -- cgit v1.2.3 From 23976728a48c3b76d34e17ead19addd52b3a280e Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Sat, 8 Dec 2007 02:09:43 +0100 Subject: Hibernation: Update messages Make hibernation messages start with one common prefix "PM: " and use the word "hibernation" in the messages as a synonym of "suspend to disk". Turn some KERN_INFO messages into debug ones. Signed-off-by: Rafael J. Wysocki Acked-by: Pavel Machek Signed-off-by: Len Brown --- kernel/power/disk.c | 28 +++++++++++++++------------- kernel/power/snapshot.c | 23 ++++++++++++----------- kernel/power/swap.c | 31 +++++++++++++++++-------------- kernel/power/swsusp.c | 5 +++-- 4 files changed, 47 insertions(+), 40 deletions(-) (limited to 'kernel') diff --git a/kernel/power/disk.c b/kernel/power/disk.c index 3e24a200f1d..64e42ab8b57 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c @@ -191,8 +191,8 @@ int create_image(int platform_mode) */ error = device_power_down(PMSG_FREEZE); if (error) { - printk(KERN_ERR "Some devices failed to power down, " - KERN_ERR "aborting suspend\n"); + printk(KERN_ERR "PM: Some devices failed to power down, " + "aborting hibernation\n"); goto Enable_irqs; } @@ -203,7 +203,8 @@ int create_image(int platform_mode) save_processor_state(); error = swsusp_arch_suspend(); if (error) - printk(KERN_ERR "Error %d while creating the image\n", error); + printk(KERN_ERR "PM: Error %d creating hibernation image\n", + error); /* Restore control flow magically appears here */ restore_processor_state(); if (!in_suspend) @@ -289,7 +290,7 @@ static int resume_target_kernel(void) local_irq_disable(); error = device_power_down(PMSG_PRETHAW); if (error) { - printk(KERN_ERR "Some devices failed to power down, " + printk(KERN_ERR "PM: Some devices failed to power down, " "aborting resume\n"); goto Enable_irqs; } @@ -438,7 +439,7 @@ static void power_down(void) * Valid image is on the disk, if we continue we risk serious data * corruption after resume. */ - printk(KERN_CRIT "Please power me down manually\n"); + printk(KERN_CRIT "PM: Please power down manually\n"); while(1); } @@ -484,7 +485,7 @@ int hibernate(void) if (error) goto Exit; - printk("Syncing filesystems ... "); + printk(KERN_INFO "PM: Syncing filesystems ... "); sys_sync(); printk("done.\n"); @@ -560,10 +561,11 @@ static int software_resume(void) return -ENOENT; } swsusp_resume_device = name_to_dev_t(resume_file); - pr_debug("swsusp: Resume From Partition %s\n", resume_file); + pr_debug("PM: Resume from partition %s\n", resume_file); } else { - pr_debug("swsusp: Resume From Partition %d:%d\n", - MAJOR(swsusp_resume_device), MINOR(swsusp_resume_device)); + pr_debug("PM: Resume from partition %d:%d\n", + MAJOR(swsusp_resume_device), + MINOR(swsusp_resume_device)); } if (noresume) { @@ -575,7 +577,7 @@ static int software_resume(void) return 0; } - pr_debug("PM: Checking swsusp image.\n"); + pr_debug("PM: Checking hibernation image.\n"); error = swsusp_check(); if (error) goto Unlock; @@ -601,7 +603,7 @@ static int software_resume(void) goto Done; } - pr_debug("PM: Reading swsusp image.\n"); + pr_debug("PM: Reading hibernation image.\n"); error = swsusp_read(&flags); if (!error) @@ -728,7 +730,7 @@ static ssize_t disk_store(struct kobject *kobj, struct kobj_attribute *attr, error = -EINVAL; if (!error) - pr_debug("PM: suspend-to-disk mode set to '%s'\n", + pr_debug("PM: Hibernation mode set to '%s'\n", hibernation_modes[mode]); mutex_unlock(&pm_mutex); return error ? error : n; @@ -760,7 +762,7 @@ static ssize_t resume_store(struct kobject *kobj, struct kobj_attribute *attr, mutex_lock(&pm_mutex); swsusp_resume_device = res; mutex_unlock(&pm_mutex); - printk("Attempting manual resume\n"); + printk(KERN_INFO "PM: Starting manual resume from disk\n"); noresume = 0; software_resume(); ret = n; diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index 1ec3eccb1a3..f6a5df934f8 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -635,7 +635,7 @@ __register_nosave_region(unsigned long start_pfn, unsigned long end_pfn, region->end_pfn = end_pfn; list_add_tail(®ion->list, &nosave_regions); Report: - printk("swsusp: Registered nosave memory region: %016lx - %016lx\n", + printk(KERN_INFO "PM: Registered nosave memory: %016lx - %016lx\n", start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT); } @@ -704,7 +704,7 @@ static void mark_nosave_pages(struct memory_bitmap *bm) list_for_each_entry(region, &nosave_regions, list) { unsigned long pfn; - printk("swsusp: Marking nosave pages: %016lx - %016lx\n", + pr_debug("PM: Marking nosave pages: %016lx - %016lx\n", region->start_pfn << PAGE_SHIFT, region->end_pfn << PAGE_SHIFT); @@ -749,7 +749,7 @@ int create_basic_memory_bitmaps(void) free_pages_map = bm2; mark_nosave_pages(forbidden_pages_map); - printk("swsusp: Basic memory bitmaps created\n"); + pr_debug("PM: Basic memory bitmaps created\n"); return 0; @@ -784,7 +784,7 @@ void free_basic_memory_bitmaps(void) memory_bm_free(bm2, PG_UNSAFE_CLEAR); kfree(bm2); - printk("swsusp: Basic memory bitmaps freed\n"); + pr_debug("PM: Basic memory bitmaps freed\n"); } /** @@ -1088,7 +1088,7 @@ static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem) } nr_pages += count_pages_for_highmem(nr_highmem); - pr_debug("swsusp: Normal pages needed: %u + %u + %u, available pages: %u\n", + pr_debug("PM: Normal pages needed: %u + %u + %u, available pages: %u\n", nr_pages, PAGES_FOR_IO, meta, free); return free > nr_pages + PAGES_FOR_IO + meta; @@ -1201,20 +1201,20 @@ asmlinkage int swsusp_save(void) { unsigned int nr_pages, nr_highmem; - printk("swsusp: critical section: \n"); + printk(KERN_INFO "PM: Creating hibernation image: \n"); drain_local_pages(); nr_pages = count_data_pages(); nr_highmem = count_highmem_pages(); - printk("swsusp: Need to copy %u pages\n", nr_pages + nr_highmem); + printk(KERN_INFO "PM: Need to copy %u pages\n", nr_pages + nr_highmem); if (!enough_free_mem(nr_pages, nr_highmem)) { - printk(KERN_ERR "swsusp: Not enough free memory\n"); + printk(KERN_ERR "PM: Not enough free memory\n"); return -ENOMEM; } if (swsusp_alloc(&orig_bm, ©_bm, nr_pages, nr_highmem)) { - printk(KERN_ERR "swsusp: Memory allocation failed\n"); + printk(KERN_ERR "PM: Memory allocation failed\n"); return -ENOMEM; } @@ -1234,7 +1234,8 @@ asmlinkage int swsusp_save(void) nr_copy_pages = nr_pages; nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE); - printk("swsusp: critical section: done (%d pages copied)\n", nr_pages); + printk(KERN_INFO "PM: Hibernation image created (%d pages copied)\n", + nr_pages); return 0; } @@ -1433,7 +1434,7 @@ static int check_header(struct swsusp_info *info) if (!reason && info->num_physpages != num_physpages) reason = "memory size"; if (reason) { - printk(KERN_ERR "swsusp: Resume mismatch: %s\n", reason); + printk(KERN_ERR "PM: Image mismatch: %s\n", reason); return -EPERM; } return 0; diff --git a/kernel/power/swap.c b/kernel/power/swap.c index ef41440879b..a0abf9a463f 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -71,7 +71,8 @@ static int submit(int rw, pgoff_t page_off, struct page *page, bio->bi_end_io = end_swap_bio_read; if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { - printk("swsusp: ERROR: adding page to bio at %ld\n", page_off); + printk(KERN_ERR "PM: Adding page to bio failed at %ld\n", + page_off); bio_put(bio); return -EFAULT; } @@ -151,7 +152,7 @@ static int mark_swapfiles(sector_t start, unsigned int flags) error = bio_write_page(swsusp_resume_block, swsusp_header, NULL); } else { - printk(KERN_ERR "swsusp: Swap header not found!\n"); + printk(KERN_ERR "PM: Swap header not found!\n"); error = -ENODEV; } return error; @@ -323,7 +324,8 @@ static int save_image(struct swap_map_handle *handle, struct timeval start; struct timeval stop; - printk("Saving image data pages (%u pages) ... ", nr_to_write); + printk(KERN_INFO "PM: Saving image data pages (%u pages) ... ", + nr_to_write); m = nr_to_write / 100; if (!m) m = 1; @@ -363,7 +365,7 @@ static int enough_swap(unsigned int nr_pages) { unsigned int free_swap = count_swap_pages(root_swap, 1); - pr_debug("swsusp: free swap pages: %u\n", free_swap); + pr_debug("PM: Free swap pages: %u\n", free_swap); return free_swap > nr_pages + PAGES_FOR_IO; } @@ -386,7 +388,7 @@ int swsusp_write(unsigned int flags) error = swsusp_swap_check(); if (error) { - printk(KERN_ERR "swsusp: Cannot find swap device, try " + printk(KERN_ERR "PM: Cannot find swap device, try " "swapon -a.\n"); return error; } @@ -400,7 +402,7 @@ int swsusp_write(unsigned int flags) } header = (struct swsusp_info *)data_of(snapshot); if (!enough_swap(header->pages)) { - printk(KERN_ERR "swsusp: Not enough free swap\n"); + printk(KERN_ERR "PM: Not enough free swap\n"); error = -ENOSPC; goto out; } @@ -415,7 +417,7 @@ int swsusp_write(unsigned int flags) if (!error) { flush_swap_writer(&handle); - printk("S"); + printk(KERN_INFO "PM: S"); error = mark_swapfiles(start, flags); printk("|\n"); } @@ -505,7 +507,8 @@ static int load_image(struct swap_map_handle *handle, int err2; unsigned nr_pages; - printk("Loading image data pages (%u pages) ... ", nr_to_read); + printk(KERN_INFO "PM: Loading image data pages (%u pages) ... ", + nr_to_read); m = nr_to_read / 100; if (!m) m = 1; @@ -556,7 +559,7 @@ int swsusp_read(unsigned int *flags_p) *flags_p = swsusp_header->flags; if (IS_ERR(resume_bdev)) { - pr_debug("swsusp: block device not initialised\n"); + pr_debug("PM: Image device not initialised\n"); return PTR_ERR(resume_bdev); } @@ -575,9 +578,9 @@ int swsusp_read(unsigned int *flags_p) blkdev_put(resume_bdev); if (!error) - pr_debug("swsusp: Reading resume file was successful\n"); + pr_debug("PM: Image successfully loaded\n"); else - pr_debug("swsusp: Error %d resuming\n", error); + pr_debug("PM: Error %d resuming\n", error); return error; } @@ -609,13 +612,13 @@ int swsusp_check(void) if (error) blkdev_put(resume_bdev); else - pr_debug("swsusp: Signature found, resuming\n"); + pr_debug("PM: Signature found, resuming\n"); } else { error = PTR_ERR(resume_bdev); } if (error) - pr_debug("swsusp: Error %d check for resume file\n", error); + pr_debug("PM: Error %d checking image file\n", error); return error; } @@ -627,7 +630,7 @@ int swsusp_check(void) void swsusp_close(void) { if (IS_ERR(resume_bdev)) { - pr_debug("swsusp: block device not initialised\n"); + pr_debug("PM: Image device not initialised\n"); return; } diff --git a/kernel/power/swsusp.c b/kernel/power/swsusp.c index dc29a20aff4..023ff2a31d8 100644 --- a/kernel/power/swsusp.c +++ b/kernel/power/swsusp.c @@ -188,7 +188,8 @@ void swsusp_show_speed(struct timeval *start, struct timeval *stop, centisecs = 1; /* avoid div-by-zero */ k = nr_pages * (PAGE_SIZE / 1024); kps = (k * 100) / centisecs; - printk("%s %d kbytes in %d.%02d seconds (%d.%02d MB/s)\n", msg, k, + printk(KERN_INFO "PM: %s %d kbytes in %d.%02d seconds (%d.%02d MB/s)\n", + msg, k, centisecs / 100, centisecs % 100, kps / 1000, (kps % 1000) / 10); } @@ -219,7 +220,7 @@ int swsusp_shrink_memory(void) char *p = "-\\|/"; struct timeval start, stop; - printk("Shrinking memory... "); + printk(KERN_INFO "PM: Shrinking memory... "); do_gettimeofday(&start); do { long size, highmem_size; -- cgit v1.2.3 From 801e4062fda6496fe9bee3e6915a2aa108f974e5 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Sat, 8 Dec 2007 02:12:39 +0100 Subject: Hibernation: Clean up Kconfig (V2) This cleans up the hibernation Kconfig and removes the need to declare centrally which architectures support hibernation. All architectures that currently support hibernation are modified accordingly. Signed-off-by: Johannes Berg Acked-by: Paul Mackerras Cc: Pavel Machek Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/Kconfig | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index 06a08f7cebd..fd76d54910d 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -84,7 +84,8 @@ config PM_TRACE_RTC config PM_SLEEP_SMP bool - depends on SUSPEND_SMP_POSSIBLE || HIBERNATION_SMP_POSSIBLE + depends on SMP + depends on SUSPEND_SMP_POSSIBLE || ARCH_HIBERNATION_POSSIBLE depends on PM_SLEEP select HOTPLUG_CPU default y @@ -118,22 +119,9 @@ config SUSPEND powered and thus its contents are preserved, such as the suspend-to-RAM state (i.e. the ACPI S3 state). -config HIBERNATION_UP_POSSIBLE - bool - depends on X86 || PPC64_SWSUSP || PPC32 - depends on !SMP - default y - -config HIBERNATION_SMP_POSSIBLE - bool - depends on (X86 && !X86_VOYAGER) || PPC64_SWSUSP - depends on SMP - default y - config HIBERNATION bool "Hibernation (aka 'suspend to disk')" - depends on PM && SWAP - depends on HIBERNATION_UP_POSSIBLE || HIBERNATION_SMP_POSSIBLE + depends on PM && SWAP && ARCH_HIBERNATION_POSSIBLE ---help--- Enable the suspend to disk (STD) functionality, which is usually called "hibernation" in user interfaces. STD checkpoints the -- cgit v1.2.3 From f4cb57007662a4ec3c2de3c027900223e0299bdd Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Sat, 8 Dec 2007 02:14:00 +0100 Subject: Suspend: Clean up Kconfig (V2) This cleans up the suspend Kconfig and removes the need to declare centrally which architectures support suspend. All architectures that currently support suspend are modified accordingly. Signed-off-by: Johannes Berg Acked-by: Russell King Acked-by: Paul Mackerras Acked-by: Ralf Baechle Acked-by: Paul Mundt Cc: Pavel Machek Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/Kconfig | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) (limited to 'kernel') diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index fd76d54910d..f8153fda06b 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -85,7 +85,7 @@ config PM_TRACE_RTC config PM_SLEEP_SMP bool depends on SMP - depends on SUSPEND_SMP_POSSIBLE || ARCH_HIBERNATION_POSSIBLE + depends on ARCH_SUSPEND_POSSIBLE || ARCH_HIBERNATION_POSSIBLE depends on PM_SLEEP select HOTPLUG_CPU default y @@ -95,29 +95,14 @@ config PM_SLEEP depends on SUSPEND || HIBERNATION default y -config SUSPEND_UP_POSSIBLE - bool - depends on (X86 && !X86_VOYAGER) || PPC || ARM || BLACKFIN || MIPS \ - || SUPERH || FRV - depends on !SMP - default y - -config SUSPEND_SMP_POSSIBLE - bool - depends on (X86 && !X86_VOYAGER) \ - || (PPC && (PPC_PSERIES || PPC_PMAC)) || ARM - depends on SMP - default y - config SUSPEND bool "Suspend to RAM and standby" - depends on PM - depends on SUSPEND_UP_POSSIBLE || SUSPEND_SMP_POSSIBLE + depends on PM && ARCH_SUSPEND_POSSIBLE default y ---help--- Allow the system to enter sleep states in which main memory is powered and thus its contents are preserved, such as the - suspend-to-RAM state (i.e. the ACPI S3 state). + suspend-to-RAM state (e.g. the ACPI S3 state). config HIBERNATION bool "Hibernation (aka 'suspend to disk')" -- cgit v1.2.3 From 7671b8ae5381a504d4c4ef8dd9c47128c2c3fd7e Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 14 Dec 2007 01:07:13 +0100 Subject: suspend: fix ia64 allmodconfig build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kernel/power/main.c:488: error: ‘pm_test_attr’ undeclared here (not in a function) Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/main.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/power/main.c b/kernel/power/main.c index 56881681f18..050a6077ea4 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -53,10 +53,6 @@ int pm_notifier_call_chain(unsigned long val) == NOTIFY_BAD) ? -EINVAL : 0; } -#endif /* CONFIG_PM_SLEEP */ - -#ifdef CONFIG_SUSPEND - #ifdef CONFIG_PM_DEBUG int pm_test_level = TEST_NONE; @@ -132,6 +128,7 @@ power_attr(pm_test); static inline int suspend_test(int level) { return 0; } #endif /* !CONFIG_PM_DEBUG */ +#endif /* CONFIG_PM_SLEEP */ #ifdef CONFIG_SUSPEND @@ -495,7 +492,7 @@ static struct attribute * g[] = { #ifdef CONFIG_PM_TRACE &pm_trace_attr.attr, #endif -#ifdef CONFIG_PM_DEBUG +#if defined(CONFIG_PM_SLEEP) && defined(CONFIG_PM_DEBUG) &pm_test_attr.attr, #endif NULL, -- cgit v1.2.3 From c697eecebc6cfc0b393afea3c4ff1a5041526ad1 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 8 Jan 2008 00:04:17 +0100 Subject: Suspend: Introduce begin() and end() callbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On ACPI systems the target state set by acpi_pm_set_target() is reset by acpi_pm_finish(), but that need not be called if the suspend fails.  All platforms that use the .set_target() global suspend callback are affected by analogous issues. For this reason, we need an additional global suspend callback that will reset the target state regardless of whether or not the suspend is successful.  Also, it is reasonable to rename the .set_target() callback, since it will be used for a different purpose on ACPI systems (due to ACPI 1.0x code ordering requirements). Introduce the global suspend callback .end() to be executed at the end of the suspend sequence and rename the .set_target() global suspend callback to .begin(). Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/main.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/power/main.c b/kernel/power/main.c index 050a6077ea4..d9bba452764 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -258,10 +258,10 @@ int suspend_devices_and_enter(suspend_state_t state) if (!suspend_ops) return -ENOSYS; - if (suspend_ops->set_target) { - error = suspend_ops->set_target(state); + if (suspend_ops->begin) { + error = suspend_ops->begin(state); if (error) - return error; + goto Close; } suspend_console(); error = device_suspend(PMSG_SUSPEND); @@ -294,6 +294,9 @@ int suspend_devices_and_enter(suspend_state_t state) device_resume(); Resume_console: resume_console(); + Close: + if (suspend_ops->end) + suspend_ops->end(); return error; } -- cgit v1.2.3 From caea99ef339af8e07cda8d03fa415e4b8820f400 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 8 Jan 2008 00:08:44 +0100 Subject: Hibernation: Introduce begin() and end() callbacks Introduce global hibernation callback .end() and rename global hibernation callback .start() to .begin(), in analogy with the recent modifications of the global suspend callbacks. Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/disk.c | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/power/disk.c b/kernel/power/disk.c index 64e42ab8b57..53c22d9cf57 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c @@ -54,8 +54,8 @@ static struct platform_hibernation_ops *hibernation_ops; void hibernation_set_ops(struct platform_hibernation_ops *ops) { - if (ops && !(ops->start && ops->pre_snapshot && ops->finish - && ops->prepare && ops->enter && ops->pre_restore + if (ops && !(ops->begin && ops->end && ops->pre_snapshot + && ops->prepare && ops->finish && ops->enter && ops->pre_restore && ops->restore_cleanup)) { WARN_ON(1); return; @@ -100,14 +100,25 @@ static int hibernation_test(int level) { return 0; } #endif /* !CONFIG_PM_DEBUG */ /** - * platform_start - tell the platform driver that we're starting + * platform_begin - tell the platform driver that we're starting * hibernation */ -static int platform_start(int platform_mode) +static int platform_begin(int platform_mode) { return (platform_mode && hibernation_ops) ? - hibernation_ops->start() : 0; + hibernation_ops->begin() : 0; +} + +/** + * platform_end - tell the platform driver that we've entered the + * working state + */ + +static void platform_end(int platform_mode) +{ + if (platform_mode && hibernation_ops) + hibernation_ops->end(); } /** @@ -237,9 +248,9 @@ int hibernation_snapshot(int platform_mode) if (error) return error; - error = platform_start(platform_mode); + error = platform_begin(platform_mode); if (error) - return error; + goto Close; suspend_console(); error = device_suspend(PMSG_FREEZE); @@ -272,6 +283,8 @@ int hibernation_snapshot(int platform_mode) device_resume(); Resume_console: resume_console(); + Close: + platform_end(platform_mode); return error; } @@ -373,9 +386,9 @@ int hibernation_platform_enter(void) * hibernation_ops->finish() before saving the image, so we should let * the firmware know that we're going to enter the sleep state after all */ - error = hibernation_ops->start(); + error = hibernation_ops->begin(); if (error) - return error; + goto Close; suspend_console(); error = device_suspend(PMSG_SUSPEND); @@ -409,6 +422,8 @@ int hibernation_platform_enter(void) device_resume(); Resume_console: resume_console(); + Close: + hibernation_ops->end(); return error; } -- cgit v1.2.3 From b28f508112c584cdfbb4d8a9489cc4b79dac68ee Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Tue, 15 Jan 2008 23:17:00 -0500 Subject: Suspend: Add config option to disable the freezer if architecture wants that This patch makes the freezer optional for suspend to allow the system to work (or not work) like the original PMU suspend. Signed-off-by: Johannes Berg Acked-by: Pavel Machek Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/Kconfig | 11 +++++++++++ kernel/power/main.c | 6 +++--- kernel/power/power.h | 22 ++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index f8153fda06b..ef9b802738a 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -104,6 +104,17 @@ config SUSPEND powered and thus its contents are preserved, such as the suspend-to-RAM state (e.g. the ACPI S3 state). +config SUSPEND_FREEZER + bool "Enable freezer for suspend to RAM/standby" \ + if ARCH_WANTS_FREEZER_CONTROL || BROKEN + depends on SUSPEND + default y + help + This allows you to turn off the freezer for suspend. If this is + done, no tasks are frozen for suspend to RAM/standby. + + Turning OFF this setting is NOT recommended! If in doubt, say Y. + config HIBERNATION bool "Hibernation (aka 'suspend to disk')" depends on PM && SWAP && ARCH_HIBERNATION_POSSIBLE diff --git a/kernel/power/main.c b/kernel/power/main.c index d9bba452764..e47214cfeb2 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -181,7 +181,7 @@ static int suspend_prepare(void) pm_prepare_console(); - if (freeze_processes()) { + if (suspend_freeze_processes()) { error = -EAGAIN; goto Thaw; } @@ -199,7 +199,7 @@ static int suspend_prepare(void) return 0; Thaw: - thaw_processes(); + suspend_thaw_processes(); pm_restore_console(); Finish: pm_notifier_call_chain(PM_POST_SUSPEND); @@ -308,7 +308,7 @@ int suspend_devices_and_enter(suspend_state_t state) */ static void suspend_finish(void) { - thaw_processes(); + suspend_thaw_processes(); pm_restore_console(); pm_notifier_call_chain(PM_POST_SUSPEND); } diff --git a/kernel/power/power.h b/kernel/power/power.h index 8ec5499c5ce..700f44ec840 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -1,6 +1,7 @@ #include #include #include +#include struct swsusp_info { struct new_utsname uts; @@ -203,3 +204,24 @@ enum { #define TEST_MAX (__TEST_AFTER_LAST - 1) extern int pm_test_level; + +#ifdef CONFIG_SUSPEND_FREEZER +static inline int suspend_freeze_processes(void) +{ + return freeze_processes(); +} + +static inline void suspend_thaw_processes(void) +{ + thaw_processes(); +} +#else +static inline int suspend_freeze_processes(void) +{ + return 0; +} + +static inline void suspend_thaw_processes(void) +{ +} +#endif -- cgit v1.2.3 From af258f516b3e4e214121f5d6d53cab32ce0d8010 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Fri, 11 Jan 2008 01:22:23 +0100 Subject: Suspend: Invoke suspend notifications after console switch In order to fix APM emulation it is necessary to enable apm-emulation notifications for suspends triggered in various ways via the suspend notifiers. However, this will cause the systems using APM emulation to lock up between X being needed to switch away from the VT and X already waiting for resume in the APM ioctl. This patch moves the console switch (if enabled) before the suspend notification (and after the resume notification) to avoid this issue. Signed-off-by: Johannes Berg Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/main.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/power/main.c b/kernel/power/main.c index e47214cfeb2..6a6d5eb3524 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -175,12 +175,12 @@ static int suspend_prepare(void) if (!suspend_ops || !suspend_ops->enter) return -EPERM; + pm_prepare_console(); + error = pm_notifier_call_chain(PM_SUSPEND_PREPARE); if (error) goto Finish; - pm_prepare_console(); - if (suspend_freeze_processes()) { error = -EAGAIN; goto Thaw; @@ -200,9 +200,9 @@ static int suspend_prepare(void) Thaw: suspend_thaw_processes(); - pm_restore_console(); Finish: pm_notifier_call_chain(PM_POST_SUSPEND); + pm_restore_console(); return error; } @@ -309,8 +309,8 @@ int suspend_devices_and_enter(suspend_state_t state) static void suspend_finish(void) { suspend_thaw_processes(); - pm_restore_console(); pm_notifier_call_chain(PM_POST_SUSPEND); + pm_restore_console(); } -- cgit v1.2.3 From 5a0a2f304612bd63948177fef05987f4bcaddcaf Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 11 Jan 2008 01:25:21 +0100 Subject: Hibernation: Invoke suspend notifications after console switch Following the recent change in the suspend code path, switch consoles before calling PM notifiers during hibernation. Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- kernel/power/disk.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/power/disk.c b/kernel/power/disk.c index 53c22d9cf57..d09da089517 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c @@ -458,20 +458,13 @@ static void power_down(void) while(1); } -static void unprepare_processes(void) -{ - thaw_processes(); - pm_restore_console(); -} - static int prepare_processes(void) { int error = 0; - pm_prepare_console(); if (freeze_processes()) { error = -EBUSY; - unprepare_processes(); + thaw_processes(); } return error; } @@ -491,6 +484,7 @@ int hibernate(void) goto Unlock; } + pm_prepare_console(); error = pm_notifier_call_chain(PM_HIBERNATION_PREPARE); if (error) goto Exit; @@ -530,11 +524,12 @@ int hibernate(void) swsusp_free(); } Thaw: - unprepare_processes(); + thaw_processes(); Finish: free_basic_memory_bitmaps(); Exit: pm_notifier_call_chain(PM_POST_HIBERNATION); + pm_restore_console(); atomic_inc(&snapshot_device_available); Unlock: mutex_unlock(&pm_mutex); @@ -603,6 +598,7 @@ static int software_resume(void) goto Unlock; } + pm_prepare_console(); error = pm_notifier_call_chain(PM_RESTORE_PREPARE); if (error) goto Finish; @@ -626,11 +622,12 @@ static int software_resume(void) printk(KERN_ERR "PM: Restore failed, recovering.\n"); swsusp_free(); - unprepare_processes(); + thaw_processes(); Done: free_basic_memory_bitmaps(); Finish: pm_notifier_call_chain(PM_POST_RESTORE); + pm_restore_console(); atomic_inc(&snapshot_device_available); /* For success case, the suspend path will release the lock */ Unlock: -- cgit v1.2.3 From ed50d6cbc394cd0966469d3e249353c9dd1d38b9 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Sat, 2 Feb 2008 00:23:08 +0100 Subject: debug: softlockup looping fix Rafael J. Wysocki reported weird, multi-seconds delays during suspend/resume and bisected it back to: commit 82a1fcb90287052aabfa235e7ffc693ea003fe69 Author: Ingo Molnar Date: Fri Jan 25 21:08:02 2008 +0100 softlockup: automatically detect hung TASK_UNINTERRUPTIBLE tasks fix it: - restore the old wakeup mechanism - fix break usage in do_each_thread() { } while_each_thread(). - fix the hotplug switch stmt, a fall-through case was broken. Bisected-by: Rafael J. Wysocki Signed-off-by: Peter Zijlstra Tested-by: Rafael J. Wysocki Signed-off-by: Ingo Molnar Acked-by: Rafael J. Wysocki Signed-off-by: Linus Torvalds --- kernel/softlockup.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/softlockup.c b/kernel/softlockup.c index c1d76552446..7c2da88db4e 100644 --- a/kernel/softlockup.c +++ b/kernel/softlockup.c @@ -101,6 +101,10 @@ void softlockup_tick(void) now = get_timestamp(this_cpu); + /* Wake up the high-prio watchdog task every second: */ + if (now > (touch_timestamp + 1)) + wake_up_process(per_cpu(watchdog_task, this_cpu)); + /* Warn about unreasonable delays: */ if (now <= (touch_timestamp + softlockup_thresh)) return; @@ -191,11 +195,11 @@ static void check_hung_uninterruptible_tasks(int this_cpu) read_lock(&tasklist_lock); do_each_thread(g, t) { if (!--max_count) - break; + goto unlock; if (t->state & TASK_UNINTERRUPTIBLE) check_hung_task(t, now); } while_each_thread(g, t); - + unlock: read_unlock(&tasklist_lock); } @@ -218,14 +222,19 @@ static int watchdog(void *__bind_cpu) * debug-printout triggers in softlockup_tick(). */ while (!kthread_should_stop()) { + set_current_state(TASK_INTERRUPTIBLE); touch_softlockup_watchdog(); - msleep_interruptible(10000); + schedule(); + + if (kthread_should_stop()) + break; if (this_cpu != check_cpu) continue; if (sysctl_hung_task_timeout_secs) check_hung_uninterruptible_tasks(this_cpu); + } return 0; @@ -259,13 +268,6 @@ cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) wake_up_process(per_cpu(watchdog_task, hotcpu)); break; #ifdef CONFIG_HOTPLUG_CPU - case CPU_UP_CANCELED: - case CPU_UP_CANCELED_FROZEN: - if (!per_cpu(watchdog_task, hotcpu)) - break; - /* Unbind so it can run. Fall thru. */ - kthread_bind(per_cpu(watchdog_task, hotcpu), - any_online_cpu(cpu_online_map)); case CPU_DOWN_PREPARE: case CPU_DOWN_PREPARE_FROZEN: if (hotcpu == check_cpu) { @@ -275,6 +277,14 @@ cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) check_cpu = any_online_cpu(temp_cpu_online_map); } break; + + case CPU_UP_CANCELED: + case CPU_UP_CANCELED_FROZEN: + if (!per_cpu(watchdog_task, hotcpu)) + break; + /* Unbind so it can run. Fall thru. */ + kthread_bind(per_cpu(watchdog_task, hotcpu), + any_online_cpu(cpu_online_map)); case CPU_DEAD: case CPU_DEAD_FROZEN: p = per_cpu(watchdog_task, hotcpu); -- cgit v1.2.3