diff options
Diffstat (limited to 'arch/sh')
31 files changed, 1125 insertions, 225 deletions
diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index e2bdd7b94fd..120bd31a46e 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -10,6 +10,7 @@ config SUPERH select EMBEDDED select HAVE_CLK select HAVE_IDE + select HAVE_LMB select HAVE_OPROFILE select HAVE_GENERIC_DMA_COHERENT select HAVE_IOREMAP_PROT if MMU @@ -31,6 +32,9 @@ config SUPERH32 select HAVE_FUNCTION_TRACER select HAVE_FTRACE_MCOUNT_RECORD select HAVE_DYNAMIC_FTRACE + select HAVE_FUNCTION_TRACE_MCOUNT_TEST + select HAVE_FTRACE_SYSCALLS + select HAVE_FUNCTION_GRAPH_TRACER select HAVE_ARCH_KGDB select ARCH_HIBERNATION_POSSIBLE if MMU diff --git a/arch/sh/Kconfig.debug b/arch/sh/Kconfig.debug index 39224b57c6e..52a132c24aa 100644 --- a/arch/sh/Kconfig.debug +++ b/arch/sh/Kconfig.debug @@ -123,4 +123,15 @@ config SH64_SR_WATCH bool "Debug: set SR.WATCH to enable hardware watchpoints and trace" depends on SUPERH64 +config STACK_DEBUG + bool "Enable diagnostic checks of the kernel stack" + depends on FUNCTION_TRACER + select DEBUG_STACKOVERFLOW + default n + help + This option allows checks to be performed on the kernel stack + at runtime. Saying Y here will add overhead to every function + call and will therefore incur a major performance hit. Most + users should say N. + endmenu diff --git a/arch/sh/include/asm/ftrace.h b/arch/sh/include/asm/ftrace.h index 8fea7d8c825..7e0bcc4d4a9 100644 --- a/arch/sh/include/asm/ftrace.h +++ b/arch/sh/include/asm/ftrace.h @@ -11,10 +11,13 @@ extern void mcount(void); #define MCOUNT_ADDR ((long)(mcount)) #ifdef CONFIG_DYNAMIC_FTRACE -#define CALLER_ADDR ((long)(ftrace_caller)) +#define CALL_ADDR ((long)(ftrace_call)) #define STUB_ADDR ((long)(ftrace_stub)) +#define GRAPH_ADDR ((long)(ftrace_graph_call)) +#define CALLER_ADDR ((long)(ftrace_caller)) -#define MCOUNT_INSN_OFFSET ((STUB_ADDR - CALLER_ADDR) >> 1) +#define MCOUNT_INSN_OFFSET ((STUB_ADDR - CALL_ADDR) - 4) +#define GRAPH_INSN_OFFSET ((CALLER_ADDR - GRAPH_ADDR) - 4) struct dyn_arch_ftrace { /* No extra data needed on sh */ diff --git a/arch/sh/include/asm/hwblk.h b/arch/sh/include/asm/hwblk.h new file mode 100644 index 00000000000..51a46f49663 --- /dev/null +++ b/arch/sh/include/asm/hwblk.h @@ -0,0 +1,61 @@ +#ifndef __ASM_SH_HWBLK_H +#define __ASM_SH_HWBLK_H + +#include <asm/clock.h> +#include <asm/io.h> + +#define HWBLK_AREA_FLAG_PARENT (1 << 0) /* valid parent */ + +#define HWBLK_AREA(_flags, _parent) \ +{ \ + .flags = _flags, \ + .parent = _parent, \ +} + +struct hwblk_area { + unsigned long cnt; + unsigned char parent; + unsigned char flags; +}; + +#define HWBLK(_mstp, _bit, _area) \ +{ \ + .mstp = (void __iomem *)_mstp, \ + .bit = _bit, \ + .area = _area, \ +} + +struct hwblk { + void __iomem *mstp; + unsigned char bit; + unsigned char area; + unsigned long cnt; +}; + +struct hwblk_info { + struct hwblk_area *areas; + int nr_areas; + struct hwblk *hwblks; + int nr_hwblks; +}; + +/* Should be defined by processor-specific code */ +int arch_hwblk_init(void); +int arch_hwblk_sleep_mode(void); + +int hwblk_register(struct hwblk_info *info); +int hwblk_init(void); + +/* allow clocks to enable and disable hardware blocks */ +#define SH_HWBLK_CLK(_name, _id, _parent, _hwblk, _flags) \ +{ \ + .name = _name, \ + .id = _id, \ + .parent = _parent, \ + .arch_flags = _hwblk, \ + .flags = _flags, \ +} + +int sh_hwblk_clk_register(struct clk *clks, int nr); + +#endif /* __ASM_SH_HWBLK_H */ diff --git a/arch/sh/include/asm/lmb.h b/arch/sh/include/asm/lmb.h new file mode 100644 index 00000000000..9b437f657ff --- /dev/null +++ b/arch/sh/include/asm/lmb.h @@ -0,0 +1,6 @@ +#ifndef __ASM_SH_LMB_H +#define __ASM_SH_LMB_H + +#define LMB_REAL_LIMIT 0 + +#endif /* __ASM_SH_LMB_H */ diff --git a/arch/sh/include/asm/suspend.h b/arch/sh/include/asm/suspend.h index b1b995370e7..5c8ea28ff7a 100644 --- a/arch/sh/include/asm/suspend.h +++ b/arch/sh/include/asm/suspend.h @@ -10,6 +10,15 @@ struct swsusp_arch_regs { struct pt_regs user_regs; unsigned long bank1_regs[8]; }; + +void sh_mobile_call_standby(unsigned long mode); + +#ifdef CONFIG_CPU_IDLE +void sh_mobile_setup_cpuidle(void); +#else +static inline void sh_mobile_setup_cpuidle(void) {} +#endif + #endif /* flags passed to assembly suspend code */ diff --git a/arch/sh/include/asm/syscall_32.h b/arch/sh/include/asm/syscall_32.h index 6f83f2cc45c..7d80df4f09c 100644 --- a/arch/sh/include/asm/syscall_32.h +++ b/arch/sh/include/asm/syscall_32.h @@ -65,6 +65,7 @@ static inline void syscall_get_arguments(struct task_struct *task, case 3: args[2] = regs->regs[6]; case 2: args[1] = regs->regs[5]; case 1: args[0] = regs->regs[4]; + case 0: break; default: BUG(); diff --git a/arch/sh/include/asm/thread_info.h b/arch/sh/include/asm/thread_info.h index d570ac2e5cb..5123bcaa850 100644 --- a/arch/sh/include/asm/thread_info.h +++ b/arch/sh/include/asm/thread_info.h @@ -97,7 +97,7 @@ static inline struct thread_info *current_thread_info(void) extern struct thread_info *alloc_thread_info(struct task_struct *tsk); extern void free_thread_info(struct thread_info *ti); - + #endif /* THREAD_SHIFT < PAGE_SHIFT */ #endif /* __ASSEMBLY__ */ @@ -116,6 +116,7 @@ extern void free_thread_info(struct thread_info *ti); #define TIF_SYSCALL_AUDIT 5 /* syscall auditing active */ #define TIF_SECCOMP 6 /* secure computing */ #define TIF_NOTIFY_RESUME 7 /* callback before returning to user */ +#define TIF_SYSCALL_FTRACE 8 /* for ftrace syscall instrumentation */ #define TIF_USEDFPU 16 /* FPU was used by this task this quantum (SMP) */ #define TIF_POLLING_NRFLAG 17 /* true if poll_idle() is polling TIF_NEED_RESCHED */ #define TIF_MEMDIE 18 @@ -129,25 +130,27 @@ extern void free_thread_info(struct thread_info *ti); #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_SECCOMP (1 << TIF_SECCOMP) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +#define _TIF_SYSCALL_FTRACE (1 << TIF_SYSCALL_FTRACE) #define _TIF_USEDFPU (1 << TIF_USEDFPU) #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) #define _TIF_FREEZE (1 << TIF_FREEZE) /* - * _TIF_ALLWORK_MASK and _TIF_WORK_MASK need to fit within a byte, or we + * _TIF_ALLWORK_MASK and _TIF_WORK_MASK need to fit within 2 bytes, or we * blow the tst immediate size constraints and need to fix up * arch/sh/kernel/entry-common.S. */ /* work to do in syscall trace */ #define _TIF_WORK_SYSCALL_MASK (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP | \ - _TIF_SYSCALL_AUDIT | _TIF_SECCOMP) + _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ + _TIF_SYSCALL_FTRACE) /* work to do on any return to u-space */ #define _TIF_ALLWORK_MASK (_TIF_SYSCALL_TRACE | _TIF_SIGPENDING | \ _TIF_NEED_RESCHED | _TIF_SYSCALL_AUDIT | \ _TIF_SINGLESTEP | _TIF_RESTORE_SIGMASK | \ - _TIF_NOTIFY_RESUME) + _TIF_NOTIFY_RESUME | _TIF_SYSCALL_FTRACE) /* work to do on interrupt/exception return */ #define _TIF_WORK_MASK (_TIF_ALLWORK_MASK & ~(_TIF_SYSCALL_TRACE | \ diff --git a/arch/sh/include/cpu-sh4/cpu/sh7722.h b/arch/sh/include/cpu-sh4/cpu/sh7722.h index 738ea43c503..48560407cbe 100644 --- a/arch/sh/include/cpu-sh4/cpu/sh7722.h +++ b/arch/sh/include/cpu-sh4/cpu/sh7722.h @@ -221,4 +221,18 @@ enum { GPIO_FN_KEYOUT3, GPIO_FN_KEYOUT4_IN6, GPIO_FN_KEYOUT5_IN5, }; +enum { + HWBLK_UNKNOWN = 0, + HWBLK_TLB, HWBLK_IC, HWBLK_OC, HWBLK_URAM, HWBLK_XYMEM, + HWBLK_INTC, HWBLK_DMAC, HWBLK_SHYWAY, HWBLK_HUDI, + HWBLK_UBC, HWBLK_TMU, HWBLK_CMT, HWBLK_RWDT, HWBLK_FLCTL, + HWBLK_SCIF0, HWBLK_SCIF1, HWBLK_SCIF2, HWBLK_SIO, + HWBLK_SIOF0, HWBLK_SIOF1, HWBLK_IIC, HWBLK_RTC, + HWBLK_TPU, HWBLK_IRDA, HWBLK_SDHI, HWBLK_SIM, HWBLK_KEYSC, + HWBLK_TSIF, HWBLK_USBF, HWBLK_2DG, HWBLK_SIU, HWBLK_VOU, + HWBLK_JPU, HWBLK_BEU, HWBLK_CEU, HWBLK_VEU, HWBLK_VPU, + HWBLK_LCDC, + HWBLK_NR, +}; + #endif /* __ASM_SH7722_H__ */ diff --git a/arch/sh/kernel/Makefile_32 b/arch/sh/kernel/Makefile_32 index 9411e3e31e6..94ed99b6800 100644 --- a/arch/sh/kernel/Makefile_32 +++ b/arch/sh/kernel/Makefile_32 @@ -29,6 +29,8 @@ obj-$(CONFIG_IO_TRAPPED) += io_trapped.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_GENERIC_GPIO) += gpio.o obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o +obj-$(CONFIG_FTRACE_SYSCALLS) += ftrace.o +obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o obj-$(CONFIG_DUMP_CODE) += disassemble.o obj-$(CONFIG_HIBERNATION) += swsusp.o diff --git a/arch/sh/kernel/asm-offsets.c b/arch/sh/kernel/asm-offsets.c index 99aceb28ee2..d218e808294 100644 --- a/arch/sh/kernel/asm-offsets.c +++ b/arch/sh/kernel/asm-offsets.c @@ -26,6 +26,7 @@ int main(void) DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); DEFINE(TI_PRE_COUNT, offsetof(struct thread_info, preempt_count)); DEFINE(TI_RESTART_BLOCK,offsetof(struct thread_info, restart_block)); + DEFINE(TI_SIZE, sizeof(struct thread_info)); #ifdef CONFIG_HIBERNATION DEFINE(PBE_ADDRESS, offsetof(struct pbe, address)); diff --git a/arch/sh/kernel/cpu/Makefile b/arch/sh/kernel/cpu/Makefile index eecad7cbd61..3d6b9312dc4 100644 --- a/arch/sh/kernel/cpu/Makefile +++ b/arch/sh/kernel/cpu/Makefile @@ -19,4 +19,4 @@ obj-$(CONFIG_UBC_WAKEUP) += ubc.o obj-$(CONFIG_SH_ADC) += adc.o obj-$(CONFIG_SH_CLK_CPG) += clock-cpg.o -obj-y += irq/ init.o clock.o +obj-y += irq/ init.o clock.o hwblk.o diff --git a/arch/sh/kernel/cpu/hwblk.c b/arch/sh/kernel/cpu/hwblk.c new file mode 100644 index 00000000000..7c3a73deff2 --- /dev/null +++ b/arch/sh/kernel/cpu/hwblk.c @@ -0,0 +1,130 @@ +#include <linux/clk.h> +#include <linux/compiler.h> +#include <linux/slab.h> +#include <linux/io.h> +#include <linux/spinlock.h> +#include <asm/suspend.h> +#include <asm/hwblk.h> +#include <asm/clock.h> + +static DEFINE_SPINLOCK(hwblk_lock); + +static void hwblk_area_inc(struct hwblk_info *info, int area) +{ + struct hwblk_area *hap = info->areas + area; + + hap->cnt++; + if (hap->cnt == 1) + if (hap->flags & HWBLK_AREA_FLAG_PARENT) + hwblk_area_inc(info, hap->parent); +} + +static void hwblk_area_dec(struct hwblk_info *info, int area) +{ + struct hwblk_area *hap = info->areas + area; + + if (hap->cnt == 1) + if (hap->flags & HWBLK_AREA_FLAG_PARENT) + hwblk_area_dec(info, hap->parent); + hap->cnt--; +} + +static void hwblk_enable(struct hwblk_info *info, int hwblk) +{ + struct hwblk *hp = info->hwblks + hwblk; + unsigned long tmp; + unsigned long flags; + + spin_lock_irqsave(&hwblk_lock, flags); + + hp->cnt++; + if (hp->cnt == 1) { + hwblk_area_inc(info, hp->area); + + tmp = __raw_readl(hp->mstp); + tmp &= ~(1 << hp->bit); + __raw_writel(tmp, hp->mstp); + } + + spin_unlock_irqrestore(&hwblk_lock, flags); +} + +static void hwblk_disable(struct hwblk_info *info, int hwblk) +{ + struct hwblk *hp = info->hwblks + hwblk; + unsigned long tmp; + unsigned long flags; + + spin_lock_irqsave(&hwblk_lock, flags); + + if (hp->cnt == 1) { + hwblk_area_dec(info, hp->area); + + tmp = __raw_readl(hp->mstp); + tmp |= 1 << hp->bit; + __raw_writel(tmp, hp->mstp); + } + hp->cnt--; + + spin_unlock_irqrestore(&hwblk_lock, flags); +} + +static struct hwblk_info *hwblk_info; + +int __init hwblk_register(struct hwblk_info *info) +{ + hwblk_info = info; + return 0; +} + +int __init __weak arch_hwblk_init(void) +{ + return 0; +} + +int __weak arch_hwblk_sleep_mode(void) +{ + return SUSP_SH_SLEEP; +} + +int __init hwblk_init(void) +{ + return arch_hwblk_init(); +} + +/* allow clocks to enable and disable hardware blocks */ +static int sh_hwblk_clk_enable(struct clk *clk) +{ + if (!hwblk_info) + return -ENOENT; + + hwblk_enable(hwblk_info, clk->arch_flags); + return 0; +} + +static void sh_hwblk_clk_disable(struct clk *clk) +{ + if (hwblk_info) + hwblk_disable(hwblk_info, clk->arch_flags); +} + +static struct clk_ops sh_hwblk_clk_ops = { + .enable = sh_hwblk_clk_enable, + .disable = sh_hwblk_clk_disable, + .recalc = followparent_recalc, +}; + +int __init sh_hwblk_clk_register(struct clk *clks, int nr) +{ + struct clk *clkp; + int ret = 0; + int k; + + for (k = 0; !ret && (k < nr); k++) { + clkp = clks + k; + clkp->ops = &sh_hwblk_clk_ops; + ret |= clk_register(clkp); + } + + return ret; +} diff --git a/arch/sh/kernel/cpu/sh4a/Makefile b/arch/sh/kernel/cpu/sh4a/Makefile index ebdd391d5f4..3cafda69637 100644 --- a/arch/sh/kernel/cpu/sh4a/Makefile +++ b/arch/sh/kernel/cpu/sh4a/Makefile @@ -25,7 +25,7 @@ clock-$(CONFIG_CPU_SUBTYPE_SH7780) := clock-sh7780.o clock-$(CONFIG_CPU_SUBTYPE_SH7785) := clock-sh7785.o clock-$(CONFIG_CPU_SUBTYPE_SH7786) := clock-sh7786.o clock-$(CONFIG_CPU_SUBTYPE_SH7343) := clock-sh7343.o -clock-$(CONFIG_CPU_SUBTYPE_SH7722) := clock-sh7722.o +clock-$(CONFIG_CPU_SUBTYPE_SH7722) := clock-sh7722.o hwblk-sh7722.o clock-$(CONFIG_CPU_SUBTYPE_SH7723) := clock-sh7723.o clock-$(CONFIG_CPU_SUBTYPE_SH7724) := clock-sh7724.o clock-$(CONFIG_CPU_SUBTYPE_SH7366) := clock-sh7366.o diff --git a/arch/sh/kernel/cpu/sh4a/clock-sh7722.c b/arch/sh/kernel/cpu/sh4a/clock-sh7722.c index 40f859354f7..1fa9e1dd1cc 100644 --- a/arch/sh/kernel/cpu/sh4a/clock-sh7722.c +++ b/arch/sh/kernel/cpu/sh4a/clock-sh7722.c @@ -22,6 +22,8 @@ #include <linux/kernel.h> #include <linux/io.h> #include <asm/clock.h> +#include <asm/hwblk.h> +#include <cpu/sh7722.h> /* SH7722 registers */ #define FRQCR 0xa4150000 @@ -140,35 +142,37 @@ struct clk div6_clks[] = { SH_CLK_DIV6("video_clk", &pll_clk, VCLKCR, 0), }; -#define MSTP(_str, _parent, _reg, _bit, _flags) \ - SH_CLK_MSTP32(_str, -1, _parent, _reg, _bit, _flags) +#define R_CLK &r_clk +#define P_CLK &div4_clks[DIV4_P] +#define B_CLK &div4_clks[DIV4_B] +#define U_CLK &div4_clks[DIV4_U] static struct clk mstp_clks[] = { - MSTP("uram0", &div4_clks[DIV4_U], MSTPCR0, 28, CLK_ENABLE_ON_INIT), - MSTP("xymem0", &div4_clks[DIV4_B], MSTPCR0, 26, CLK_ENABLE_ON_INIT), - MSTP("tmu0", &div4_clks[DIV4_P], MSTPCR0, 15, 0), - MSTP("cmt0", &r_clk, MSTPCR0, 14, 0), - MSTP("rwdt0", &r_clk, MSTPCR0, 13, 0), - MSTP("flctl0", &div4_clks[DIV4_P], MSTPCR0, 10, 0), - MSTP("scif0", &div4_clks[DIV4_P], MSTPCR0, 7, 0), - MSTP("scif1", &div4_clks[DIV4_P], MSTPCR0, 6, 0), - MSTP("scif2", &div4_clks[DIV4_P], MSTPCR0, 5, 0), - - MSTP("i2c0", &div4_clks[DIV4_P], MSTPCR1, 9, 0), - MSTP("rtc0", &r_clk, MSTPCR1, 8, 0), - - MSTP("sdhi0", &div4_clks[DIV4_P], MSTPCR2, 18, 0), - MSTP("keysc0", &r_clk, MSTPCR2, 14, 0), - MSTP("usbf0", &div4_clks[DIV4_P], MSTPCR2, 11, 0), - MSTP("2dg0", &div4_clks[DIV4_B], MSTPCR2, 9, 0), - MSTP("siu0", &div4_clks[DIV4_B], MSTPCR2, 8, 0), - MSTP("vou0", &div4_clks[DIV4_B], MSTPCR2, 5, 0), - MSTP("jpu0", &div4_clks[DIV4_B], MSTPCR2, 6, CLK_ENABLE_ON_INIT), - MSTP("beu0", &div4_clks[DIV4_B], MSTPCR2, 4, 0), - MSTP("ceu0", &div4_clks[DIV4_B], MSTPCR2, 3, 0), - MSTP("veu0", &div4_clks[DIV4_B], MSTPCR2, 2, CLK_ENABLE_ON_INIT), - MSTP("vpu0", &div4_clks[DIV4_B], MSTPCR2, 1, CLK_ENABLE_ON_INIT), - MSTP("lcdc0", &div4_clks[DIV4_B], MSTPCR2, 0, 0), + SH_HWBLK_CLK("uram0", -1, U_CLK, HWBLK_URAM, CLK_ENABLE_ON_INIT), + SH_HWBLK_CLK("xymem0", -1, B_CLK, HWBLK_XYMEM, CLK_ENABLE_ON_INIT), + SH_HWBLK_CLK("tmu0", -1, P_CLK, HWBLK_TMU, 0), + SH_HWBLK_CLK("cmt0", -1, R_CLK, HWBLK_CMT, 0), + SH_HWBLK_CLK("rwdt0", -1, R_CLK, HWBLK_RWDT, 0), + SH_HWBLK_CLK("flctl0", -1, P_CLK, HWBLK_FLCTL, 0), + SH_HWBLK_CLK("scif0", -1, P_CLK, HWBLK_SCIF0, 0), + SH_HWBLK_CLK("scif1", -1, P_CLK, HWBLK_SCIF1, 0), + SH_HWBLK_CLK("scif2", -1, P_CLK, HWBLK_SCIF2, 0), + + SH_HWBLK_CLK("i2c0", -1, P_CLK, HWBLK_IIC, 0), + SH_HWBLK_CLK("rtc0", -1, R_CLK, HWBLK_RTC, 0), + + SH_HWBLK_CLK("sdhi0", -1, P_CLK, HWBLK_SDHI, 0), + SH_HWBLK_CLK("keysc0", -1, R_CLK, HWBLK_KEYSC, 0), + SH_HWBLK_CLK("usbf0", -1, P_CLK, HWBLK_USBF, 0), + SH_HWBLK_CLK("2dg0", -1, B_CLK, HWBLK_2DG, 0), + SH_HWBLK_CLK("siu0", -1, B_CLK, HWBLK_SIU, 0), + SH_HWBLK_CLK("vou0", -1, B_CLK, HWBLK_VOU, 0), + SH_HWBLK_CLK("jpu0", -1, B_CLK, HWBLK_JPU, CLK_ENABLE_ON_INIT), + SH_HWBLK_CLK("beu0", -1, B_CLK, HWBLK_BEU, 0), + SH_HWBLK_CLK("ceu0", -1, B_CLK, HWBLK_CEU, 0), + SH_HWBLK_CLK("veu0", -1, B_CLK, HWBLK_VEU, CLK_ENABLE_ON_INIT), + SH_HWBLK_CLK("vpu0", -1, B_CLK, HWBLK_VPU, CLK_ENABLE_ON_INIT), + SH_HWBLK_CLK("lcdc0", -1, P_CLK, HWBLK_LCDC, 0), }; int __init arch_clk_init(void) @@ -191,7 +195,7 @@ int __init arch_clk_init(void) ret = sh_clk_div6_register(div6_clks, ARRAY_SIZE(div6_clks)); if (!ret) - ret = sh_clk_mstp32_register(mstp_clks, ARRAY_SIZE(mstp_clks)); + ret = sh_hwblk_clk_register(mstp_clks, ARRAY_SIZE(mstp_clks)); return ret; } diff --git a/arch/sh/kernel/cpu/sh4a/hwblk-sh7722.c b/arch/sh/kernel/cpu/sh4a/hwblk-sh7722.c new file mode 100644 index 00000000000..00a1c02d82b --- /dev/null +++ b/arch/sh/kernel/cpu/sh4a/hwblk-sh7722.c @@ -0,0 +1,106 @@ +/* + * arch/sh/kernel/cpu/sh4a/hwblk-sh7722.c + * + * SH7722 hardware block support + * + * Copyright (C) 2009 Magnus Damm + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/io.h> +#include <asm/suspend.h> +#include <asm/hwblk.h> +#include <cpu/sh7722.h> + +/* SH7722 registers */ +#define MSTPCR0 0xa4150030 +#define MSTPCR1 0xa4150034 +#define MSTPCR2 0xa4150038 + +/* SH7722 Power Domains */ +enum { CORE_AREA, SUB_AREA, CORE_AREA_BM }; +static struct hwblk_area sh7722_hwblk_area[] = { + [CORE_AREA] = HWBLK_AREA(0, 0), + [CORE_AREA_BM] = HWBLK_AREA(HWBLK_AREA_FLAG_PARENT, CORE_AREA), + [SUB_AREA] = HWBLK_AREA(0, 0), +}; + +/* Table mapping HWBLK to Module Stop Bit and Power Domain */ +static struct hwblk sh7722_hwblk[HWBLK_NR] = { + [HWBLK_TLB] = HWBLK(MSTPCR0, 31, CORE_AREA), + [HWBLK_IC] = HWBLK(MSTPCR0, 30, CORE_AREA), + [HWBLK_OC] = HWBLK(MSTPCR0, 29, CORE_AREA), + [HWBLK_URAM] = HWBLK(MSTPCR0, 28, CORE_AREA), + [HWBLK_XYMEM] = HWBLK(MSTPCR0, 26, CORE_AREA), + [HWBLK_INTC] = HWBLK(MSTPCR0, 22, CORE_AREA), + [HWBLK_DMAC] = HWBLK(MSTPCR0, 21, CORE_AREA_BM), + [HWBLK_SHYWAY] = HWBLK(MSTPCR0, 20, CORE_AREA), + [HWBLK_HUDI] = HWBLK(MSTPCR0, 19, CORE_AREA), + [HWBLK_UBC] = HWBLK(MSTPCR0, 17, CORE_AREA), + [HWBLK_TMU] = HWBLK(MSTPCR0, 15, CORE_AREA), + [HWBLK_CMT] = HWBLK(MSTPCR0, 14, SUB_AREA), + [HWBLK_RWDT] = HWBLK(MSTPCR0, 13, SUB_AREA), + [HWBLK_FLCTL] = HWBLK(MSTPCR0, 10, CORE_AREA), + [HWBLK_SCIF0] = HWBLK(MSTPCR0, 7, CORE_AREA), + [HWBLK_SCIF1] = HWBLK(MSTPCR0, 6, CORE_AREA), + [HWBLK_SCIF2] = HWBLK(MSTPCR0, 5, CORE_AREA), + [HWBLK_SIO] = HWBLK(MSTPCR0, 3, CORE_AREA), + [HWBLK_SIOF0] = HWBLK(MSTPCR0, 2, CORE_AREA), + [HWBLK_SIOF1] = HWBLK(MSTPCR0, 1, CORE_AREA), + + [HWBLK_IIC] = HWBLK(MSTPCR1, 9, CORE_AREA), + [HWBLK_RTC] = HWBLK(MSTPCR1, 8, SUB_AREA), + + [HWBLK_TPU] = HWBLK(MSTPCR2, 25, CORE_AREA), + [HWBLK_IRDA] = HWBLK(MSTPCR2, 24, CORE_AREA), + [HWBLK_SDHI] = HWBLK(MSTPCR2, 18, CORE_AREA), + [HWBLK_SIM] = HWBLK(MSTPCR2, 16, CORE_AREA), + [HWBLK_KEYSC] = HWBLK(MSTPCR2, 14, SUB_AREA), + [HWBLK_TSIF] = HWBLK(MSTPCR2, 13, SUB_AREA), + [HWBLK_USBF] = HWBLK(MSTPCR2, 11, CORE_AREA), + [HWBLK_2DG] = HWBLK(MSTPCR2, 9, CORE_AREA_BM), + [HWBLK_SIU] = HWBLK(MSTPCR2, 8, CORE_AREA), + [HWBLK_JPU] = HWBLK(MSTPCR2, 6, CORE_AREA_BM), + [HWBLK_VOU] = HWBLK(MSTPCR2, 5, CORE_AREA_BM), + [HWBLK_BEU] = HWBLK(MSTPCR2, 4, CORE_AREA_BM), + [HWBLK_CEU] = HWBLK(MSTPCR2, 3, CORE_AREA_BM), + [HWBLK_VEU] = HWBLK(MSTPCR2, 2, CORE_AREA_BM), + [HWBLK_VPU] = HWBLK(MSTPCR2, 1, CORE_AREA_BM), + [HWBLK_LCDC] = HWBLK(MSTPCR2, 0, CORE_AREA_BM), +}; + +static struct hwblk_info sh7722_hwblk_info = { + .areas = sh7722_hwblk_area, + .nr_areas = ARRAY_SIZE(sh7722_hwblk_area), + .hwblks = sh7722_hwblk, + .nr_hwblks = ARRAY_SIZE(sh7722_hwblk), +}; + +int arch_hwblk_sleep_mode(void) +{ + if (!sh7722_hwblk_area[CORE_AREA].cnt) + return SUSP_SH_STANDBY | SUSP_SH_SF; + + if (!sh7722_hwblk_area[CORE_AREA_BM].cnt) + return SUSP_SH_SLEEP | SUSP_SH_SF; + + return SUSP_SH_SLEEP; +} + +int __init arch_hwblk_init(void) +{ + return hwblk_register(&sh7722_hwblk_info); +} diff --git a/arch/sh/kernel/cpu/shmobile/Makefile b/arch/sh/kernel/cpu/shmobile/Makefile index 08bfa7c7db2..e8a5111e848 100644 --- a/arch/sh/kernel/cpu/shmobile/Makefile +++ b/arch/sh/kernel/cpu/shmobile/Makefile @@ -4,3 +4,4 @@ # Power Management & Sleep mode obj-$(CONFIG_PM) += pm.o sleep.o +obj-$(CONFIG_CPU_IDLE) += cpuidle.o diff --git a/arch/sh/kernel/cpu/shmobile/cpuidle.c b/arch/sh/kernel/cpu/shmobile/cpuidle.c new file mode 100644 index 00000000000..4afdd975cc6 --- /dev/null +++ b/arch/sh/kernel/cpu/shmobile/cpuidle.c @@ -0,0 +1,102 @@ +/* + * arch/sh/kernel/cpu/shmobile/cpuidle.c + * + * Cpuidle support code for SuperH Mobile + * + * Copyright (C) 2009 Magnus Damm + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/io.h> +#include <linux/suspend.h> +#include <linux/cpuidle.h> +#include <asm/suspend.h> +#include <asm/uaccess.h> +#include <asm/hwblk.h> + +static unsigned long cpuidle_mode[] = { + SUSP_SH_SLEEP, /* regular sleep mode */ + SUSP_SH_SLEEP | SUSP_SH_SF, /* sleep mode + self refresh */ +}; + +static int cpuidle_sleep_enter(struct cpuidle_device *dev, + struct cpuidle_state *state) +{ + unsigned long allowed_mode = arch_hwblk_sleep_mode(); + ktime_t before, after; + int requested_state = state - &dev->states[0]; + int allowed_state; + int k; + + /* convert allowed mode to allowed state */ + for (k = ARRAY_SIZE(cpuidle_mode) - 1; k > 0; k--) + if (cpuidle_mode[k] == allowed_mode) + break; + + allowed_state = k; + + /* take the following into account for sleep mode selection: + * - allowed_state: best mode allowed by hardware (clock deps) + * - requested_state: best mode allowed by software (latencies) + */ + k = min_t(int, allowed_state, requested_state); + + dev->last_state = &dev->states[k]; + before = ktime_get(); + sh_mobile_call_standby(cpuidle_mode[k]); + after = ktime_get(); + return ktime_to_ns(ktime_sub(after, before)) >> 10; +} + +static struct cpuidle_device cpuidle_dev; +static struct cpuidle_driver cpuidle_driver = { + .name = "sh_idle", + .owner = THIS_MODULE, +}; + +void sh_mobile_setup_cpuidle(void) +{ + struct cpuidle_device *dev = &cpuidle_dev; + struct cpuidle_state *state; + int i; + + cpuidle_register_driver(&cpuidle_driver); + + for (i = 0; i < CPUIDLE_STATE_MAX; i++) { + dev->states[i].name[0] = '\0'; + dev->states[i].desc[0] = '\0'; + } + + i = CPUIDLE_DRIVER_STATE_START; + + state = &dev->states[i++]; + snprintf(state->name, CPUIDLE_NAME_LEN, "C0"); + strncpy(state->desc, "SuperH Sleep Mode", CPUIDLE_DESC_LEN); + state->exit_latency = 1; + state->target_residency = 1 * 2; + state->power_usage = 3; + state->flags = 0; + state->flags |= CPUIDLE_FLAG_SHALLOW; + state->flags |= CPUIDLE_FLAG_TIME_VALID; + state->enter = cpuidle_sleep_enter; + + dev->safe_state = state; + + state = &dev->states[i++]; + snprintf(state->name, CPUIDLE_NAME_LEN, "C1"); + strncpy(state->desc, "SuperH Sleep Mode [SF]", CPUIDLE_DESC_LEN); + state->exit_latency = 100; + state->target_residency = 1 * 2; + state->power_usage = 1; + state->flags = 0; + state->flags |= CPUIDLE_FLAG_TIME_VALID; + state->enter = cpuidle_sleep_enter; + + dev->state_count = i; + + cpuidle_register_device(dev); +} diff --git a/arch/sh/kernel/cpu/shmobile/pm.c b/arch/sh/kernel/cpu/shmobile/pm.c index 8c067adf683..de078d24ce5 100644 --- a/arch/sh/kernel/cpu/shmobile/pm.c +++ b/arch/sh/kernel/cpu/shmobile/pm.c @@ -1,5 +1,5 @@ /* - * arch/sh/kernel/cpu/sh4a/pm-sh_mobile.c + * arch/sh/kernel/cpu/shmobile/pm.c * * Power management support code for SuperH Mobile * @@ -32,20 +32,17 @@ * * R-standby mode is unsupported, but will be added in the future * U-standby mode is low priority since it needs bootloader hacks - * - * All modes should be tied in with cpuidle. But before that can - * happen we need to keep track of enabled hardware blocks so we - * can avoid entering sleep modes that stop clocks to hardware - * blocks that are in use even though the cpu core is idle. */ +#define ILRAM_BASE 0xe5200000 + extern const unsigned char sh_mobile_standby[]; extern const unsigned int sh_mobile_standby_size; -static void sh_mobile_call_standby(unsigned long mode) +void sh_mobile_call_standby(unsigned long mode) { extern void *vbr_base; - void *onchip_mem = (void *)0xe5200000; /* ILRAM */ + void *onchip_mem = (void *)ILRAM_BASE; void (*standby_onchip_mem)(unsigned long) = onchip_mem; /* Note: Wake up from sleep may generate exceptions! @@ -55,11 +52,6 @@ static void sh_mobile_call_standby(unsigned long mode) if (mode & SUSP_SH_SF) asm volatile("ldc %0, vbr" : : "r" (onchip_mem) : "memory"); - /* Copy the assembly snippet to the otherwise ununsed ILRAM */ - memcpy(onchip_mem, sh_mobile_standby, sh_mobile_standby_size); - wmb(); - ctrl_barrier(); - /* Let assembly snippet in on-chip memory handle the rest */ standby_onchip_mem(mode); @@ -85,7 +77,15 @@ static struct platform_suspend_ops sh_pm_ops = { static int __init sh_pm_init(void) { + void *onchip_mem = (void *)ILRAM_BASE; + + /* Copy the assembly snippet to the otherwise ununsed ILRAM */ + memcpy(onchip_mem, sh_mobile_standby, sh_mobile_standby_size); + wmb(); + ctrl_barrier(); + suspend_set_ops(&sh_pm_ops); + sh_mobile_setup_cpuidle(); return 0; } diff --git a/arch/sh/kernel/entry-common.S b/arch/sh/kernel/entry-common.S index d62359cfbbe..d62175650c5 100644 --- a/arch/sh/kernel/entry-common.S +++ b/arch/sh/kernel/entry-common.S @@ -131,7 +131,7 @@ ENTRY(resume_userspace) nop #endif mov.l @(TI_FLAGS,r8), r0 ! current_thread_info->flags - tst #_TIF_WORK_MASK, r0 + tst #(_TIF_WORK_MASK & 0xff), r0 bt/s __restore_all tst #_TIF_NEED_RESCHED, r0 @@ -163,7 +163,7 @@ work_resched: #endif ! mov.l @(TI_FLAGS,r8), r0 ! current_thread_info->flags - tst #_TIF_WORK_MASK, r0 + tst #(_TIF_WORK_MASK & 0xff), r0 bt __restore_all bra work_pending tst #_TIF_NEED_RESCHED, r0 @@ -181,7 +181,7 @@ work_resched: syscall_exit_work: ! r0: current_thread_info->flags ! r8: current_thread_info - tst #_TIF_WORK_SYSCALL_MASK, r0 + tst #(_TIF_WORK_SYSCALL_MASK & 0xff), r0 bt/s work_pending tst #_TIF_NEED_RESCHED, r0 #ifdef CONFIG_TRACE_IRQFLAGS @@ -331,8 +331,12 @@ ENTRY(system_call) ! get_current_thread_info r8, r10 mov.l @(TI_FLAGS,r8), r8 - mov #_TIF_WORK_SYSCALL_MASK, r10 + mov #(_TIF_WORK_SYSCALL_MASK & 0xff), r10 + mov #(_TIF_WORK_SYSCALL_MASK >> 8), r9 tst r10, r8 + shll8 r9 + bf syscall_trace_entry + tst r9, r8 bf syscall_trace_entry ! mov.l 2f, r8 ! Number of syscalls @@ -359,7 +363,11 @@ syscall_exit: ! get_current_thread_info r8, r0 mov.l @(TI_FLAGS,r8), r0 ! current_thread_info->flags - tst #_TIF_ALLWORK_MASK, r0 + tst #(_TIF_ALLWORK_MASK & 0xff), r0 + mov #(_TIF_ALLWORK_MASK >> 8), r1 + bf syscall_exit_work + shlr8 r0 + tst r0, r1 bf syscall_exit_work bra __restore_all nop diff --git a/arch/sh/kernel/ftrace.c b/arch/sh/kernel/ftrace.c index 066f37dc32a..6647dfcb781 100644 --- a/arch/sh/kernel/ftrace.c +++ b/arch/sh/kernel/ftrace.c @@ -16,9 +16,13 @@ #include <linux/string.h> #include <linux/init.h> #include <linux/io.h> +#include <linux/kernel.h> #include <asm/ftrace.h> #include <asm/cacheflush.h> +#include <asm/unistd.h> +#include <trace/syscall.h> +#ifdef CONFIG_DYNAMIC_FTRACE static unsigned char ftrace_replaced_code[MCOUNT_INSN_SIZE]; static unsigned char ftrace_nop[4]; @@ -131,3 +135,189 @@ int __init ftrace_dyn_arch_init(void *data) return 0; } +#endif /* CONFIG_DYNAMIC_FTRACE */ + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +#ifdef CONFIG_DYNAMIC_FTRACE +extern void ftrace_graph_call(void); + +static int ftrace_mod(unsigned long ip, unsigned long old_addr, + unsigned long new_addr) +{ + unsigned char code[MCOUNT_INSN_SIZE]; + + if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE)) + return -EFAULT; + + if (old_addr != __raw_readl((unsigned long *)code)) + return -EINVAL; + + __raw_writel(new_addr, ip); + return 0; +} + +int ftrace_enable_ftrace_graph_caller(void) +{ + unsigned long ip, old_addr, new_addr; + + ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET; + old_addr = (unsigned long)(&skip_trace); + new_addr = (unsigned long)(&ftrace_graph_caller); + + return ftrace_mod(ip, old_addr, new_addr); +} + +int ftrace_disable_ftrace_graph_caller(void) +{ + unsigned long ip, old_addr, new_addr; + + ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET; + old_addr = (unsigned long)(&ftrace_graph_caller); + new_addr = (unsigned long)(&skip_trace); + + return ftrace_mod(ip, old_addr, new_addr); +} +#endif /* CONFIG_DYNAMIC_FTRACE */ + +/* + * Hook the return address and push it in the stack of return addrs + * in the current thread info. + * + * This is the main routine for the function graph tracer. The function + * graph tracer essentially works like this: + * + * parent is the stack address containing self_addr's return address. + * We pull the real return address out of parent and store it in + * current's ret_stack. Then, we replace the return address on the stack + * with the address of return_to_handler. self_addr is the function that + * called mcount. + * + * When self_addr returns, it will jump to return_to_handler which calls + * ftrace_return_to_handler. ftrace_return_to_handler will pull the real + * return address off of current's ret_stack and jump to it. + */ +void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr) +{ + unsigned long old; + int faulted, err; + struct ftrace_graph_ent trace; + unsigned long return_hooker = (unsigned long)&return_to_handler; + + if (unlikely(atomic_read(¤t->tracing_graph_pause))) + return; + + /* + * Protect against fault, even if it shouldn't + * happen. This tool is too much intrusive to + * ignore such a protection. + */ + __asm__ __volatile__( + "1: \n\t" + "mov.l @%2, %0 \n\t" + "2: \n\t" + "mov.l %3, @%2 \n\t" + "mov #0, %1 \n\t" + "3: \n\t" + ".section .fixup, \"ax\" \n\t" + "4: \n\t" + "mov.l 5f, %0 \n\t" + "jmp @%0 \n\t" + " mov #1, %1 \n\t" + ".balign 4 \n\t" + "5: .long 3b \n\t" + ".previous \n\t" + ".section __ex_table,\"a\" \n\t" + ".long 1b, 4b \n\t" + ".long 2b, 4b \n\t" + ".previous \n\t" + : "=&r" (old), "=r" (faulted) + : "r" (parent), "r" (return_hooker) + ); + + if (unlikely(faulted)) { + ftrace_graph_stop(); + WARN_ON(1); + return; + } + + err = ftrace_push_return_trace(old, self_addr, &trace.depth, 0); + if (err == -EBUSY) { + __raw_writel(old, parent); + return; + } + + trace.func = self_addr; + + /* Only trace if the calling function expects to */ + if (!ftrace_graph_entry(&trace)) { + current->curr_ret_stack--; + __raw_writel(old, parent); + } +} +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ + +#ifdef CONFIG_FTRACE_SYSCALLS + +extern unsigned long __start_syscalls_metadata[]; +extern unsigned long __stop_syscalls_metadata[]; +extern unsigned long *sys_call_table; + +static struct syscall_metadata **syscalls_metadata; + +static struct syscall_metadata *find_syscall_meta(unsigned long *syscall) +{ + struct syscall_metadata *start; + struct syscall_metadata *stop; + char str[KSYM_SYMBOL_LEN]; + + + start = (struct syscall_metadata *)__start_syscalls_metadata; + stop = (struct syscall_metadata *)__stop_syscalls_metadata; + kallsyms_lookup((unsigned long) syscall, NULL, NULL, NULL, str); + + for ( ; start < stop; start++) { + if (start->name && !strcmp(start->name, str)) + return start; + } + + return NULL; +} + +#define FTRACE_SYSCALL_MAX (NR_syscalls - 1) + +struct syscall_metadata *syscall_nr_to_meta(int nr) +{ + if (!syscalls_metadata || nr >= FTRACE_SYSCALL_MAX || nr < 0) + return NULL; + + return syscalls_metadata[nr]; +} + +void arch_init_ftrace_syscalls(void) +{ + int i; + struct syscall_metadata *meta; + unsigned long **psys_syscall_table = &sys_call_table; + static atomic_t refs; + + if (atomic_inc_return(&refs) != 1) + goto end; + + syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) * + FTRACE_SYSCALL_MAX, GFP_KERNEL); + if (!syscalls_metadata) { + WARN_ON(1); + return; + } + + for (i = 0; i < FTRACE_SYSCALL_MAX; i++) { + meta = find_syscall_meta(psys_syscall_table[i]); + syscalls_metadata[i] = meta; + } + return; + + /* Paranoid: avoid overflow */ +end: + atomic_dec(&refs); +} +#endif /* CONFIG_FTRACE_SYSCALLS */ diff --git a/arch/sh/kernel/process_32.c b/arch/sh/kernel/process_32.c index 92d7740faab..9fee977f176 100644 --- a/arch/sh/kernel/process_32.c +++ b/arch/sh/kernel/process_32.c @@ -23,6 +23,7 @@ #include <linux/tick.h> #include <linux/reboot.h> #include <linux/fs.h> +#include <linux/ftrace.h> #include <linux/preempt.h> #include <asm/uaccess.h> #include <asm/mmu_context.h> @@ -264,8 +265,8 @@ static void ubc_set_tracing(int asid, unsigned long pc) * switch_to(x,y) should switch tasks from x to y. * */ -struct task_struct *__switch_to(struct task_struct *prev, - struct task_struct *next) +__notrace_funcgraph struct task_struct * +__switch_to(struct task_struct *prev, struct task_struct *next) { #if defined(CONFIG_SH_FPU) unlazy_fpu(prev, task_pt_regs(prev)); diff --git a/arch/sh/kernel/ptrace_32.c b/arch/sh/kernel/ptrace_32.c index 3392e835a37..c198eceaee9 100644 --- a/arch/sh/kernel/ptrace_32.c +++ b/arch/sh/kernel/ptrace_32.c @@ -34,6 +34,8 @@ #include <asm/syscalls.h> #include <asm/fpu.h> +#include <trace/syscall.h> + /* * This routine will get a word off of the process kernel stack. */ @@ -459,6 +461,9 @@ asmlinkage long do_syscall_trace_enter(struct pt_regs *regs) */ ret = -1L; + if (unlikely(test_thread_flag(TIF_SYSCALL_FTRACE))) + ftrace_syscall_enter(regs); + if (unlikely(current->audit_context)) audit_syscall_entry(audit_arch(), regs->regs[3], regs->regs[4], regs->regs[5], @@ -475,6 +480,9 @@ asmlinkage void do_syscall_trace_leave(struct pt_regs *regs) audit_syscall_exit(AUDITSC_RESULT(regs->regs[0]), regs->regs[0]); + if (unlikely(test_thread_flag(TIF_SYSCALL_FTRACE))) + ftrace_syscall_exit(regs); + step = test_thread_flag(TIF_SINGLESTEP); if (step || test_thread_flag(TIF_SYSCALL_TRACE)) tracehook_report_syscall_exit(regs, step); diff --git a/arch/sh/kernel/setup.c b/arch/sh/kernel/setup.c index dd38338553e..ceb409bf774 100644 --- a/arch/sh/kernel/setup.c +++ b/arch/sh/kernel/setup.c @@ -30,6 +30,7 @@ #include <linux/clk.h> #include <linux/delay.h> #include <linux/platform_device.h> +#include <linux/lmb.h> #include <asm/uaccess.h> #include <asm/io.h> #include <asm/page.h> @@ -233,39 +234,45 @@ void __init __add_active_range(unsigned int nid, unsigned long start_pfn, void __init setup_bootmem_allocator(unsigned long free_pfn) { unsigned long bootmap_size; + unsigned long bootmap_pages, bootmem_paddr; + u64 total_pages = (lmb_end_of_DRAM() - __MEMORY_START) >> PAGE_SHIFT; + int i; + + bootmap_pages = bootmem_bootmap_pages(total_pages); + + bootmem_paddr = lmb_alloc(bootmap_pages << PAGE_SHIFT, PAGE_SIZE); /* * Find a proper area for the bootmem bitmap. After this * bootstrap step all allocations (until the page allocator * is intact) must be done via bootmem_alloc(). */ - bootmap_size = init_bootmem_node(NODE_DATA(0), free_pfn, + bootmap_size = init_bootmem_node(NODE_DATA(0), + bootmem_paddr >> PAGE_SHIFT, min_low_pfn, max_low_pfn); - __add_active_range(0, min_low_pfn, max_low_pfn); - register_bootmem_low_pages(); - - node_set_online(0); + /* Add active regions with valid PFNs. */ + for (i = 0; i < lmb.memory.cnt; i++) { + unsigned long start_pfn, end_pfn; + start_pfn = lmb.memory.region[i].base >> PAGE_SHIFT; + end_pfn = start_pfn + lmb_size_pages(&lmb.memory, i); + __add_active_range(0, start_pfn, end_pfn); + } /* - * Reserve the kernel text and - * Reserve the bootmem bitmap. We do this in two steps (first step - * was init_bootmem()), because this catches the (definitely buggy) - * case of us accidentally initializing the bootmem allocator with - * an invalid RAM area. + * Add all physical memory to the bootmem map and mark each + * area as present. */ - reserve_bootmem(__MEMORY_START + CONFIG_ZERO_PAGE_OFFSET, - (PFN_PHYS(free_pfn) + bootmap_size + PAGE_SIZE - 1) - - (__MEMORY_START + CONFIG_ZERO_PAGE_OFFSET), - BOOTMEM_DEFAULT); + register_bootmem_low_pages(); - /* - * Reserve physical pages below CONFIG_ZERO_PAGE_OFFSET. - */ - if (CONFIG_ZERO_PAGE_OFFSET != 0) - reserve_bootmem(__MEMORY_START, CONFIG_ZERO_PAGE_OFFSET, + /* Reserve the sections we're already using. */ + for (i = 0; i < lmb.reserved.cnt; i++) + reserve_bootmem(lmb.reserved.region[i].base, + lmb_size_bytes(&lmb.reserved, i), BOOTMEM_DEFAULT); + node_set_online(0); + sparse_memory_present_with_active_regions(0); #ifdef CONFIG_BLK_DEV_INITRD @@ -296,12 +303,37 @@ void __init setup_bootmem_allocator(unsigned long free_pfn) static void __init setup_memory(void) { unsigned long start_pfn; + u64 base = min_low_pfn << PAGE_SHIFT; + u64 size = (max_low_pfn << PAGE_SHIFT) - base; /* * Partially used pages are not usable - thus * we are rounding upwards: */ start_pfn = PFN_UP(__pa(_end)); + + lmb_add(base, size); + + /* + * Reserve the kernel text and + * Reserve the bootmem bitmap. We do this in two steps (first step + * was init_bootmem()), because this catches the (definitely buggy) + * case of us accidentally initializing the bootmem allocator with + * an invalid RAM area. + */ + lmb_reserve(__MEMORY_START + CONFIG_ZERO_PAGE_OFFSET, + (PFN_PHYS(start_pfn) + PAGE_SIZE - 1) - + (__MEMORY_START + CONFIG_ZERO_PAGE_OFFSET)); + + /* + * Reserve physical pages below CONFIG_ZERO_PAGE_OFFSET. + */ + if (CONFIG_ZERO_PAGE_OFFSET != 0) + lmb_reserve(__MEMORY_START, CONFIG_ZERO_PAGE_OFFSET); + + lmb_analyze(); + lmb_dump_all(); + setup_bootmem_allocator(start_pfn); } #else @@ -402,6 +434,7 @@ void __init setup_arch(char **cmdline_p) nodes_clear(node_online_map); /* Setup bootmem with available RAM */ + lmb_init(); setup_memory(); sparse_init(); diff --git a/arch/sh/kernel/time.c b/arch/sh/kernel/time.c index 9b352a1e3fb..d2424b068b7 100644 --- a/arch/sh/kernel/time.c +++ b/arch/sh/kernel/time.c @@ -21,6 +21,7 @@ #include <linux/smp.h> #include <linux/rtc.h> #include <asm/clock.h> +#include <asm/hwblk.h> #include <asm/rtc.h> /* Dummy RTC ops */ @@ -96,6 +97,7 @@ void __init time_init(void) if (board_time_init) board_time_init(); + hwblk_init(); clk_init(); rtc_sh_get_time(&xtime); diff --git a/arch/sh/kernel/vmlinux.lds.S b/arch/sh/kernel/vmlinux.lds.S index f53c76acaed..674ed8feb8a 100644 --- a/arch/sh/kernel/vmlinux.lds.S +++ b/arch/sh/kernel/vmlinux.lds.S @@ -50,12 +50,7 @@ SECTIONS _etext = .; /* End of text section */ } = 0x0009 - . = ALIGN(16); /* Exception table */ - __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) { - __start___ex_table = .; - *(__ex_table) - __stop___ex_table = .; - } + EXCEPTION_TABLE(16) NOTES RO_DATA(PAGE_SIZE) @@ -71,69 +66,14 @@ SECTIONS __uncached_end = .; } - . = ALIGN(THREAD_SIZE); - .data : AT(ADDR(.data) - LOAD_OFFSET) { /* Data */ - *(.data.init_task) - - . = ALIGN(L1_CACHE_BYTES); - *(.data.cacheline_aligned) - - . = ALIGN(L1_CACHE_BYTES); - *(.data.read_mostly) - - . = ALIGN(PAGE_SIZE); - *(.data.page_aligned) - - __nosave_begin = .; - *(.data.nosave) - . = ALIGN(PAGE_SIZE); - __nosave_end = .; - - DATA_DATA - CONSTRUCTORS - } + RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) _edata = .; /* End of data section */ . = ALIGN(PAGE_SIZE); /* Init code and data */ - .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) { - __init_begin = .; - _sinittext = .; - INIT_TEXT - _einittext = .; - } - - .init.data : AT(ADDR(.init.data) - LOAD_OFFSET) { INIT_DATA } - - . = ALIGN(16); - .init.setup : AT(ADDR(.init.setup) - LOAD_OFFSET) { - __setup_start = .; - *(.init.setup) - __setup_end = .; - } - - .initcall.init : AT(ADDR(.initcall.init) - LOAD_OFFSET) { - __initcall_start = .; - INITCALLS - __initcall_end = .; - } - - .con_initcall.init : AT(ADDR(.con_initcall.init) - LOAD_OFFSET) { - __con_initcall_start = .; - *(.con_initcall.init) - __con_initcall_end = .; - } - - SECURITY_INIT - -#ifdef CONFIG_BLK_DEV_INITRD - . = ALIGN(PAGE_SIZE); - .init.ramfs : AT(ADDR(.init.ramfs) - LOAD_OFFSET) { - __initramfs_start = .; - *(.init.ramfs) - __initramfs_end = .; - } -#endif + __init_begin = .; + INIT_TEXT_SECTION(PAGE_SIZE) + INIT_DATA_SECTION(16) . = ALIGN(4); .machvec.init : AT(ADDR(.machvec.init) - LOAD_OFFSET) { @@ -152,16 +92,11 @@ SECTIONS .exit.data : AT(ADDR(.exit.data) - LOAD_OFFSET) { EXIT_DATA } . = ALIGN(PAGE_SIZE); - .bss : AT(ADDR(.bss) - LOAD_OFFSET) { - __init_end = .; - __bss_start = .; /* BSS */ - *(.bss.page_aligned) - *(.bss) - *(COMMON) - . = ALIGN(4); - _ebss = .; /* uClinux MTD sucks */ - _end = . ; - } + __init_end = .; + BSS(PAGE_SIZE) + . = ALIGN(4); + _ebss = .; /* uClinux MTD sucks */ + _end = . ; /* * When something in the kernel is NOT compiled as a module, the @@ -170,7 +105,7 @@ SECTIONS * it's a module. */ /DISCARD/ : { - *(.exitcall.exit) + EXIT_CALL } STABS_DEBUG diff --git a/arch/sh/kernel/vmlinux_64.lds.S b/arch/sh/kernel/vmlinux_64.lds.S new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/arch/sh/kernel/vmlinux_64.lds.S diff --git a/arch/sh/lib/Makefile b/arch/sh/lib/Makefile index aaea580b65b..19328d90a2d 100644 --- a/arch/sh/lib/Makefile +++ b/arch/sh/lib/Makefile @@ -25,6 +25,7 @@ memcpy-$(CONFIG_CPU_SH4) := memcpy-sh4.o lib-$(CONFIG_MMU) += copy_page.o clear_page.o lib-$(CONFIG_FUNCTION_TRACER) += mcount.o +lib-$(CONFIG_FUNCTION_GRAPH_TRACER) += mcount.o lib-y += $(memcpy-y) $(udivsi3-y) EXTRA_CFLAGS += -Werror diff --git a/arch/sh/lib/mcount.S b/arch/sh/lib/mcount.S index 110fbfe1831..bd3ec648bec 100644 --- a/arch/sh/lib/mcount.S +++ b/arch/sh/lib/mcount.S @@ -2,13 +2,15 @@ * arch/sh/lib/mcount.S * * Copyright (C) 2008 Paul Mundt - * Copyright (C) 2008 Matt Fleming + * Copyright (C) 2008, 2009 Matt Fleming * * This file is subject to the terms and conditions of the GNU General Public * License. See the file "COPYING" in the main directory of this archive * for more details. */ #include <asm/ftrace.h> +#include <asm/thread_info.h> +#include <asm/asm-offsets.h> #define MCOUNT_ENTER() \ mov.l r4, @-r15; \ @@ -28,6 +30,55 @@ rts; \ mov.l @r15+, r4 +#ifdef CONFIG_STACK_DEBUG +/* + * Perform diagnostic checks on the state of the kernel stack. + * + * Check for stack overflow. If there is less than 1KB free + * then it has overflowed. + * + * Make sure the stack pointer contains a valid address. Valid + * addresses for kernel stacks are anywhere after the bss + * (after _ebss) and anywhere in init_thread_union (init_stack). + */ +#define STACK_CHECK() \ + mov #(THREAD_SIZE >> 10), r0; \ + shll8 r0; \ + shll2 r0; \ + \ + /* r1 = sp & (THREAD_SIZE - 1) */ \ + mov #-1, r1; \ + add r0, r1; \ + and r15, r1; \ + \ + mov #TI_SIZE, r3; \ + mov #(STACK_WARN >> 8), r2; \ + shll8 r2; \ + add r3, r2; \ + \ + /* Is the stack overflowing? */ \ + cmp/hi r2, r1; \ + bf stack_panic; \ + \ + /* If sp > _ebss then we're OK. */ \ + mov.l .L_ebss, r1; \ + cmp/hi r1, r15; \ + bt 1f; \ + \ + /* If sp < init_stack, we're not OK. */ \ + mov.l .L_init_thread_union, r1; \ + cmp/hs r1, r15; \ + bf stack_panic; \ + \ + /* If sp > init_stack && sp < _ebss, not OK. */ \ + add r0, r1; \ + cmp/hs r1, r15; \ + bt stack_panic; \ +1: +#else +#define STACK_CHECK() +#endif /* CONFIG_STACK_DEBUG */ + .align 2 .globl _mcount .type _mcount,@function @@ -35,6 +86,14 @@ .type mcount,@function _mcount: mcount: +#ifndef CONFIG_DYNAMIC_FTRACE + mov.l .Lfunction_trace_stop, r0 + mov.l @r0, r0 + tst r0, r0 + bf ftrace_stub +#endif + STACK_CHECK() + MCOUNT_ENTER() #ifdef CONFIG_DYNAMIC_FTRACE @@ -52,16 +111,71 @@ mcount_call: jsr @r6 nop +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + mov.l .Lftrace_graph_return, r6 + mov.l .Lftrace_stub, r7 + cmp/eq r6, r7 + bt 1f + + mov.l .Lftrace_graph_caller, r0 + jmp @r0 + nop + +1: + mov.l .Lftrace_graph_entry, r6 + mov.l .Lftrace_graph_entry_stub, r7 + cmp/eq r6, r7 + bt skip_trace + + mov.l .Lftrace_graph_caller, r0 + jmp @r0 + nop + + .align 2 +.Lftrace_graph_return: + .long ftrace_graph_return +.Lftrace_graph_entry: + .long ftrace_graph_entry +.Lftrace_graph_entry_stub: + .long ftrace_graph_entry_stub +.Lftrace_graph_caller: + .long ftrace_graph_caller +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ + + .globl skip_trace skip_trace: MCOUNT_LEAVE() .align 2 .Lftrace_trace_function: - .long ftrace_trace_function + .long ftrace_trace_function #ifdef CONFIG_DYNAMIC_FTRACE +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * NOTE: Do not move either ftrace_graph_call or ftrace_caller + * as this will affect the calculation of GRAPH_INSN_OFFSET. + */ + .globl ftrace_graph_call +ftrace_graph_call: + mov.l .Lskip_trace, r0 + jmp @r0 + nop + + .align 2 +.Lskip_trace: + .long skip_trace +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ + .globl ftrace_caller ftrace_caller: + mov.l .Lfunction_trace_stop, r0 + mov.l @r0, r0 + tst r0, r0 + bf ftrace_stub + + STACK_CHECK() + MCOUNT_ENTER() .globl ftrace_call @@ -70,7 +184,12 @@ ftrace_call: jsr @r6 nop +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + bra ftrace_graph_call + nop +#else MCOUNT_LEAVE() +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ #endif /* CONFIG_DYNAMIC_FTRACE */ /* @@ -88,3 +207,99 @@ ftrace_call: ftrace_stub: rts nop + +#ifdef CONFIG_STACK_DEBUG + .globl stack_panic +stack_panic: + mov.l .Ldump_stack, r0 + jsr @r0 + nop + + mov.l .Lpanic, r0 + jsr @r0 + mov.l .Lpanic_s, r4 + + rts + nop + + .align 2 +.Lfunction_trace_stop: + .long function_trace_stop +.L_ebss: + .long _ebss +.L_init_thread_union: + .long init_thread_union +.Lpanic: + .long panic +.Lpanic_s: + .long .Lpanic_str +.Ldump_stack: + .long dump_stack + + .section .rodata + .align 2 +.Lpanic_str: + .string "Stack error" +#endif /* CONFIG_STACK_DEBUG */ + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + .globl ftrace_graph_caller +ftrace_graph_caller: + mov.l 2f, r0 + mov.l @r0, r0 + tst r0, r0 + bt 1f + + mov.l 3f, r1 + jmp @r1 + nop +1: + /* + * MCOUNT_ENTER() pushed 5 registers onto the stack, so + * the stack address containing our return address is + * r15 + 20. + */ + mov #20, r0 + add r15, r0 + mov r0, r4 + + mov.l .Lprepare_ftrace_return, r0 + jsr @r0 + nop + + MCOUNT_LEAVE() + + .align 2 +2: .long function_trace_stop +3: .long skip_trace +.Lprepare_ftrace_return: + .long prepare_ftrace_return + + .globl return_to_handler +return_to_handler: + /* + * Save the return values. + */ + mov.l r0, @-r15 + mov.l r1, @-r15 + + mov #0, r4 + + mov.l .Lftrace_return_to_handler, r0 + jsr @r0 + nop + + /* + * The return value from ftrace_return_handler has the real + * address that we should return to. + */ + lds r0, pr + mov.l @r15+, r1 + rts + mov.l @r15+, r0 + + + .align 2 +.Lftrace_return_to_handler: + .long ftrace_return_to_handler +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/sh/mm/fault_32.c b/arch/sh/mm/fault_32.c index 71925946f1e..08d0117d90f 100644 --- a/arch/sh/mm/fault_32.c +++ b/arch/sh/mm/fault_32.c @@ -2,7 +2,7 @@ * Page fault handler for SH with an MMU. * * Copyright (C) 1999 Niibe Yutaka - * Copyright (C) 2003 - 2008 Paul Mundt + * Copyright (C) 2003 - 2009 Paul Mundt * * Based on linux/arch/i386/mm/fault.c: * Copyright (C) 1995 Linus Torvalds @@ -25,18 +25,84 @@ static inline int notify_page_fault(struct pt_regs *regs, int trap) { int ret = 0; -#ifdef CONFIG_KPROBES - if (!user_mode(regs)) { + if (kprobes_built_in() && !user_mode(regs)) { preempt_disable(); if (kprobe_running() && kprobe_fault_handler(regs, trap)) ret = 1; preempt_enable(); } -#endif return ret; } +static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address) +{ + unsigned index = pgd_index(address); + pgd_t *pgd_k; + pud_t *pud, *pud_k; + pmd_t *pmd, *pmd_k; + + pgd += index; + pgd_k = init_mm.pgd + index; + + if (!pgd_present(*pgd_k)) + return NULL; + + pud = pud_offset(pgd, address); + pud_k = pud_offset(pgd_k, address); + if (!pud_present(*pud_k)) + return NULL; + + pmd = pmd_offset(pud, address); + pmd_k = pmd_offset(pud_k, address); + if (!pmd_present(*pmd_k)) + return NULL; + + if (!pmd_present(*pmd)) + set_pmd(pmd, *pmd_k); + else + BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k)); + + return pmd_k; +} + +/* + * Handle a fault on the vmalloc or module mapping area + */ +static noinline int vmalloc_fault(unsigned long address) +{ + pgd_t *pgd_k; + pmd_t *pmd_k; + pte_t *pte_k; + + /* Make sure we are in vmalloc area: */ + if (!(address >= VMALLOC_START && address < VMALLOC_END)) + return -1; + + /* + * Synchronize this task's top level page-table + * with the 'reference' page table. + * + * Do _not_ use "current" here. We might be inside + * an interrupt in the middle of a task switch.. + */ + pgd_k = get_TTB(); + pmd_k = vmalloc_sync_one(__va((unsigned long)pgd_k), address); + if (!pmd_k) + return -1; + + pte_k = pte_offset_kernel(pmd_k, address); + if (!pte_present(*pte_k)) + return -1; + + return 0; +} + +static int fault_in_kernel_space(unsigned long address) +{ + return address >= TASK_SIZE; +} + /* * This routine handles page faults. It determines the address, * and the problem, and then passes it off to one of the appropriate @@ -46,6 +112,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, unsigned long writeaccess, unsigned long address) { + unsigned long vec; struct task_struct *tsk; struct mm_struct *mm; struct vm_area_struct * vma; @@ -53,59 +120,30 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, int fault; siginfo_t info; - /* - * We don't bother with any notifier callbacks here, as they are - * all handled through the __do_page_fault() fast-path. - */ - tsk = current; + mm = tsk->mm; si_code = SEGV_MAPERR; + vec = lookup_exception_vector(); - if (unlikely(address >= TASK_SIZE)) { - /* - * Synchronize this task's top level page-table - * with the 'reference' page table. - * - * Do _not_ use "tsk" here. We might be inside - * an interrupt in the middle of a task switch.. - */ - int offset = pgd_index(address); - pgd_t *pgd, *pgd_k; - pud_t *pud, *pud_k; - pmd_t *pmd, *pmd_k; - - pgd = get_TTB() + offset; - pgd_k = swapper_pg_dir + offset; - - if (!pgd_present(*pgd)) { - if (!pgd_present(*pgd_k)) - goto bad_area_nosemaphore; - set_pgd(pgd, *pgd_k); + /* + * We fault-in kernel-space virtual memory on-demand. The + * 'reference' page table is init_mm.pgd. + * + * NOTE! We MUST NOT take any locks for this case. We may + * be in an interrupt or a critical region, and should + * only copy the information from the master page table, + * nothing more. + */ + if (unlikely(fault_in_kernel_space(address))) { + if (vmalloc_fault(address) >= 0) return; - } - - pud = pud_offset(pgd, address); - pud_k = pud_offset(pgd_k, address); - - if (!pud_present(*pud)) { - if (!pud_present(*pud_k)) - goto bad_area_nosemaphore; - set_pud(pud, *pud_k); + if (notify_page_fault(regs, vec)) return; - } - - pmd = pmd_offset(pud, address); - pmd_k = pmd_offset(pud_k, address); - if (pmd_present(*pmd) || !pmd_present(*pmd_k)) - goto bad_area_nosemaphore; - set_pmd(pmd, *pmd_k); - return; + goto bad_area_nosemaphore; } - mm = tsk->mm; - - if (unlikely(notify_page_fault(regs, lookup_exception_vector()))) + if (unlikely(notify_page_fault(regs, vec))) return; /* Only enable interrupts if they were on before the fault */ @@ -115,8 +153,8 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); /* - * If we're in an interrupt or have no user - * context, we must not take the fault.. + * If we're in an interrupt, have no user context or are running + * in an atomic region then we must not take the fault: */ if (in_atomic() || !mm) goto no_context; @@ -132,10 +170,11 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, goto bad_area; if (expand_stack(vma, address)) goto bad_area; -/* - * Ok, we have a good vm_area for this memory access, so - * we can handle it.. - */ + + /* + * Ok, we have a good vm_area for this memory access, so + * we can handle it.. + */ good_area: si_code = SEGV_ACCERR; if (writeaccess) { @@ -173,10 +212,10 @@ survive: up_read(&mm->mmap_sem); return; -/* - * Something tried to access memory that isn't in our memory map.. - * Fix it, but check if it's kernel or user first.. - */ + /* + * Something tried to access memory that isn't in our memory map.. + * Fix it, but check if it's kernel or user first.. + */ bad_area: up_read(&mm->mmap_sem); diff --git a/arch/sh/mm/numa.c b/arch/sh/mm/numa.c index 095d93bec7c..9b784fdb947 100644 --- a/arch/sh/mm/numa.c +++ b/arch/sh/mm/numa.c @@ -9,6 +9,7 @@ */ #include <linux/module.h> #include <linux/bootmem.h> +#include <linux/lmb.h> #include <linux/mm.h> #include <linux/numa.h> #include <linux/pfn.h> @@ -26,6 +27,15 @@ EXPORT_SYMBOL_GPL(node_data); void __init setup_memory(void) { unsigned long free_pfn = PFN_UP(__pa(_end)); + u64 base = min_low_pfn << PAGE_SHIFT; + u64 size = (max_low_pfn << PAGE_SHIFT) - min_low_pfn; + + lmb_add(base, size); + + /* Reserve the LMB regions used by the kernel, initrd, etc.. */ + lmb_reserve(__MEMORY_START + CONFIG_ZERO_PAGE_OFFSET, + (PFN_PHYS(free_pfn) + PAGE_SIZE - 1) - + (__MEMORY_START + CONFIG_ZERO_PAGE_OFFSET)); /* * Node 0 sets up its pgdat at the first available pfn, @@ -45,24 +55,23 @@ void __init setup_memory(void) void __init setup_bootmem_node(int nid, unsigned long start, unsigned long end) { - unsigned long bootmap_pages, bootmap_start, bootmap_size; - unsigned long start_pfn, free_pfn, end_pfn; + unsigned long bootmap_pages; + unsigned long start_pfn, end_pfn; + unsigned long bootmem_paddr; /* Don't allow bogus node assignment */ BUG_ON(nid > MAX_NUMNODES || nid == 0); - /* - * The free pfn starts at the beginning of the range, and is - * advanced as necessary for pgdat and node map allocations. - */ - free_pfn = start_pfn = start >> PAGE_SHIFT; + start_pfn = start >> PAGE_SHIFT; end_pfn = end >> PAGE_SHIFT; + lmb_add(start, end - start); + __add_active_range(nid, start_pfn, end_pfn); /* Node-local pgdat */ - NODE_DATA(nid) = pfn_to_kaddr(free_pfn); - free_pfn += PFN_UP(sizeof(struct pglist_data)); + NODE_DATA(nid) = __va(lmb_alloc_base(sizeof(struct pglist_data), + SMP_CACHE_BYTES, end_pfn)); memset(NODE_DATA(nid), 0, sizeof(struct pglist_data)); NODE_DATA(nid)->bdata = &bootmem_node_data[nid]; @@ -71,16 +80,17 @@ void __init setup_bootmem_node(int nid, unsigned long start, unsigned long end) /* Node-local bootmap */ bootmap_pages = bootmem_bootmap_pages(end_pfn - start_pfn); - bootmap_start = (unsigned long)pfn_to_kaddr(free_pfn); - bootmap_size = init_bootmem_node(NODE_DATA(nid), free_pfn, start_pfn, - end_pfn); + bootmem_paddr = lmb_alloc_base(bootmap_pages << PAGE_SHIFT, + PAGE_SIZE, end_pfn); + init_bootmem_node(NODE_DATA(nid), bootmem_paddr >> PAGE_SHIFT, + start_pfn, end_pfn); free_bootmem_with_active_regions(nid, end_pfn); /* Reserve the pgdat and bootmap space with the bootmem allocator */ reserve_bootmem_node(NODE_DATA(nid), start_pfn << PAGE_SHIFT, sizeof(struct pglist_data), BOOTMEM_DEFAULT); - reserve_bootmem_node(NODE_DATA(nid), free_pfn << PAGE_SHIFT, + reserve_bootmem_node(NODE_DATA(nid), bootmem_paddr, bootmap_pages << PAGE_SHIFT, BOOTMEM_DEFAULT); /* It's up */ |