From ff1b15b646177c6cc465ac2dd0be6ae16e965654 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 09:27:19 -0300 Subject: x86: don't use size specifiers. Remove the "l" from inline asm at arch/x86/lib/delay_32.c. It is not needed. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/delay_32.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/delay_32.c b/arch/x86/lib/delay_32.c index ef691316f8b..54013f87d95 100644 --- a/arch/x86/lib/delay_32.c +++ b/arch/x86/lib/delay_32.c @@ -38,9 +38,9 @@ static void delay_loop(unsigned long loops) "1: jmp 2f \n" ".align 16 \n" - "2: decl %0 \n" + "2: dec %0 \n" " jnz 2b \n" - "3: decl %0 \n" + "3: dec %0 \n" : /* we don't need output */ :"a" (loops) -- cgit v1.2.3 From 0a4d8a472f645d99f86303db1462b64e371b090d Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 09:34:08 -0300 Subject: x86: provide delay loop for x86_64. This is for consistency with i386. We call use_tsc_delay() at tsc initialization for x86_64, so we'll be always using it. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/kernel/tsc.c | 1 + arch/x86/lib/delay_64.c | 44 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 3c36f92160c..4a775d00195 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -513,6 +513,7 @@ void __init tsc_init(void) */ for_each_possible_cpu(cpu) set_cyc2ns_scale(cpu_khz, cpu); + use_tsc_delay(); if (tsc_disabled > 0) return; diff --git a/arch/x86/lib/delay_64.c b/arch/x86/lib/delay_64.c index 4c441be9264..d0326d07c84 100644 --- a/arch/x86/lib/delay_64.c +++ b/arch/x86/lib/delay_64.c @@ -22,13 +22,28 @@ #include #endif -int __devinit read_current_timer(unsigned long *timer_value) +/* simple loop based delay: */ +static void delay_loop(unsigned long loops) { - rdtscll(*timer_value); - return 0; + asm volatile( + " test %0,%0 \n" + " jz 3f \n" + " jmp 1f \n" + + ".align 16 \n" + "1: jmp 2f \n" + + ".align 16 \n" + "2: dec %0 \n" + " jnz 2b \n" + "3: dec %0 \n" + + : /* we don't need output */ + :"a" (loops) + ); } -void __delay(unsigned long loops) +static void delay_tsc(unsigned long loops) { unsigned bclock, now; int cpu; @@ -63,6 +78,27 @@ void __delay(unsigned long loops) } preempt_enable(); } + +static void (*delay_fn)(unsigned long) = delay_loop; + +void use_tsc_delay(void) +{ + delay_fn = delay_tsc; +} + +int __devinit read_current_timer(unsigned long *timer_value) +{ + if (delay_fn == delay_tsc) { + rdtscll(*timer_value); + return 0; + } + return -1; +} + +void __delay(unsigned long loops) +{ + delay_fn(loops); +} EXPORT_SYMBOL(__delay); inline void __const_udelay(unsigned long xloops) -- cgit v1.2.3 From a76febe975997b933b7285b6e20bb0a21c09d453 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 09:52:36 -0300 Subject: x86: use rdtscll in read_current_timer for i386. This way we achieve the same code for both arches. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/delay_32.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/lib/delay_32.c b/arch/x86/lib/delay_32.c index 54013f87d95..bf6de05445b 100644 --- a/arch/x86/lib/delay_32.c +++ b/arch/x86/lib/delay_32.c @@ -98,7 +98,7 @@ void use_tsc_delay(void) int __devinit read_current_timer(unsigned long *timer_val) { if (delay_fn == delay_tsc) { - rdtscl(*timer_val); + rdtscll(*timer_val); return 0; } return -1; -- cgit v1.2.3 From 7e58818d32c18197602d1869b22cfda99efd05fe Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 10:21:25 -0300 Subject: x86: explicitly use edx in const delay function. For x86_64, we can't just use %0, as it would generate a mul against rdx, which is not really what we want (note the ">> 32" in x86_64 version). Using a u64 variable with a shift in i386 generates bad code, so the solution is to explicitly use %%edx in inline assembly for both. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/delay_32.c | 2 +- arch/x86/lib/delay_64.c | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/delay_32.c b/arch/x86/lib/delay_32.c index bf6de05445b..0b659a320b1 100644 --- a/arch/x86/lib/delay_32.c +++ b/arch/x86/lib/delay_32.c @@ -114,7 +114,7 @@ inline void __const_udelay(unsigned long xloops) int d0; xloops *= 4; - __asm__("mull %0" + __asm__("mull %%edx" :"=d" (xloops), "=&a" (d0) :"1" (xloops), "0" (cpu_data(raw_smp_processor_id()).loops_per_jiffy * (HZ/4))); diff --git a/arch/x86/lib/delay_64.c b/arch/x86/lib/delay_64.c index d0326d07c84..ff3dfecdb6f 100644 --- a/arch/x86/lib/delay_64.c +++ b/arch/x86/lib/delay_64.c @@ -103,9 +103,16 @@ EXPORT_SYMBOL(__delay); inline void __const_udelay(unsigned long xloops) { - __delay(((xloops * HZ * - cpu_data(raw_smp_processor_id()).loops_per_jiffy) >> 32) + 1); + int d0; + xloops *= 4; + __asm__("mull %%edx" + :"=d" (xloops), "=&a" (d0) + :"1" (xloops), "0" + (cpu_data(raw_smp_processor_id()).loops_per_jiffy * (HZ/4))); + + __delay(++xloops); } + EXPORT_SYMBOL(__const_udelay); void __udelay(unsigned long usecs) -- cgit v1.2.3 From f0fbf0abc093ec8bf64506eee4ede9e5daf40ffd Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Thu, 3 Jul 2008 12:35:41 -0300 Subject: x86: integrate delay functions. delay_32.c, delay_64.c are now equal, and are integrated into delay.c. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/Makefile | 2 +- arch/x86/lib/delay.c | 137 +++++++++++++++++++++++++++++++++++++++++++++++ arch/x86/lib/delay_32.c | 138 ------------------------------------------------ arch/x86/lib/delay_64.c | 128 -------------------------------------------- 4 files changed, 138 insertions(+), 267 deletions(-) create mode 100644 arch/x86/lib/delay.c delete mode 100644 arch/x86/lib/delay_32.c delete mode 100644 arch/x86/lib/delay_64.c (limited to 'arch') diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index 76f60f52a88..86960a6c41c 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -4,7 +4,7 @@ obj-$(CONFIG_SMP) := msr-on-cpu.o -lib-y := delay_$(BITS).o +lib-y := delay.o lib-y += usercopy_$(BITS).o getuser_$(BITS).o putuser_$(BITS).o lib-y += memcpy_$(BITS).o diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c new file mode 100644 index 00000000000..f4568605d7d --- /dev/null +++ b/arch/x86/lib/delay.c @@ -0,0 +1,137 @@ +/* + * Precise Delay Loops for i386 + * + * Copyright (C) 1993 Linus Torvalds + * Copyright (C) 1997 Martin Mares + * Copyright (C) 2008 Jiri Hladky + * + * The __delay function must _NOT_ be inlined as its execution time + * depends wildly on alignment on many x86 processors. The additional + * jump magic is needed to get the timing stable on all the CPU's + * we have to worry about. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#ifdef CONFIG_SMP +# include +#endif + +/* simple loop based delay: */ +static void delay_loop(unsigned long loops) +{ + asm volatile( + " test %0,%0 \n" + " jz 3f \n" + " jmp 1f \n" + + ".align 16 \n" + "1: jmp 2f \n" + + ".align 16 \n" + "2: dec %0 \n" + " jnz 2b \n" + "3: dec %0 \n" + + : /* we don't need output */ + :"a" (loops) + ); +} + +/* TSC based delay: */ +static void delay_tsc(unsigned long loops) +{ + unsigned long bclock, now; + int cpu; + + preempt_disable(); + cpu = smp_processor_id(); + rdtscl(bclock); + for (;;) { + rdtscl(now); + if ((now - bclock) >= loops) + break; + + /* Allow RT tasks to run */ + preempt_enable(); + rep_nop(); + preempt_disable(); + + /* + * It is possible that we moved to another CPU, and + * since TSC's are per-cpu we need to calculate + * that. The delay must guarantee that we wait "at + * least" the amount of time. Being moved to another + * CPU could make the wait longer but we just need to + * make sure we waited long enough. Rebalance the + * counter for this CPU. + */ + if (unlikely(cpu != smp_processor_id())) { + loops -= (now - bclock); + cpu = smp_processor_id(); + rdtscl(bclock); + } + } + preempt_enable(); +} + +/* + * Since we calibrate only once at boot, this + * function should be set once at boot and not changed + */ +static void (*delay_fn)(unsigned long) = delay_loop; + +void use_tsc_delay(void) +{ + delay_fn = delay_tsc; +} + +int __devinit read_current_timer(unsigned long *timer_val) +{ + if (delay_fn == delay_tsc) { + rdtscll(*timer_val); + return 0; + } + return -1; +} + +void __delay(unsigned long loops) +{ + delay_fn(loops); +} +EXPORT_SYMBOL(__delay); + +inline void __const_udelay(unsigned long xloops) +{ + int d0; + + xloops *= 4; + asm("mull %%edx" + :"=d" (xloops), "=&a" (d0) + :"1" (xloops), "0" + (cpu_data(raw_smp_processor_id()).loops_per_jiffy * (HZ/4))); + + __delay(++xloops); +} +EXPORT_SYMBOL(__const_udelay); + +void __udelay(unsigned long usecs) +{ + __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */ +} +EXPORT_SYMBOL(__udelay); + +void __ndelay(unsigned long nsecs) +{ + __const_udelay(nsecs * 0x00005); /* 2**32 / 1000000000 (rounded up) */ +} +EXPORT_SYMBOL(__ndelay); diff --git a/arch/x86/lib/delay_32.c b/arch/x86/lib/delay_32.c deleted file mode 100644 index 0b659a320b1..00000000000 --- a/arch/x86/lib/delay_32.c +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Precise Delay Loops for i386 - * - * Copyright (C) 1993 Linus Torvalds - * Copyright (C) 1997 Martin Mares - * Copyright (C) 2008 Jiri Hladky - * - * The __delay function must _NOT_ be inlined as its execution time - * depends wildly on alignment on many x86 processors. The additional - * jump magic is needed to get the timing stable on all the CPU's - * we have to worry about. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#ifdef CONFIG_SMP -# include -#endif - -/* simple loop based delay: */ -static void delay_loop(unsigned long loops) -{ - __asm__ __volatile__( - " test %0,%0 \n" - " jz 3f \n" - " jmp 1f \n" - - ".align 16 \n" - "1: jmp 2f \n" - - ".align 16 \n" - "2: dec %0 \n" - " jnz 2b \n" - "3: dec %0 \n" - - : /* we don't need output */ - :"a" (loops) - ); -} - -/* TSC based delay: */ -static void delay_tsc(unsigned long loops) -{ - unsigned long bclock, now; - int cpu; - - preempt_disable(); - cpu = smp_processor_id(); - rdtscl(bclock); - for (;;) { - rdtscl(now); - if ((now - bclock) >= loops) - break; - - /* Allow RT tasks to run */ - preempt_enable(); - rep_nop(); - preempt_disable(); - - /* - * It is possible that we moved to another CPU, and - * since TSC's are per-cpu we need to calculate - * that. The delay must guarantee that we wait "at - * least" the amount of time. Being moved to another - * CPU could make the wait longer but we just need to - * make sure we waited long enough. Rebalance the - * counter for this CPU. - */ - if (unlikely(cpu != smp_processor_id())) { - loops -= (now - bclock); - cpu = smp_processor_id(); - rdtscl(bclock); - } - } - preempt_enable(); -} - -/* - * Since we calibrate only once at boot, this - * function should be set once at boot and not changed - */ -static void (*delay_fn)(unsigned long) = delay_loop; - -void use_tsc_delay(void) -{ - delay_fn = delay_tsc; -} - -int __devinit read_current_timer(unsigned long *timer_val) -{ - if (delay_fn == delay_tsc) { - rdtscll(*timer_val); - return 0; - } - return -1; -} - -void __delay(unsigned long loops) -{ - delay_fn(loops); -} - -inline void __const_udelay(unsigned long xloops) -{ - int d0; - - xloops *= 4; - __asm__("mull %%edx" - :"=d" (xloops), "=&a" (d0) - :"1" (xloops), "0" - (cpu_data(raw_smp_processor_id()).loops_per_jiffy * (HZ/4))); - - __delay(++xloops); -} - -void __udelay(unsigned long usecs) -{ - __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */ -} - -void __ndelay(unsigned long nsecs) -{ - __const_udelay(nsecs * 0x00005); /* 2**32 / 1000000000 (rounded up) */ -} - -EXPORT_SYMBOL(__delay); -EXPORT_SYMBOL(__const_udelay); -EXPORT_SYMBOL(__udelay); -EXPORT_SYMBOL(__ndelay); diff --git a/arch/x86/lib/delay_64.c b/arch/x86/lib/delay_64.c deleted file mode 100644 index ff3dfecdb6f..00000000000 --- a/arch/x86/lib/delay_64.c +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Precise Delay Loops for x86-64 - * - * Copyright (C) 1993 Linus Torvalds - * Copyright (C) 1997 Martin Mares - * - * The __delay function must _NOT_ be inlined as its execution time - * depends wildly on alignment on many x86 processors. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include - -#ifdef CONFIG_SMP -#include -#endif - -/* simple loop based delay: */ -static void delay_loop(unsigned long loops) -{ - asm volatile( - " test %0,%0 \n" - " jz 3f \n" - " jmp 1f \n" - - ".align 16 \n" - "1: jmp 2f \n" - - ".align 16 \n" - "2: dec %0 \n" - " jnz 2b \n" - "3: dec %0 \n" - - : /* we don't need output */ - :"a" (loops) - ); -} - -static void delay_tsc(unsigned long loops) -{ - unsigned bclock, now; - int cpu; - - preempt_disable(); - cpu = smp_processor_id(); - rdtscl(bclock); - for (;;) { - rdtscl(now); - if ((now - bclock) >= loops) - break; - - /* Allow RT tasks to run */ - preempt_enable(); - rep_nop(); - preempt_disable(); - - /* - * It is possible that we moved to another CPU, and - * since TSC's are per-cpu we need to calculate - * that. The delay must guarantee that we wait "at - * least" the amount of time. Being moved to another - * CPU could make the wait longer but we just need to - * make sure we waited long enough. Rebalance the - * counter for this CPU. - */ - if (unlikely(cpu != smp_processor_id())) { - loops -= (now - bclock); - cpu = smp_processor_id(); - rdtscl(bclock); - } - } - preempt_enable(); -} - -static void (*delay_fn)(unsigned long) = delay_loop; - -void use_tsc_delay(void) -{ - delay_fn = delay_tsc; -} - -int __devinit read_current_timer(unsigned long *timer_value) -{ - if (delay_fn == delay_tsc) { - rdtscll(*timer_value); - return 0; - } - return -1; -} - -void __delay(unsigned long loops) -{ - delay_fn(loops); -} -EXPORT_SYMBOL(__delay); - -inline void __const_udelay(unsigned long xloops) -{ - int d0; - xloops *= 4; - __asm__("mull %%edx" - :"=d" (xloops), "=&a" (d0) - :"1" (xloops), "0" - (cpu_data(raw_smp_processor_id()).loops_per_jiffy * (HZ/4))); - - __delay(++xloops); -} - -EXPORT_SYMBOL(__const_udelay); - -void __udelay(unsigned long usecs) -{ - __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */ -} -EXPORT_SYMBOL(__udelay); - -void __ndelay(unsigned long nsecs) -{ - __const_udelay(nsecs * 0x00005); /* 2**32 / 1000000000 (rounded up) */ -} -EXPORT_SYMBOL(__ndelay); -- cgit v1.2.3 From edf10162b2c5ad78ada8e63e960f9d0949c6c219 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Fri, 13 Jun 2008 16:35:52 -0300 Subject: x86: don't clobber r8 nor use rcx. There's really no reason to clobber r8 or pass the address in rcx. We can safely use only two registers (which we already have to touch anyway) to do the job. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_64.S | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index 5448876261f..2b003d31348 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -36,10 +36,10 @@ .text ENTRY(__get_user_1) CFI_STARTPROC - GET_THREAD_INFO(%r8) - cmpq threadinfo_addr_limit(%r8),%rcx + GET_THREAD_INFO(%rdx) + cmpq threadinfo_addr_limit(%rdx),%rax jae bad_get_user -1: movzb (%rcx),%edx +1: movzb (%rax),%edx xorl %eax,%eax ret CFI_ENDPROC @@ -47,48 +47,48 @@ ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - GET_THREAD_INFO(%r8) - addq $1,%rcx + GET_THREAD_INFO(%rdx) + addq $1,%rax jc 20f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq threadinfo_addr_limit(%rdx),%rax jae 20f - decq %rcx -2: movzwl (%rcx),%edx + decq %rax +2: movzwl (%rax),%edx xorl %eax,%eax ret -20: decq %rcx +20: decq %rax jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - GET_THREAD_INFO(%r8) - addq $3,%rcx + GET_THREAD_INFO(%rdx) + addq $3,%rax jc 30f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq threadinfo_addr_limit(%rdx),%rax jae 30f - subq $3,%rcx -3: movl (%rcx),%edx + subq $3,%rax +3: movl (%rax),%edx xorl %eax,%eax ret -30: subq $3,%rcx +30: subq $3,%rax jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_4) ENTRY(__get_user_8) CFI_STARTPROC - GET_THREAD_INFO(%r8) - addq $7,%rcx + GET_THREAD_INFO(%rdx) + addq $7,%rax jc 40f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq threadinfo_addr_limit(%rdx),%rax jae 40f - subq $7,%rcx -4: movq (%rcx),%rdx + subq $7,%rax +4: movq (%rax),%rdx xorl %eax,%eax ret -40: subq $7,%rcx +40: subq $7,%rax jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_8) -- cgit v1.2.3 From 9aa038815b5756e20a00b8e1efd5740434b37aea Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Fri, 13 Jun 2008 22:41:51 -0300 Subject: x86: don't use word-size specifiers. Since the instructions refer to registers, they'll be able to figure it out. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_32.S | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/getuser_32.S b/arch/x86/lib/getuser_32.S index 6d84b53f12a..8200fde55f5 100644 --- a/arch/x86/lib/getuser_32.S +++ b/arch/x86/lib/getuser_32.S @@ -29,44 +29,44 @@ ENTRY(__get_user_1) CFI_STARTPROC GET_THREAD_INFO(%edx) - cmpl TI_addr_limit(%edx),%eax + cmp TI_addr_limit(%edx),%eax jae bad_get_user -1: movzbl (%eax),%edx - xorl %eax,%eax +1: movzb (%eax),%edx + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - addl $1,%eax + add $1,%eax jc bad_get_user GET_THREAD_INFO(%edx) - cmpl TI_addr_limit(%edx),%eax + cmp TI_addr_limit(%edx),%eax jae bad_get_user 2: movzwl -1(%eax),%edx - xorl %eax,%eax + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - addl $3,%eax + add $3,%eax jc bad_get_user GET_THREAD_INFO(%edx) - cmpl TI_addr_limit(%edx),%eax + cmp TI_addr_limit(%edx),%eax jae bad_get_user -3: movl -3(%eax),%edx - xorl %eax,%eax +3: mov -3(%eax),%edx + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_4) bad_get_user: CFI_STARTPROC - xorl %edx,%edx - movl $-14,%eax + xor %edx,%edx + mov $-14,%eax ret CFI_ENDPROC END(bad_get_user) -- cgit v1.2.3 From 9262875395cf22b5a90dd8a640e1070cedf55d0e Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 11:13:16 -0300 Subject: x86: adapt x86_64 getuser functions. Instead of doing a sub after the addition, use the offset directly at the memory operand of the mov instructions. This is the way i386 do. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_64.S | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index 2b003d31348..df37d3a9ba2 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -47,49 +47,40 @@ ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - GET_THREAD_INFO(%rdx) addq $1,%rax - jc 20f + jc bad_get_user + GET_THREAD_INFO(%rdx) cmpq threadinfo_addr_limit(%rdx),%rax - jae 20f - decq %rax -2: movzwl (%rax),%edx + jae bad_get_user +2: movzwl -1(%rax),%edx xorl %eax,%eax ret -20: decq %rax - jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - GET_THREAD_INFO(%rdx) addq $3,%rax - jc 30f + jc bad_get_user + GET_THREAD_INFO(%rdx) cmpq threadinfo_addr_limit(%rdx),%rax - jae 30f - subq $3,%rax -3: movl (%rax),%edx + jae bad_get_user +3: movl -3(%rax),%edx xorl %eax,%eax ret -30: subq $3,%rax - jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_4) ENTRY(__get_user_8) CFI_STARTPROC - GET_THREAD_INFO(%rdx) addq $7,%rax - jc 40f + jc bad_get_user + GET_THREAD_INFO(%rdx) cmpq threadinfo_addr_limit(%rdx),%rax - jae 40f - subq $7,%rax -4: movq (%rax),%rdx + jae bad_get_user +4: movq -7(%rax),%rdx xorl %eax,%eax ret -40: subq $7,%rax - jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_8) -- cgit v1.2.3 From 26ccb8a7183eed424ff9c874c83af20dafe7cdef Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 11:19:35 -0300 Subject: x86: rename threadinfo to TI. This is for consistency with i386. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/ia32/ia32entry.S | 25 ++++++++++++++----------- arch/x86/kernel/asm-offsets_64.c | 2 +- arch/x86/kernel/entry_64.S | 23 ++++++++++++----------- arch/x86/lib/copy_user_64.S | 4 ++-- arch/x86/lib/getuser_64.S | 8 ++++---- arch/x86/lib/putuser_64.S | 8 ++++---- 6 files changed, 37 insertions(+), 33 deletions(-) (limited to 'arch') diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S index 24e4d4928d6..20371d0635e 100644 --- a/arch/x86/ia32/ia32entry.S +++ b/arch/x86/ia32/ia32entry.S @@ -116,7 +116,7 @@ ENTRY(ia32_sysenter_target) pushfq CFI_ADJUST_CFA_OFFSET 8 /*CFI_REL_OFFSET rflags,0*/ - movl 8*3-THREAD_SIZE+threadinfo_sysenter_return(%rsp), %r10d + movl 8*3-THREAD_SIZE+TI_sysenter_return(%rsp), %r10d CFI_REGISTER rip,r10 pushq $__USER32_CS CFI_ADJUST_CFA_OFFSET 8 @@ -136,8 +136,9 @@ ENTRY(ia32_sysenter_target) .quad 1b,ia32_badarg .previous GET_THREAD_INFO(%r10) - orl $TS_COMPAT,threadinfo_status(%r10) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) + orl $TS_COMPAT,TI_status(%r10) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ + TI_flags(%r10) CFI_REMEMBER_STATE jnz sysenter_tracesys sysenter_do_call: @@ -149,9 +150,9 @@ sysenter_do_call: GET_THREAD_INFO(%r10) DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF - testl $_TIF_ALLWORK_MASK,threadinfo_flags(%r10) + testl $_TIF_ALLWORK_MASK,TI_flags(%r10) jnz int_ret_from_sys_call - andl $~TS_COMPAT,threadinfo_status(%r10) + andl $~TS_COMPAT,TI_status(%r10) /* clear IF, that popfq doesn't enable interrupts early */ andl $~0x200,EFLAGS-R11(%rsp) movl RIP-R11(%rsp),%edx /* User %eip */ @@ -240,8 +241,9 @@ ENTRY(ia32_cstar_target) .quad 1b,ia32_badarg .previous GET_THREAD_INFO(%r10) - orl $TS_COMPAT,threadinfo_status(%r10) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) + orl $TS_COMPAT,TI_status(%r10) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ + TI_flags(%r10) CFI_REMEMBER_STATE jnz cstar_tracesys cstar_do_call: @@ -253,9 +255,9 @@ cstar_do_call: GET_THREAD_INFO(%r10) DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF - testl $_TIF_ALLWORK_MASK,threadinfo_flags(%r10) + testl $_TIF_ALLWORK_MASK,TI_flags(%r10) jnz int_ret_from_sys_call - andl $~TS_COMPAT,threadinfo_status(%r10) + andl $~TS_COMPAT,TI_status(%r10) RESTORE_ARGS 1,-ARG_SKIP,1,1,1 movl RIP-ARGOFFSET(%rsp),%ecx CFI_REGISTER rip,rcx @@ -333,8 +335,9 @@ ENTRY(ia32_syscall) this could be a problem. */ SAVE_ARGS 0,0,1 GET_THREAD_INFO(%r10) - orl $TS_COMPAT,threadinfo_status(%r10) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) + orl $TS_COMPAT,TI_status(%r10) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ + TI_flags(%r10) jnz ia32_tracesys ia32_do_syscall: cmpl $(IA32_NR_syscalls-1),%eax diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c index 3295e7c08fe..bacf5deeec2 100644 --- a/arch/x86/kernel/asm-offsets_64.c +++ b/arch/x86/kernel/asm-offsets_64.c @@ -34,7 +34,7 @@ int main(void) ENTRY(pid); BLANK(); #undef ENTRY -#define ENTRY(entry) DEFINE(threadinfo_ ## entry, offsetof(struct thread_info, entry)) +#define ENTRY(entry) DEFINE(TI_ ## entry, offsetof(struct thread_info, entry)) ENTRY(flags); ENTRY(addr_limit); ENTRY(preempt_count); diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 07d69f26233..466b9284ed2 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -168,13 +168,13 @@ ENTRY(ret_from_fork) CFI_ADJUST_CFA_OFFSET -4 call schedule_tail GET_THREAD_INFO(%rcx) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx) jnz rff_trace rff_action: RESTORE_REST testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread? je int_ret_from_sys_call - testl $_TIF_IA32,threadinfo_flags(%rcx) + testl $_TIF_IA32,TI_flags(%rcx) jnz int_ret_from_sys_call RESTORE_TOP_OF_STACK %rdi,ARGOFFSET jmp ret_from_sys_call @@ -243,7 +243,8 @@ ENTRY(system_call_after_swapgs) movq %rcx,RIP-ARGOFFSET(%rsp) CFI_REL_OFFSET rip,RIP-ARGOFFSET GET_THREAD_INFO(%rcx) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ + TI_flags(%rcx) jnz tracesys cmpq $__NR_syscall_max,%rax ja badsys @@ -262,7 +263,7 @@ sysret_check: GET_THREAD_INFO(%rcx) DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF - movl threadinfo_flags(%rcx),%edx + movl TI_flags(%rcx),%edx andl %edi,%edx jnz sysret_careful CFI_REMEMBER_STATE @@ -347,10 +348,10 @@ int_ret_from_sys_call: int_with_check: LOCKDEP_SYS_EXIT_IRQ GET_THREAD_INFO(%rcx) - movl threadinfo_flags(%rcx),%edx + movl TI_flags(%rcx),%edx andl %edi,%edx jnz int_careful - andl $~TS_COMPAT,threadinfo_status(%rcx) + andl $~TS_COMPAT,TI_status(%rcx) jmp retint_swapgs /* Either reschedule or signal or syscall exit tracking needed. */ @@ -558,7 +559,7 @@ retint_with_reschedule: movl $_TIF_WORK_MASK,%edi retint_check: LOCKDEP_SYS_EXIT_IRQ - movl threadinfo_flags(%rcx),%edx + movl TI_flags(%rcx),%edx andl %edi,%edx CFI_REMEMBER_STATE jnz retint_careful @@ -654,9 +655,9 @@ retint_signal: /* Returning to kernel space. Check if we need preemption */ /* rcx: threadinfo. interrupts off. */ ENTRY(retint_kernel) - cmpl $0,threadinfo_preempt_count(%rcx) + cmpl $0,TI_preempt_count(%rcx) jnz retint_restore_args - bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx) + bt $TIF_NEED_RESCHED,TI_flags(%rcx) jnc retint_restore_args bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ jnc retint_restore_args @@ -819,7 +820,7 @@ paranoid_restore\trace: jmp irq_return paranoid_userspace\trace: GET_THREAD_INFO(%rcx) - movl threadinfo_flags(%rcx),%ebx + movl TI_flags(%rcx),%ebx andl $_TIF_WORK_MASK,%ebx jz paranoid_swapgs\trace movq %rsp,%rdi /* &pt_regs */ @@ -917,7 +918,7 @@ error_exit: testl %eax,%eax jne retint_kernel LOCKDEP_SYS_EXIT_IRQ - movl threadinfo_flags(%rcx),%edx + movl TI_flags(%rcx),%edx movl $_TIF_WORK_MASK,%edi andl %edi,%edx jnz retint_careful diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S index ee1c3f63515..7eaaf0123b4 100644 --- a/arch/x86/lib/copy_user_64.S +++ b/arch/x86/lib/copy_user_64.S @@ -40,7 +40,7 @@ ENTRY(copy_to_user) movq %rdi,%rcx addq %rdx,%rcx jc bad_to_user - cmpq threadinfo_addr_limit(%rax),%rcx + cmpq TI_addr_limit(%rax),%rcx jae bad_to_user xorl %eax,%eax /* clear zero flag */ ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string @@ -65,7 +65,7 @@ ENTRY(copy_from_user) movq %rsi,%rcx addq %rdx,%rcx jc bad_from_user - cmpq threadinfo_addr_limit(%rax),%rcx + cmpq TI_addr_limit(%rax),%rcx jae bad_from_user movl $1,%ecx /* set zero flag */ ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index df37d3a9ba2..0ec7890f9dc 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -37,7 +37,7 @@ ENTRY(__get_user_1) CFI_STARTPROC GET_THREAD_INFO(%rdx) - cmpq threadinfo_addr_limit(%rdx),%rax + cmpq TI_addr_limit(%rdx),%rax jae bad_get_user 1: movzb (%rax),%edx xorl %eax,%eax @@ -50,7 +50,7 @@ ENTRY(__get_user_2) addq $1,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq threadinfo_addr_limit(%rdx),%rax + cmpq TI_addr_limit(%rdx),%rax jae bad_get_user 2: movzwl -1(%rax),%edx xorl %eax,%eax @@ -63,7 +63,7 @@ ENTRY(__get_user_4) addq $3,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq threadinfo_addr_limit(%rdx),%rax + cmpq TI_addr_limit(%rdx),%rax jae bad_get_user 3: movl -3(%rax),%edx xorl %eax,%eax @@ -76,7 +76,7 @@ ENTRY(__get_user_8) addq $7,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq threadinfo_addr_limit(%rdx),%rax + cmpq TI_addr_limit(%rdx),%rax jae bad_get_user 4: movq -7(%rax),%rdx xorl %eax,%eax diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index 4989f5a8fa9..940796fa0d9 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -35,7 +35,7 @@ ENTRY(__put_user_1) CFI_STARTPROC GET_THREAD_INFO(%r8) - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%r8),%rcx jae bad_put_user 1: movb %dl,(%rcx) xorl %eax,%eax @@ -48,7 +48,7 @@ ENTRY(__put_user_2) GET_THREAD_INFO(%r8) addq $1,%rcx jc 20f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%r8),%rcx jae 20f decq %rcx 2: movw %dx,(%rcx) @@ -64,7 +64,7 @@ ENTRY(__put_user_4) GET_THREAD_INFO(%r8) addq $3,%rcx jc 30f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%r8),%rcx jae 30f subq $3,%rcx 3: movl %edx,(%rcx) @@ -80,7 +80,7 @@ ENTRY(__put_user_8) GET_THREAD_INFO(%r8) addq $7,%rcx jc 40f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%r8),%rcx jae 40f subq $7,%rcx 4: movq %rdx,(%rcx) -- cgit v1.2.3 From ef8c1a2d0e990d0f4f15e1d45eeb262755e3d4c3 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 11:21:53 -0300 Subject: x86: don't use word-size specifiers on getuser_64. The instructions access registers, so the size is unambiguous. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_64.S | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index 0ec7890f9dc..6134752a75f 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -37,57 +37,57 @@ ENTRY(__get_user_1) CFI_STARTPROC GET_THREAD_INFO(%rdx) - cmpq TI_addr_limit(%rdx),%rax + cmp TI_addr_limit(%rdx),%rax jae bad_get_user 1: movzb (%rax),%edx - xorl %eax,%eax + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - addq $1,%rax + add $1,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq TI_addr_limit(%rdx),%rax + cmp TI_addr_limit(%rdx),%rax jae bad_get_user 2: movzwl -1(%rax),%edx - xorl %eax,%eax + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - addq $3,%rax + add $3,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq TI_addr_limit(%rdx),%rax + cmp TI_addr_limit(%rdx),%rax jae bad_get_user -3: movl -3(%rax),%edx - xorl %eax,%eax +3: mov -3(%rax),%edx + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_4) ENTRY(__get_user_8) CFI_STARTPROC - addq $7,%rax + add $7,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq TI_addr_limit(%rdx),%rax + cmp TI_addr_limit(%rdx),%rax jae bad_get_user 4: movq -7(%rax),%rdx - xorl %eax,%eax + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_8) bad_get_user: CFI_STARTPROC - xorl %edx,%edx - movq $(-EFAULT),%rax + xor %edx,%edx + mov $(-EFAULT),%rax ret CFI_ENDPROC END(bad_get_user) -- cgit v1.2.3 From 40faf463e62de0b29722910eded7dd26cd8b684b Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 11:37:57 -0300 Subject: x86: introduce __ASM_REG macro. There are situations in which the architecture wants to use the register that represents its word-size, whatever it is. For those, introduce __ASM_REG in asm.h, along with the first users _ASM_AX and _ASM_DX. They have users waiting for it, namely the getuser functions. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_32.S | 25 +++++++++++++------------ arch/x86/lib/getuser_64.S | 36 ++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 30 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/getuser_32.S b/arch/x86/lib/getuser_32.S index 8200fde55f5..2cc3ceee8f9 100644 --- a/arch/x86/lib/getuser_32.S +++ b/arch/x86/lib/getuser_32.S @@ -11,6 +11,7 @@ #include #include #include +#include /* @@ -28,10 +29,10 @@ .text ENTRY(__get_user_1) CFI_STARTPROC - GET_THREAD_INFO(%edx) - cmp TI_addr_limit(%edx),%eax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -1: movzb (%eax),%edx +1: movzb (%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -39,12 +40,12 @@ ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - add $1,%eax + add $1,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%edx) - cmp TI_addr_limit(%edx),%eax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -2: movzwl -1(%eax),%edx +2: movzwl -1(%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -52,12 +53,12 @@ ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - add $3,%eax + add $3,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%edx) - cmp TI_addr_limit(%edx),%eax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -3: mov -3(%eax),%edx +3: mov -3(%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -66,7 +67,7 @@ ENDPROC(__get_user_4) bad_get_user: CFI_STARTPROC xor %edx,%edx - mov $-14,%eax + mov $-14,%_ASM_AX ret CFI_ENDPROC END(bad_get_user) diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index 6134752a75f..63b0e5c1e58 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -13,14 +13,13 @@ /* * __get_user_X * - * Inputs: %rcx contains the address. + * Inputs: %rax contains the address. * The register is modified, but all changes are undone * before returning because the C code doesn't know about it. * * Outputs: %rax is error code (0 or -EFAULT) * %rdx contains zero-extended value * - * %r8 is destroyed. * * These functions should not modify any other registers, * as they get called from within inline assembly. @@ -32,14 +31,15 @@ #include #include #include +#include .text ENTRY(__get_user_1) CFI_STARTPROC - GET_THREAD_INFO(%rdx) - cmp TI_addr_limit(%rdx),%rax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -1: movzb (%rax),%edx +1: movzb (%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -47,12 +47,12 @@ ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - add $1,%rax + add $1,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%rdx) - cmp TI_addr_limit(%rdx),%rax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -2: movzwl -1(%rax),%edx +2: movzwl -1(%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -60,12 +60,12 @@ ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - add $3,%rax + add $3,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%rdx) - cmp TI_addr_limit(%rdx),%rax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -3: mov -3(%rax),%edx +3: mov -3(%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -73,12 +73,12 @@ ENDPROC(__get_user_4) ENTRY(__get_user_8) CFI_STARTPROC - add $7,%rax + add $7,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%rdx) - cmp TI_addr_limit(%rdx),%rax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -4: movq -7(%rax),%rdx +4: movq -7(%_ASM_AX),%_ASM_DX xor %eax,%eax ret CFI_ENDPROC @@ -87,7 +87,7 @@ ENDPROC(__get_user_8) bad_get_user: CFI_STARTPROC xor %edx,%edx - mov $(-EFAULT),%rax + mov $(-EFAULT),%_ASM_AX ret CFI_ENDPROC END(bad_get_user) -- cgit v1.2.3 From 87e2f1e7f6ab66306320403d4502d7938d3c703e Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 12:02:44 -0300 Subject: x86: use _ASM_PTR instead of explicit word-size pointers. Switch .long and .quad with _ASM_PTR in getuser*.S. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_32.S | 6 +++--- arch/x86/lib/getuser_64.S | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/getuser_32.S b/arch/x86/lib/getuser_32.S index 2cc3ceee8f9..2bb0a183e06 100644 --- a/arch/x86/lib/getuser_32.S +++ b/arch/x86/lib/getuser_32.S @@ -73,7 +73,7 @@ bad_get_user: END(bad_get_user) .section __ex_table,"a" - .long 1b,bad_get_user - .long 2b,bad_get_user - .long 3b,bad_get_user + _ASM_PTR 1b,bad_get_user + _ASM_PTR 2b,bad_get_user + _ASM_PTR 3b,bad_get_user .previous diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index 63b0e5c1e58..e33388419b7 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -93,8 +93,8 @@ bad_get_user: END(bad_get_user) .section __ex_table,"a" - .quad 1b,bad_get_user - .quad 2b,bad_get_user - .quad 3b,bad_get_user - .quad 4b,bad_get_user + _ASM_PTR 1b,bad_get_user + _ASM_PTR 2b,bad_get_user + _ASM_PTR 3b,bad_get_user + _ASM_PTR 4b,bad_get_user .previous -- cgit v1.2.3 From 6c2d458680d49d939ffd4b4cdc84d9e004d65910 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 12:05:11 -0300 Subject: x86: merge getuser asm functions. getuser_32.S and getuser_64.S are merged into getuser.S. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/Makefile | 2 +- arch/x86/lib/getuser.S | 104 ++++++++++++++++++++++++++++++++++++++++++++++ arch/x86/lib/getuser_32.S | 79 ----------------------------------- arch/x86/lib/getuser_64.S | 100 -------------------------------------------- 4 files changed, 105 insertions(+), 180 deletions(-) create mode 100644 arch/x86/lib/getuser.S delete mode 100644 arch/x86/lib/getuser_32.S delete mode 100644 arch/x86/lib/getuser_64.S (limited to 'arch') diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index 86960a6c41c..e92948203a5 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -5,7 +5,7 @@ obj-$(CONFIG_SMP) := msr-on-cpu.o lib-y := delay.o -lib-y += usercopy_$(BITS).o getuser_$(BITS).o putuser_$(BITS).o +lib-y += usercopy_$(BITS).o getuser.o putuser_$(BITS).o lib-y += memcpy_$(BITS).o ifeq ($(CONFIG_X86_32),y) diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S new file mode 100644 index 00000000000..ad374003742 --- /dev/null +++ b/arch/x86/lib/getuser.S @@ -0,0 +1,104 @@ +/* + * __get_user functions. + * + * (C) Copyright 1998 Linus Torvalds + * (C) Copyright 2005 Andi Kleen + * (C) Copyright 2008 Glauber Costa + * + * These functions have a non-standard call interface + * to make them more efficient, especially as they + * return an error value in addition to the "real" + * return value. + */ + +/* + * __get_user_X + * + * Inputs: %[r|e]ax contains the address. + * The register is modified, but all changes are undone + * before returning because the C code doesn't know about it. + * + * Outputs: %[r|e]ax is error code (0 or -EFAULT) + * %[r|e]dx contains zero-extended value + * + * + * These functions should not modify any other registers, + * as they get called from within inline assembly. + */ + +#include +#include +#include +#include +#include +#include +#include + + .text +ENTRY(__get_user_1) + CFI_STARTPROC + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX + jae bad_get_user +1: movzb (%_ASM_AX),%edx + xor %eax,%eax + ret + CFI_ENDPROC +ENDPROC(__get_user_1) + +ENTRY(__get_user_2) + CFI_STARTPROC + add $1,%_ASM_AX + jc bad_get_user + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX + jae bad_get_user +2: movzwl -1(%_ASM_AX),%edx + xor %eax,%eax + ret + CFI_ENDPROC +ENDPROC(__get_user_2) + +ENTRY(__get_user_4) + CFI_STARTPROC + add $3,%_ASM_AX + jc bad_get_user + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX + jae bad_get_user +3: mov -3(%_ASM_AX),%edx + xor %eax,%eax + ret + CFI_ENDPROC +ENDPROC(__get_user_4) + +#ifdef CONFIG_X86_64 +ENTRY(__get_user_8) + CFI_STARTPROC + add $7,%_ASM_AX + jc bad_get_user + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX + jae bad_get_user +4: movq -7(%_ASM_AX),%_ASM_DX + xor %eax,%eax + ret + CFI_ENDPROC +ENDPROC(__get_user_8) +#endif + +bad_get_user: + CFI_STARTPROC + xor %edx,%edx + mov $(-EFAULT),%_ASM_AX + ret + CFI_ENDPROC +END(bad_get_user) + +.section __ex_table,"a" + _ASM_PTR 1b,bad_get_user + _ASM_PTR 2b,bad_get_user + _ASM_PTR 3b,bad_get_user +#ifdef CONFIG_X86_64 + _ASM_PTR 4b,bad_get_user +#endif diff --git a/arch/x86/lib/getuser_32.S b/arch/x86/lib/getuser_32.S deleted file mode 100644 index 2bb0a183e06..00000000000 --- a/arch/x86/lib/getuser_32.S +++ /dev/null @@ -1,79 +0,0 @@ -/* - * __get_user functions. - * - * (C) Copyright 1998 Linus Torvalds - * - * These functions have a non-standard call interface - * to make them more efficient, especially as they - * return an error value in addition to the "real" - * return value. - */ -#include -#include -#include -#include - - -/* - * __get_user_X - * - * Inputs: %eax contains the address - * - * Outputs: %eax is error code (0 or -EFAULT) - * %edx contains zero-extended value - * - * These functions should not modify any other registers, - * as they get called from within inline assembly. - */ - -.text -ENTRY(__get_user_1) - CFI_STARTPROC - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -1: movzb (%_ASM_AX),%edx - xor %eax,%eax - ret - CFI_ENDPROC -ENDPROC(__get_user_1) - -ENTRY(__get_user_2) - CFI_STARTPROC - add $1,%_ASM_AX - jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -2: movzwl -1(%_ASM_AX),%edx - xor %eax,%eax - ret - CFI_ENDPROC -ENDPROC(__get_user_2) - -ENTRY(__get_user_4) - CFI_STARTPROC - add $3,%_ASM_AX - jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -3: mov -3(%_ASM_AX),%edx - xor %eax,%eax - ret - CFI_ENDPROC -ENDPROC(__get_user_4) - -bad_get_user: - CFI_STARTPROC - xor %edx,%edx - mov $-14,%_ASM_AX - ret - CFI_ENDPROC -END(bad_get_user) - -.section __ex_table,"a" - _ASM_PTR 1b,bad_get_user - _ASM_PTR 2b,bad_get_user - _ASM_PTR 3b,bad_get_user -.previous diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S deleted file mode 100644 index e33388419b7..00000000000 --- a/arch/x86/lib/getuser_64.S +++ /dev/null @@ -1,100 +0,0 @@ -/* - * __get_user functions. - * - * (C) Copyright 1998 Linus Torvalds - * (C) Copyright 2005 Andi Kleen - * - * These functions have a non-standard call interface - * to make them more efficient, especially as they - * return an error value in addition to the "real" - * return value. - */ - -/* - * __get_user_X - * - * Inputs: %rax contains the address. - * The register is modified, but all changes are undone - * before returning because the C code doesn't know about it. - * - * Outputs: %rax is error code (0 or -EFAULT) - * %rdx contains zero-extended value - * - * - * These functions should not modify any other registers, - * as they get called from within inline assembly. - */ - -#include -#include -#include -#include -#include -#include -#include - - .text -ENTRY(__get_user_1) - CFI_STARTPROC - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -1: movzb (%_ASM_AX),%edx - xor %eax,%eax - ret - CFI_ENDPROC -ENDPROC(__get_user_1) - -ENTRY(__get_user_2) - CFI_STARTPROC - add $1,%_ASM_AX - jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -2: movzwl -1(%_ASM_AX),%edx - xor %eax,%eax - ret - CFI_ENDPROC -ENDPROC(__get_user_2) - -ENTRY(__get_user_4) - CFI_STARTPROC - add $3,%_ASM_AX - jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -3: mov -3(%_ASM_AX),%edx - xor %eax,%eax - ret - CFI_ENDPROC -ENDPROC(__get_user_4) - -ENTRY(__get_user_8) - CFI_STARTPROC - add $7,%_ASM_AX - jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -4: movq -7(%_ASM_AX),%_ASM_DX - xor %eax,%eax - ret - CFI_ENDPROC -ENDPROC(__get_user_8) - -bad_get_user: - CFI_STARTPROC - xor %edx,%edx - mov $(-EFAULT),%_ASM_AX - ret - CFI_ENDPROC -END(bad_get_user) - -.section __ex_table,"a" - _ASM_PTR 1b,bad_get_user - _ASM_PTR 2b,bad_get_user - _ASM_PTR 3b,bad_get_user - _ASM_PTR 4b,bad_get_user -.previous -- cgit v1.2.3 From 268cf048c890d10bd3a86bd87922ed8a722d502f Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 12:40:55 -0300 Subject: x86: don't save ebx in putuser_32.S. Clobber it in the inline asm macros, and let the compiler do this for us. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_32.S | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/putuser_32.S b/arch/x86/lib/putuser_32.S index f58fba109d1..5b2a926f0e2 100644 --- a/arch/x86/lib/putuser_32.S +++ b/arch/x86/lib/putuser_32.S @@ -26,14 +26,8 @@ */ #define ENTER CFI_STARTPROC ; \ - pushl %ebx ; \ - CFI_ADJUST_CFA_OFFSET 4 ; \ - CFI_REL_OFFSET ebx, 0 ; \ GET_THREAD_INFO(%ebx) -#define EXIT popl %ebx ; \ - CFI_ADJUST_CFA_OFFSET -4 ; \ - CFI_RESTORE ebx ; \ - ret ; \ +#define EXIT ret ; \ CFI_ENDPROC .text @@ -81,10 +75,7 @@ ENTRY(__put_user_8) ENDPROC(__put_user_8) bad_put_user: - CFI_STARTPROC simple - CFI_DEF_CFA esp, 2*4 - CFI_OFFSET eip, -1*4 - CFI_OFFSET ebx, -2*4 + CFI_STARTPROC movl $-14,%eax EXIT END(bad_put_user) -- cgit v1.2.3 From 770546b99fb99e71a3aa4181980d42664f9c18bd Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 15:03:40 -0300 Subject: x86: clobber rbx in putuser_64.S. Instead of clobbering r8, clobber rbx, which is the i386 way. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_64.S | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index 940796fa0d9..07028851064 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -18,7 +18,7 @@ * * Outputs: %rax is error code (0 or -EFAULT) * - * %r8 is destroyed. + * %rbx is destroyed. * * These functions should not modify any other registers, * as they get called from within inline assembly. @@ -34,8 +34,8 @@ .text ENTRY(__put_user_1) CFI_STARTPROC - GET_THREAD_INFO(%r8) - cmpq TI_addr_limit(%r8),%rcx + GET_THREAD_INFO(%rbx) + cmpq TI_addr_limit(%rbx),%rcx jae bad_put_user 1: movb %dl,(%rcx) xorl %eax,%eax @@ -45,10 +45,10 @@ ENDPROC(__put_user_1) ENTRY(__put_user_2) CFI_STARTPROC - GET_THREAD_INFO(%r8) + GET_THREAD_INFO(%rbx) addq $1,%rcx jc 20f - cmpq TI_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%rbx),%rcx jae 20f decq %rcx 2: movw %dx,(%rcx) @@ -61,10 +61,10 @@ ENDPROC(__put_user_2) ENTRY(__put_user_4) CFI_STARTPROC - GET_THREAD_INFO(%r8) + GET_THREAD_INFO(%rbx) addq $3,%rcx jc 30f - cmpq TI_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%rbx),%rcx jae 30f subq $3,%rcx 3: movl %edx,(%rcx) @@ -77,10 +77,10 @@ ENDPROC(__put_user_4) ENTRY(__put_user_8) CFI_STARTPROC - GET_THREAD_INFO(%r8) + GET_THREAD_INFO(%rbx) addq $7,%rcx jc 40f - cmpq TI_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%rbx),%rcx jae 40f subq $7,%rcx 4: movq %rdx,(%rcx) -- cgit v1.2.3 From 0ada3164031162b4e1b7ff6b36ba8cc80ff7fe96 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 16:44:39 -0300 Subject: x86: pass argument to putuser_64 functions in ax register. This is consistent with i386 usage. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_64.S | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index 07028851064..ce5fcd5d8c1 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -37,7 +37,7 @@ ENTRY(__put_user_1) GET_THREAD_INFO(%rbx) cmpq TI_addr_limit(%rbx),%rcx jae bad_put_user -1: movb %dl,(%rcx) +1: movb %al,(%rcx) xorl %eax,%eax ret CFI_ENDPROC @@ -51,7 +51,7 @@ ENTRY(__put_user_2) cmpq TI_addr_limit(%rbx),%rcx jae 20f decq %rcx -2: movw %dx,(%rcx) +2: movw %ax,(%rcx) xorl %eax,%eax ret 20: decq %rcx @@ -67,7 +67,7 @@ ENTRY(__put_user_4) cmpq TI_addr_limit(%rbx),%rcx jae 30f subq $3,%rcx -3: movl %edx,(%rcx) +3: movl %eax,(%rcx) xorl %eax,%eax ret 30: subq $3,%rcx @@ -83,7 +83,7 @@ ENTRY(__put_user_8) cmpq TI_addr_limit(%rbx),%rcx jae 40f subq $7,%rcx -4: movq %rdx,(%rcx) +4: movq %rax,(%rcx) xorl %eax,%eax ret 40: subq $7,%rcx -- cgit v1.2.3 From 663aa96df32af9c4141ef3179282f95c7537643a Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 16:51:59 -0300 Subject: x86: change testing logic in putuser_64.S. Instead of operating over a register we need to put back into normal state afterwards (the memory position), just sub from rbx, which is trashed anyway. We can save a few instructions. Also, this is the i386 way. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_64.S | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index ce5fcd5d8c1..a96bd8a5298 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -46,48 +46,39 @@ ENDPROC(__put_user_1) ENTRY(__put_user_2) CFI_STARTPROC GET_THREAD_INFO(%rbx) - addq $1,%rcx - jc 20f - cmpq TI_addr_limit(%rbx),%rcx - jae 20f - decq %rcx + mov TI_addr_limit(%rbx),%rbx + sub $1, %rbx + cmpq %rbx ,%rcx + jae bad_put_user 2: movw %ax,(%rcx) xorl %eax,%eax ret -20: decq %rcx - jmp bad_put_user CFI_ENDPROC ENDPROC(__put_user_2) ENTRY(__put_user_4) CFI_STARTPROC GET_THREAD_INFO(%rbx) - addq $3,%rcx - jc 30f - cmpq TI_addr_limit(%rbx),%rcx - jae 30f - subq $3,%rcx + mov TI_addr_limit(%rbx),%rbx + sub $3, %rbx + cmp %rbx, %rcx + jae bad_put_user 3: movl %eax,(%rcx) xorl %eax,%eax ret -30: subq $3,%rcx - jmp bad_put_user CFI_ENDPROC ENDPROC(__put_user_4) ENTRY(__put_user_8) CFI_STARTPROC GET_THREAD_INFO(%rbx) - addq $7,%rcx - jc 40f - cmpq TI_addr_limit(%rbx),%rcx - jae 40f - subq $7,%rcx + mov TI_addr_limit(%rbx),%rbx + sub $7, %rbx + cmp %rbx, %rcx + jae bad_put_user 4: movq %rax,(%rcx) xorl %eax,%eax ret -40: subq $7,%rcx - jmp bad_put_user CFI_ENDPROC ENDPROC(__put_user_8) -- cgit v1.2.3 From 766ed4282114eab616741107745b0dd11075e496 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 16:56:30 -0300 Subject: x86: replace function headers by macros. In putuser_64.S, do it the i386 way, and replace the code in beginning and end of functions with macros, since it's always the same thing. Save lines. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_64.S | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index a96bd8a5298..6d7513bf885 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -31,62 +31,58 @@ #include #include +#define ENTER CFI_STARTPROC ; \ + GET_THREAD_INFO(%rbx) +#define EXIT ret ; \ + CFI_ENDPROC + .text ENTRY(__put_user_1) - CFI_STARTPROC - GET_THREAD_INFO(%rbx) + ENTER cmpq TI_addr_limit(%rbx),%rcx jae bad_put_user 1: movb %al,(%rcx) xorl %eax,%eax - ret - CFI_ENDPROC + EXIT ENDPROC(__put_user_1) ENTRY(__put_user_2) - CFI_STARTPROC - GET_THREAD_INFO(%rbx) + ENTER mov TI_addr_limit(%rbx),%rbx sub $1, %rbx cmpq %rbx ,%rcx jae bad_put_user 2: movw %ax,(%rcx) xorl %eax,%eax - ret - CFI_ENDPROC + EXIT ENDPROC(__put_user_2) ENTRY(__put_user_4) - CFI_STARTPROC - GET_THREAD_INFO(%rbx) + ENTER mov TI_addr_limit(%rbx),%rbx sub $3, %rbx cmp %rbx, %rcx jae bad_put_user 3: movl %eax,(%rcx) xorl %eax,%eax - ret - CFI_ENDPROC + EXIT ENDPROC(__put_user_4) ENTRY(__put_user_8) - CFI_STARTPROC - GET_THREAD_INFO(%rbx) + ENTER mov TI_addr_limit(%rbx),%rbx sub $7, %rbx cmp %rbx, %rcx jae bad_put_user 4: movq %rax,(%rcx) xorl %eax,%eax - ret - CFI_ENDPROC + EXIT ENDPROC(__put_user_8) bad_put_user: CFI_STARTPROC movq $(-EFAULT),%rax - ret - CFI_ENDPROC + EXIT END(bad_put_user) .section __ex_table,"a" -- cgit v1.2.3 From efea505d83873cfc8a7cdbb8a2a11d2c67467843 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 16:59:05 -0300 Subject: x86: don't use word-size specifiers in putuser files. Remove them where unambiguous. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_32.S | 28 ++++++++++++++-------------- arch/x86/lib/putuser_64.S | 14 +++++++------- 2 files changed, 21 insertions(+), 21 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/putuser_32.S b/arch/x86/lib/putuser_32.S index 5b2a926f0e2..b67a37cab1b 100644 --- a/arch/x86/lib/putuser_32.S +++ b/arch/x86/lib/putuser_32.S @@ -33,44 +33,44 @@ .text ENTRY(__put_user_1) ENTER - cmpl TI_addr_limit(%ebx),%ecx + cmp TI_addr_limit(%ebx),%ecx jae bad_put_user 1: movb %al,(%ecx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_1) ENTRY(__put_user_2) ENTER - movl TI_addr_limit(%ebx),%ebx - subl $1,%ebx - cmpl %ebx,%ecx + mov TI_addr_limit(%ebx),%ebx + sub $1,%ebx + cmp %ebx,%ecx jae bad_put_user 2: movw %ax,(%ecx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_2) ENTRY(__put_user_4) ENTER - movl TI_addr_limit(%ebx),%ebx - subl $3,%ebx - cmpl %ebx,%ecx + mov TI_addr_limit(%ebx),%ebx + sub $3,%ebx + cmp %ebx,%ecx jae bad_put_user 3: movl %eax,(%ecx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_4) ENTRY(__put_user_8) ENTER - movl TI_addr_limit(%ebx),%ebx - subl $7,%ebx - cmpl %ebx,%ecx + mov TI_addr_limit(%ebx),%ebx + sub $7,%ebx + cmp %ebx,%ecx jae bad_put_user 4: movl %eax,(%ecx) 5: movl %edx,4(%ecx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_8) diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index 6d7513bf885..c18fc0f5256 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -39,10 +39,10 @@ .text ENTRY(__put_user_1) ENTER - cmpq TI_addr_limit(%rbx),%rcx + cmp TI_addr_limit(%rbx),%rcx jae bad_put_user 1: movb %al,(%rcx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_1) @@ -50,10 +50,10 @@ ENTRY(__put_user_2) ENTER mov TI_addr_limit(%rbx),%rbx sub $1, %rbx - cmpq %rbx ,%rcx + cmp %rbx ,%rcx jae bad_put_user 2: movw %ax,(%rcx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_2) @@ -64,7 +64,7 @@ ENTRY(__put_user_4) cmp %rbx, %rcx jae bad_put_user 3: movl %eax,(%rcx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_4) @@ -75,13 +75,13 @@ ENTRY(__put_user_8) cmp %rbx, %rcx jae bad_put_user 4: movq %rax,(%rcx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_8) bad_put_user: CFI_STARTPROC - movq $(-EFAULT),%rax + mov $(-EFAULT),%rax EXIT END(bad_put_user) -- cgit v1.2.3 From 2528de431ddb200653d1dc6ca90074bad9520f09 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 17:36:31 -0300 Subject: x86: use macros from asm.h. In putuser_32.S and putuser_64.S, replace things like .quad, .long, and explicit references to [r|e]ax for the apropriate macros in asm/asm.h. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_32.S | 43 ++++++++++++++++++++++--------------------- arch/x86/lib/putuser_64.S | 41 +++++++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 41 deletions(-) (limited to 'arch') diff --git a/arch/x86/lib/putuser_32.S b/arch/x86/lib/putuser_32.S index b67a37cab1b..e7eda34feb3 100644 --- a/arch/x86/lib/putuser_32.S +++ b/arch/x86/lib/putuser_32.S @@ -11,6 +11,7 @@ #include #include #include +#include /* @@ -26,50 +27,50 @@ */ #define ENTER CFI_STARTPROC ; \ - GET_THREAD_INFO(%ebx) + GET_THREAD_INFO(%_ASM_BX) #define EXIT ret ; \ CFI_ENDPROC .text ENTRY(__put_user_1) ENTER - cmp TI_addr_limit(%ebx),%ecx + cmp TI_addr_limit(%_ASM_BX),%_ASM_CX jae bad_put_user -1: movb %al,(%ecx) +1: movb %al,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_1) ENTRY(__put_user_2) ENTER - mov TI_addr_limit(%ebx),%ebx - sub $1,%ebx - cmp %ebx,%ecx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $1,%_ASM_BX + cmp %_ASM_BX,%_ASM_CX jae bad_put_user -2: movw %ax,(%ecx) +2: movw %ax,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_2) ENTRY(__put_user_4) ENTER - mov TI_addr_limit(%ebx),%ebx - sub $3,%ebx - cmp %ebx,%ecx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $3,%_ASM_BX + cmp %_ASM_BX,%_ASM_CX jae bad_put_user -3: movl %eax,(%ecx) +3: movl %eax,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_4) ENTRY(__put_user_8) ENTER - mov TI_addr_limit(%ebx),%ebx - sub $7,%ebx - cmp %ebx,%ecx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $7,%_ASM_BX + cmp %_ASM_BX,%_ASM_CX jae bad_put_user -4: movl %eax,(%ecx) -5: movl %edx,4(%ecx) +4: movl %_ASM_AX,(%_ASM_CX) +5: movl %edx,4(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_8) @@ -81,9 +82,9 @@ bad_put_user: END(bad_put_user) .section __ex_table,"a" - .long 1b,bad_put_user - .long 2b,bad_put_user - .long 3b,bad_put_user - .long 4b,bad_put_user - .long 5b,bad_put_user + _ASM_PTR 1b,bad_put_user + _ASM_PTR 2b,bad_put_user + _ASM_PTR 3b,bad_put_user + _ASM_PTR 4b,bad_put_user + _ASM_PTR 5b,bad_put_user .previous diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index c18fc0f5256..d496cc8e730 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -30,64 +30,65 @@ #include #include #include +#include #define ENTER CFI_STARTPROC ; \ - GET_THREAD_INFO(%rbx) + GET_THREAD_INFO(%_ASM_BX) #define EXIT ret ; \ CFI_ENDPROC .text ENTRY(__put_user_1) ENTER - cmp TI_addr_limit(%rbx),%rcx + cmp TI_addr_limit(%_ASM_BX),%_ASM_CX jae bad_put_user -1: movb %al,(%rcx) +1: movb %al,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_1) ENTRY(__put_user_2) ENTER - mov TI_addr_limit(%rbx),%rbx - sub $1, %rbx - cmp %rbx ,%rcx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $1, %_ASM_BX + cmp %_ASM_BX ,%_ASM_CX jae bad_put_user -2: movw %ax,(%rcx) +2: movw %ax,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_2) ENTRY(__put_user_4) ENTER - mov TI_addr_limit(%rbx),%rbx - sub $3, %rbx - cmp %rbx, %rcx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $3, %_ASM_BX + cmp %_ASM_BX, %_ASM_CX jae bad_put_user -3: movl %eax,(%rcx) +3: movl %eax,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_4) ENTRY(__put_user_8) ENTER - mov TI_addr_limit(%rbx),%rbx - sub $7, %rbx - cmp %rbx, %rcx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $7, %_ASM_BX + cmp %_ASM_BX, %_ASM_CX jae bad_put_user -4: movq %rax,(%rcx) +4: movq %_ASM_AX,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_8) bad_put_user: CFI_STARTPROC - mov $(-EFAULT),%rax + mov $(-EFAULT),%eax EXIT END(bad_put_user) .section __ex_table,"a" - .quad 1b,bad_put_user - .quad 2b,bad_put_user - .quad 3b,bad_put_user - .quad 4b,bad_put_user + _ASM_PTR 1b,bad_put_user + _ASM_PTR 2b,bad_put_user + _ASM_PTR 3b,bad_put_user + _ASM_PTR 4b,bad_put_user .previous -- cgit v1.2.3 From 5cbbc3b1eb37bdc72eefd2de03b39f5e784400c2 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 17:40:14 -0300 Subject: x86: merge putuser asm functions. putuser_32.S and putuser_64.S are merged into putuser.S. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/Makefile | 2 +- arch/x86/lib/putuser.S | 97 +++++++++++++++++++++++++++++++++++++++++++++++ arch/x86/lib/putuser_32.S | 90 ------------------------------------------- arch/x86/lib/putuser_64.S | 94 --------------------------------------------- 4 files changed, 98 insertions(+), 185 deletions(-) create mode 100644 arch/x86/lib/putuser.S delete mode 100644 arch/x86/lib/putuser_32.S delete mode 100644 arch/x86/lib/putuser_64.S (limited to 'arch') diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index e92948203a5..83226e0a7ce 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -5,7 +5,7 @@ obj-$(CONFIG_SMP) := msr-on-cpu.o lib-y := delay.o -lib-y += usercopy_$(BITS).o getuser.o putuser_$(BITS).o +lib-y += usercopy_$(BITS).o getuser.o putuser.o lib-y += memcpy_$(BITS).o ifeq ($(CONFIG_X86_32),y) diff --git a/arch/x86/lib/putuser.S b/arch/x86/lib/putuser.S new file mode 100644 index 00000000000..36b0d15ae6e --- /dev/null +++ b/arch/x86/lib/putuser.S @@ -0,0 +1,97 @@ +/* + * __put_user functions. + * + * (C) Copyright 2005 Linus Torvalds + * (C) Copyright 2005 Andi Kleen + * (C) Copyright 2008 Glauber Costa + * + * These functions have a non-standard call interface + * to make them more efficient, especially as they + * return an error value in addition to the "real" + * return value. + */ +#include +#include +#include +#include +#include + + +/* + * __put_user_X + * + * Inputs: %eax[:%edx] contains the data + * %ecx contains the address + * + * Outputs: %eax is error code (0 or -EFAULT) + * + * These functions should not modify any other registers, + * as they get called from within inline assembly. + */ + +#define ENTER CFI_STARTPROC ; \ + GET_THREAD_INFO(%_ASM_BX) +#define EXIT ret ; \ + CFI_ENDPROC + +.text +ENTRY(__put_user_1) + ENTER + cmp TI_addr_limit(%_ASM_BX),%_ASM_CX + jae bad_put_user +1: movb %al,(%_ASM_CX) + xor %eax,%eax + EXIT +ENDPROC(__put_user_1) + +ENTRY(__put_user_2) + ENTER + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $1,%_ASM_BX + cmp %_ASM_BX,%_ASM_CX + jae bad_put_user +2: movw %ax,(%_ASM_CX) + xor %eax,%eax + EXIT +ENDPROC(__put_user_2) + +ENTRY(__put_user_4) + ENTER + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $3,%_ASM_BX + cmp %_ASM_BX,%_ASM_CX + jae bad_put_user +3: movl %eax,(%_ASM_CX) + xor %eax,%eax + EXIT +ENDPROC(__put_user_4) + +ENTRY(__put_user_8) + ENTER + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $7,%_ASM_BX + cmp %_ASM_BX,%_ASM_CX + jae bad_put_user +4: mov %_ASM_AX,(%_ASM_CX) +#ifdef CONFIG_X86_32 +5: movl %edx,4(%_ASM_CX) +#endif + xor %eax,%eax + EXIT +ENDPROC(__put_user_8) + +bad_put_user: + CFI_STARTPROC + movl $-EFAULT,%eax + EXIT +END(bad_put_user) + +.section __ex_table,"a" + _ASM_PTR 1b,bad_put_user + _ASM_PTR 2b,bad_put_user + _ASM_PTR 3b,bad_put_user + _ASM_PTR 4b,bad_put_user +#ifdef CONFIG_X86_32 + _ASM_PTR 5b,bad_put_user +#endif +.previous diff --git a/arch/x86/lib/putuser_32.S b/arch/x86/lib/putuser_32.S deleted file mode 100644 index e7eda34feb3..00000000000 --- a/arch/x86/lib/putuser_32.S +++ /dev/null @@ -1,90 +0,0 @@ -/* - * __put_user functions. - * - * (C) Copyright 2005 Linus Torvalds - * - * These functions have a non-standard call interface - * to make them more efficient, especially as they - * return an error value in addition to the "real" - * return value. - */ -#include -#include -#include -#include - - -/* - * __put_user_X - * - * Inputs: %eax[:%edx] contains the data - * %ecx contains the address - * - * Outputs: %eax is error code (0 or -EFAULT) - * - * These functions should not modify any other registers, - * as they get called from within inline assembly. - */ - -#define ENTER CFI_STARTPROC ; \ - GET_THREAD_INFO(%_ASM_BX) -#define EXIT ret ; \ - CFI_ENDPROC - -.text -ENTRY(__put_user_1) - ENTER - cmp TI_addr_limit(%_ASM_BX),%_ASM_CX - jae bad_put_user -1: movb %al,(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_1) - -ENTRY(__put_user_2) - ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $1,%_ASM_BX - cmp %_ASM_BX,%_ASM_CX - jae bad_put_user -2: movw %ax,(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_2) - -ENTRY(__put_user_4) - ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $3,%_ASM_BX - cmp %_ASM_BX,%_ASM_CX - jae bad_put_user -3: movl %eax,(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_4) - -ENTRY(__put_user_8) - ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $7,%_ASM_BX - cmp %_ASM_BX,%_ASM_CX - jae bad_put_user -4: movl %_ASM_AX,(%_ASM_CX) -5: movl %edx,4(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_8) - -bad_put_user: - CFI_STARTPROC - movl $-14,%eax - EXIT -END(bad_put_user) - -.section __ex_table,"a" - _ASM_PTR 1b,bad_put_user - _ASM_PTR 2b,bad_put_user - _ASM_PTR 3b,bad_put_user - _ASM_PTR 4b,bad_put_user - _ASM_PTR 5b,bad_put_user -.previous diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S deleted file mode 100644 index d496cc8e730..00000000000 --- a/arch/x86/lib/putuser_64.S +++ /dev/null @@ -1,94 +0,0 @@ -/* - * __put_user functions. - * - * (C) Copyright 1998 Linus Torvalds - * (C) Copyright 2005 Andi Kleen - * - * These functions have a non-standard call interface - * to make them more efficient, especially as they - * return an error value in addition to the "real" - * return value. - */ - -/* - * __put_user_X - * - * Inputs: %rcx contains the address - * %rdx contains new value - * - * Outputs: %rax is error code (0 or -EFAULT) - * - * %rbx is destroyed. - * - * These functions should not modify any other registers, - * as they get called from within inline assembly. - */ - -#include -#include -#include -#include -#include -#include -#include - -#define ENTER CFI_STARTPROC ; \ - GET_THREAD_INFO(%_ASM_BX) -#define EXIT ret ; \ - CFI_ENDPROC - - .text -ENTRY(__put_user_1) - ENTER - cmp TI_addr_limit(%_ASM_BX),%_ASM_CX - jae bad_put_user -1: movb %al,(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_1) - -ENTRY(__put_user_2) - ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $1, %_ASM_BX - cmp %_ASM_BX ,%_ASM_CX - jae bad_put_user -2: movw %ax,(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_2) - -ENTRY(__put_user_4) - ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $3, %_ASM_BX - cmp %_ASM_BX, %_ASM_CX - jae bad_put_user -3: movl %eax,(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_4) - -ENTRY(__put_user_8) - ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $7, %_ASM_BX - cmp %_ASM_BX, %_ASM_CX - jae bad_put_user -4: movq %_ASM_AX,(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_8) - -bad_put_user: - CFI_STARTPROC - mov $(-EFAULT),%eax - EXIT -END(bad_put_user) - -.section __ex_table,"a" - _ASM_PTR 1b,bad_put_user - _ASM_PTR 2b,bad_put_user - _ASM_PTR 3b,bad_put_user - _ASM_PTR 4b,bad_put_user -.previous -- cgit v1.2.3 From 293e6a2524ae775fef41d21d17af288d4b2ee91e Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 11:40:42 -0300 Subject: x86: mark x86_64 as having a working WP. Select X86_WP_WORKS_OK for x86_64 too. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/Kconfig.cpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu index 3d22bb8175b..abff1b84ed5 100644 --- a/arch/x86/Kconfig.cpu +++ b/arch/x86/Kconfig.cpu @@ -344,7 +344,7 @@ config X86_F00F_BUG config X86_WP_WORKS_OK def_bool y - depends on X86_32 && !M386 + depends on !M386 config X86_INVLPG def_bool y -- cgit v1.2.3