diff options
Diffstat (limited to 'include/asm-x86')
200 files changed, 6729 insertions, 5763 deletions
diff --git a/include/asm-x86/Kbuild b/include/asm-x86/Kbuild index 3b8160a2b47..1e3554596f7 100644 --- a/include/asm-x86/Kbuild +++ b/include/asm-x86/Kbuild @@ -10,6 +10,7 @@ header-y += prctl.h header-y += ptrace-abi.h header-y += sigcontext32.h header-y += ucontext.h +header-y += processor-flags.h unifdef-y += e820.h unifdef-y += ist.h diff --git a/include/asm-x86/a.out-core.h b/include/asm-x86/a.out-core.h index d2b6e11d3e9..714207a1c38 100644 --- a/include/asm-x86/a.out-core.h +++ b/include/asm-x86/a.out-core.h @@ -29,8 +29,9 @@ static inline void aout_dump_thread(struct pt_regs *regs, struct user *dump) dump->magic = CMAGIC; dump->start_code = 0; dump->start_stack = regs->sp & ~(PAGE_SIZE - 1); - dump->u_tsize = ((unsigned long) current->mm->end_code) >> PAGE_SHIFT; - dump->u_dsize = ((unsigned long) (current->mm->brk + (PAGE_SIZE-1))) >> PAGE_SHIFT; + dump->u_tsize = ((unsigned long)current->mm->end_code) >> PAGE_SHIFT; + dump->u_dsize = ((unsigned long)(current->mm->brk + (PAGE_SIZE - 1))) + >> PAGE_SHIFT; dump->u_dsize -= dump->u_tsize; dump->u_ssize = 0; dump->u_debugreg[0] = current->thread.debugreg0; @@ -43,7 +44,8 @@ static inline void aout_dump_thread(struct pt_regs *regs, struct user *dump) dump->u_debugreg[7] = current->thread.debugreg7; if (dump->start_stack < TASK_SIZE) - dump->u_ssize = ((unsigned long) (TASK_SIZE - dump->start_stack)) >> PAGE_SHIFT; + dump->u_ssize = ((unsigned long)(TASK_SIZE - dump->start_stack)) + >> PAGE_SHIFT; dump->regs.bx = regs->bx; dump->regs.cx = regs->cx; @@ -55,7 +57,7 @@ static inline void aout_dump_thread(struct pt_regs *regs, struct user *dump) dump->regs.ds = (u16)regs->ds; dump->regs.es = (u16)regs->es; dump->regs.fs = (u16)regs->fs; - savesegment(gs,gs); + savesegment(gs, gs); dump->regs.orig_ax = regs->orig_ax; dump->regs.ip = regs->ip; dump->regs.cs = (u16)regs->cs; @@ -63,7 +65,7 @@ static inline void aout_dump_thread(struct pt_regs *regs, struct user *dump) dump->regs.sp = regs->sp; dump->regs.ss = (u16)regs->ss; - dump->u_fpvalid = dump_fpu (regs, &dump->i387); + dump->u_fpvalid = dump_fpu(regs, &dump->i387); } #endif /* CONFIG_X86_32 */ diff --git a/include/asm-x86/acpi.h b/include/asm-x86/acpi.h index 7a72d6aa50b..14411c9de46 100644 --- a/include/asm-x86/acpi.h +++ b/include/asm-x86/acpi.h @@ -67,16 +67,16 @@ int __acpi_release_global_lock(unsigned int *lock); */ #define ACPI_DIV_64_BY_32(n_hi, n_lo, d32, q32, r32) \ asm("divl %2;" \ - :"=a"(q32), "=d"(r32) \ - :"r"(d32), \ + : "=a"(q32), "=d"(r32) \ + : "r"(d32), \ "0"(n_lo), "1"(n_hi)) #define ACPI_SHIFT_RIGHT_64(n_hi, n_lo) \ asm("shrl $1,%2 ;" \ "rcrl $1,%3;" \ - :"=r"(n_hi), "=r"(n_lo) \ - :"0"(n_hi), "1"(n_lo)) + : "=r"(n_hi), "=r"(n_lo) \ + : "0"(n_hi), "1"(n_lo)) #ifdef CONFIG_ACPI extern int acpi_lapic; diff --git a/include/asm-x86/alternative.h b/include/asm-x86/alternative.h index d8bacf3c4b0..1f6a9ca1012 100644 --- a/include/asm-x86/alternative.h +++ b/include/asm-x86/alternative.h @@ -66,8 +66,8 @@ extern void alternatives_smp_module_del(struct module *mod); extern void alternatives_smp_switch(int smp); #else static inline void alternatives_smp_module_add(struct module *mod, char *name, - void *locks, void *locks_end, - void *text, void *text_end) {} + void *locks, void *locks_end, + void *text, void *text_end) {} static inline void alternatives_smp_module_del(struct module *mod) {} static inline void alternatives_smp_switch(int smp) {} #endif /* CONFIG_SMP */ @@ -148,14 +148,34 @@ struct paravirt_patch_site; void apply_paravirt(struct paravirt_patch_site *start, struct paravirt_patch_site *end); #else -static inline void -apply_paravirt(struct paravirt_patch_site *start, - struct paravirt_patch_site *end) +static inline void apply_paravirt(struct paravirt_patch_site *start, + struct paravirt_patch_site *end) {} #define __parainstructions NULL #define __parainstructions_end NULL #endif -extern void text_poke(void *addr, unsigned char *opcode, int len); +extern void add_nops(void *insns, unsigned int len); + +/* + * Clear and restore the kernel write-protection flag on the local CPU. + * Allows the kernel to edit read-only pages. + * Side-effect: any interrupt handler running between save and restore will have + * the ability to write to read-only pages. + * + * Warning: + * Code patching in the UP case is safe if NMIs and MCE handlers are stopped and + * no thread can be preempted in the instructions being modified (no iret to an + * invalid instruction possible) or if the instructions are changed from a + * consistent state to another consistent state atomically. + * More care must be taken when modifying code in the SMP case because of + * Intel's errata. + * On the local CPU you need to be protected again NMI or MCE handlers seeing an + * inconsistent instruction while you patch. + * The _early version expects the memory to already be RW. + */ + +extern void *text_poke(void *addr, const void *opcode, size_t len); +extern void *text_poke_early(void *addr, const void *opcode, size_t len); #endif /* _ASM_X86_ALTERNATIVE_H */ diff --git a/include/asm-x86/apic.h b/include/asm-x86/apic.h index bcfc07fd366..be9639a9a18 100644 --- a/include/asm-x86/apic.h +++ b/include/asm-x86/apic.h @@ -44,7 +44,6 @@ extern int apic_runs_main_timer; extern int ioapic_force; extern int disable_apic; extern int disable_apic_timer; -extern unsigned boot_cpu_id; /* * Basic functions accessing APICs. @@ -59,6 +58,8 @@ extern unsigned boot_cpu_id; #define setup_secondary_clock setup_secondary_APIC_clock #endif +extern int is_vsmp_box(void); + static inline void native_apic_write(unsigned long reg, u32 v) { *((volatile u32 *)(APIC_BASE + reg)) = v; @@ -66,7 +67,7 @@ static inline void native_apic_write(unsigned long reg, u32 v) static inline void native_apic_write_atomic(unsigned long reg, u32 v) { - (void) xchg((u32*)(APIC_BASE + reg), v); + (void)xchg((u32 *)(APIC_BASE + reg), v); } static inline u32 native_apic_read(unsigned long reg) @@ -123,7 +124,7 @@ extern void enable_NMI_through_LVT0(void); * On 32bit this is mach-xxx local */ #ifdef CONFIG_X86_64 -extern void setup_apic_routing(void); +extern void early_init_lapic_mapping(void); #endif extern u8 setup_APIC_eilvt_mce(u8 vector, u8 msg_type, u8 mask); diff --git a/include/asm-x86/apicdef.h b/include/asm-x86/apicdef.h index 550af7a6f88..6b9008c7873 100644 --- a/include/asm-x86/apicdef.h +++ b/include/asm-x86/apicdef.h @@ -12,17 +12,15 @@ #define APIC_ID 0x20 -#ifdef CONFIG_X86_64 -# define APIC_ID_MASK (0xFFu<<24) -# define GET_APIC_ID(x) (((x)>>24)&0xFFu) -# define SET_APIC_ID(x) (((x)<<24)) -#endif - #define APIC_LVR 0x30 #define APIC_LVR_MASK 0xFF00FF -#define GET_APIC_VERSION(x) ((x)&0xFFu) -#define GET_APIC_MAXLVT(x) (((x)>>16)&0xFFu) -#define APIC_INTEGRATED(x) ((x)&0xF0u) +#define GET_APIC_VERSION(x) ((x) & 0xFFu) +#define GET_APIC_MAXLVT(x) (((x) >> 16) & 0xFFu) +#ifdef CONFIG_X86_32 +# define APIC_INTEGRATED(x) ((x) & 0xF0u) +#else +# define APIC_INTEGRATED(x) (1) +#endif #define APIC_XAPIC(x) ((x) >= 0x14) #define APIC_TASKPRI 0x80 #define APIC_TPRI_MASK 0xFFu @@ -33,16 +31,16 @@ #define APIC_EIO_ACK 0x0 #define APIC_RRR 0xC0 #define APIC_LDR 0xD0 -#define APIC_LDR_MASK (0xFFu<<24) -#define GET_APIC_LOGICAL_ID(x) (((x)>>24)&0xFFu) -#define SET_APIC_LOGICAL_ID(x) (((x)<<24)) +#define APIC_LDR_MASK (0xFFu << 24) +#define GET_APIC_LOGICAL_ID(x) (((x) >> 24) & 0xFFu) +#define SET_APIC_LOGICAL_ID(x) (((x) << 24)) #define APIC_ALL_CPUS 0xFFu #define APIC_DFR 0xE0 #define APIC_DFR_CLUSTER 0x0FFFFFFFul #define APIC_DFR_FLAT 0xFFFFFFFFul #define APIC_SPIV 0xF0 -#define APIC_SPIV_FOCUS_DISABLED (1<<9) -#define APIC_SPIV_APIC_ENABLED (1<<8) +#define APIC_SPIV_FOCUS_DISABLED (1 << 9) +#define APIC_SPIV_APIC_ENABLED (1 << 8) #define APIC_ISR 0x100 #define APIC_ISR_NR 0x8 /* Number of 32 bit ISR registers. */ #define APIC_TMR 0x180 @@ -78,27 +76,27 @@ #define APIC_DM_EXTINT 0x00700 #define APIC_VECTOR_MASK 0x000FF #define APIC_ICR2 0x310 -#define GET_APIC_DEST_FIELD(x) (((x)>>24)&0xFF) -#define SET_APIC_DEST_FIELD(x) ((x)<<24) +#define GET_APIC_DEST_FIELD(x) (((x) >> 24) & 0xFF) +#define SET_APIC_DEST_FIELD(x) ((x) << 24) #define APIC_LVTT 0x320 #define APIC_LVTTHMR 0x330 #define APIC_LVTPC 0x340 #define APIC_LVT0 0x350 -#define APIC_LVT_TIMER_BASE_MASK (0x3<<18) -#define GET_APIC_TIMER_BASE(x) (((x)>>18)&0x3) -#define SET_APIC_TIMER_BASE(x) (((x)<<18)) +#define APIC_LVT_TIMER_BASE_MASK (0x3 << 18) +#define GET_APIC_TIMER_BASE(x) (((x) >> 18) & 0x3) +#define SET_APIC_TIMER_BASE(x) (((x) << 18)) #define APIC_TIMER_BASE_CLKIN 0x0 #define APIC_TIMER_BASE_TMBASE 0x1 #define APIC_TIMER_BASE_DIV 0x2 -#define APIC_LVT_TIMER_PERIODIC (1<<17) -#define APIC_LVT_MASKED (1<<16) -#define APIC_LVT_LEVEL_TRIGGER (1<<15) -#define APIC_LVT_REMOTE_IRR (1<<14) -#define APIC_INPUT_POLARITY (1<<13) -#define APIC_SEND_PENDING (1<<12) +#define APIC_LVT_TIMER_PERIODIC (1 << 17) +#define APIC_LVT_MASKED (1 << 16) +#define APIC_LVT_LEVEL_TRIGGER (1 << 15) +#define APIC_LVT_REMOTE_IRR (1 << 14) +#define APIC_INPUT_POLARITY (1 << 13) +#define APIC_SEND_PENDING (1 << 12) #define APIC_MODE_MASK 0x700 -#define GET_APIC_DELIVERY_MODE(x) (((x)>>8)&0x7) -#define SET_APIC_DELIVERY_MODE(x, y) (((x)&~0x700)|((y)<<8)) +#define GET_APIC_DELIVERY_MODE(x) (((x) >> 8) & 0x7) +#define SET_APIC_DELIVERY_MODE(x, y) (((x) & ~0x700) | ((y) << 8)) #define APIC_MODE_FIXED 0x0 #define APIC_MODE_NMI 0x4 #define APIC_MODE_EXTINT 0x7 @@ -107,7 +105,7 @@ #define APIC_TMICT 0x380 #define APIC_TMCCT 0x390 #define APIC_TDCR 0x3E0 -#define APIC_TDR_DIV_TMBASE (1<<2) +#define APIC_TDR_DIV_TMBASE (1 << 2) #define APIC_TDR_DIV_1 0xB #define APIC_TDR_DIV_2 0x0 #define APIC_TDR_DIV_4 0x1 @@ -117,14 +115,14 @@ #define APIC_TDR_DIV_64 0x9 #define APIC_TDR_DIV_128 0xA #define APIC_EILVT0 0x500 -#define APIC_EILVT_NR_AMD_K8 1 /* Number of extended interrupts */ +#define APIC_EILVT_NR_AMD_K8 1 /* # of extended interrupts */ #define APIC_EILVT_NR_AMD_10H 4 -#define APIC_EILVT_LVTOFF(x) (((x)>>4)&0xF) +#define APIC_EILVT_LVTOFF(x) (((x) >> 4) & 0xF) #define APIC_EILVT_MSG_FIX 0x0 #define APIC_EILVT_MSG_SMI 0x2 #define APIC_EILVT_MSG_NMI 0x4 #define APIC_EILVT_MSG_EXT 0x7 -#define APIC_EILVT_MASKED (1<<16) +#define APIC_EILVT_MASKED (1 << 16) #define APIC_EILVT1 0x510 #define APIC_EILVT2 0x520 #define APIC_EILVT3 0x530 @@ -135,7 +133,7 @@ # define MAX_IO_APICS 64 #else # define MAX_IO_APICS 128 -# define MAX_LOCAL_APIC 256 +# define MAX_LOCAL_APIC 32768 #endif /* @@ -408,6 +406,9 @@ struct local_apic { #undef u32 -#define BAD_APICID 0xFFu - +#ifdef CONFIG_X86_32 + #define BAD_APICID 0xFFu +#else + #define BAD_APICID 0xFFFFu +#endif #endif diff --git a/include/asm-x86/atomic_32.h b/include/asm-x86/atomic_32.h index 437aac80171..21a4825148c 100644 --- a/include/asm-x86/atomic_32.h +++ b/include/asm-x86/atomic_32.h @@ -15,138 +15,133 @@ * on us. We need to use _exactly_ the address the user gave us, * not some alias that contains the same information. */ -typedef struct { int counter; } atomic_t; +typedef struct { + int counter; +} atomic_t; #define ATOMIC_INIT(i) { (i) } /** * atomic_read - read atomic variable * @v: pointer of type atomic_t - * + * * Atomically reads the value of @v. - */ + */ #define atomic_read(v) ((v)->counter) /** * atomic_set - set atomic variable * @v: pointer of type atomic_t * @i: required value - * + * * Atomically sets the value of @v to @i. - */ -#define atomic_set(v,i) (((v)->counter) = (i)) + */ +#define atomic_set(v, i) (((v)->counter) = (i)) /** * atomic_add - add integer to atomic variable * @i: integer value to add * @v: pointer of type atomic_t - * + * * Atomically adds @i to @v. */ -static __inline__ void atomic_add(int i, atomic_t *v) +static inline void atomic_add(int i, atomic_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "addl %1,%0" - :"+m" (v->counter) - :"ir" (i)); + asm volatile(LOCK_PREFIX "addl %1,%0" + : "+m" (v->counter) + : "ir" (i)); } /** * atomic_sub - subtract integer from atomic variable * @i: integer value to subtract * @v: pointer of type atomic_t - * + * * Atomically subtracts @i from @v. */ -static __inline__ void atomic_sub(int i, atomic_t *v) +static inline void atomic_sub(int i, atomic_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "subl %1,%0" - :"+m" (v->counter) - :"ir" (i)); + asm volatile(LOCK_PREFIX "subl %1,%0" + : "+m" (v->counter) + : "ir" (i)); } /** * atomic_sub_and_test - subtract value from variable and test result * @i: integer value to subtract * @v: pointer of type atomic_t - * + * * Atomically subtracts @i from @v and returns * true if the result is zero, or false for all * other cases. */ -static __inline__ int atomic_sub_and_test(int i, atomic_t *v) +static inline int atomic_sub_and_test(int i, atomic_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "subl %2,%0; sete %1" - :"+m" (v->counter), "=qm" (c) - :"ir" (i) : "memory"); + asm volatile(LOCK_PREFIX "subl %2,%0; sete %1" + : "+m" (v->counter), "=qm" (c) + : "ir" (i) : "memory"); return c; } /** * atomic_inc - increment atomic variable * @v: pointer of type atomic_t - * + * * Atomically increments @v by 1. - */ -static __inline__ void atomic_inc(atomic_t *v) + */ +static inline void atomic_inc(atomic_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "incl %0" - :"+m" (v->counter)); + asm volatile(LOCK_PREFIX "incl %0" + : "+m" (v->counter)); } /** * atomic_dec - decrement atomic variable * @v: pointer of type atomic_t - * + * * Atomically decrements @v by 1. - */ -static __inline__ void atomic_dec(atomic_t *v) + */ +static inline void atomic_dec(atomic_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "decl %0" - :"+m" (v->counter)); + asm volatile(LOCK_PREFIX "decl %0" + : "+m" (v->counter)); } /** * atomic_dec_and_test - decrement and test * @v: pointer of type atomic_t - * + * * Atomically decrements @v by 1 and * returns true if the result is 0, or false for all other * cases. - */ -static __inline__ int atomic_dec_and_test(atomic_t *v) + */ +static inline int atomic_dec_and_test(atomic_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "decl %0; sete %1" - :"+m" (v->counter), "=qm" (c) - : : "memory"); + asm volatile(LOCK_PREFIX "decl %0; sete %1" + : "+m" (v->counter), "=qm" (c) + : : "memory"); return c != 0; } /** - * atomic_inc_and_test - increment and test + * atomic_inc_and_test - increment and test * @v: pointer of type atomic_t - * + * * Atomically increments @v by 1 * and returns true if the result is zero, or false for all * other cases. - */ -static __inline__ int atomic_inc_and_test(atomic_t *v) + */ +static inline int atomic_inc_and_test(atomic_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "incl %0; sete %1" - :"+m" (v->counter), "=qm" (c) - : : "memory"); + asm volatile(LOCK_PREFIX "incl %0; sete %1" + : "+m" (v->counter), "=qm" (c) + : : "memory"); return c != 0; } @@ -154,19 +149,18 @@ static __inline__ int atomic_inc_and_test(atomic_t *v) * atomic_add_negative - add and test if negative * @v: pointer of type atomic_t * @i: integer value to add - * + * * Atomically adds @i to @v and returns true * if the result is negative, or false when * result is greater than or equal to zero. - */ -static __inline__ int atomic_add_negative(int i, atomic_t *v) + */ +static inline int atomic_add_negative(int i, atomic_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "addl %2,%0; sets %1" - :"+m" (v->counter), "=qm" (c) - :"ir" (i) : "memory"); + asm volatile(LOCK_PREFIX "addl %2,%0; sets %1" + : "+m" (v->counter), "=qm" (c) + : "ir" (i) : "memory"); return c; } @@ -177,20 +171,19 @@ static __inline__ int atomic_add_negative(int i, atomic_t *v) * * Atomically adds @i to @v and returns @i + @v */ -static __inline__ int atomic_add_return(int i, atomic_t *v) +static inline int atomic_add_return(int i, atomic_t *v) { int __i; #ifdef CONFIG_M386 unsigned long flags; - if(unlikely(boot_cpu_data.x86 <= 3)) + if (unlikely(boot_cpu_data.x86 <= 3)) goto no_xadd; #endif /* Modern 486+ processor */ __i = i; - __asm__ __volatile__( - LOCK_PREFIX "xaddl %0, %1" - :"+r" (i), "+m" (v->counter) - : : "memory"); + asm volatile(LOCK_PREFIX "xaddl %0, %1" + : "+r" (i), "+m" (v->counter) + : : "memory"); return i + __i; #ifdef CONFIG_M386 @@ -210,9 +203,9 @@ no_xadd: /* Legacy 386 processor */ * * Atomically subtracts @i from @v and returns @v - @i */ -static __inline__ int atomic_sub_return(int i, atomic_t *v) +static inline int atomic_sub_return(int i, atomic_t *v) { - return atomic_add_return(-i,v); + return atomic_add_return(-i, v); } #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new))) @@ -227,7 +220,7 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v) * Atomically adds @a to @v, so long as @v was not already @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +static inline int atomic_add_unless(atomic_t *v, int a, int u) { int c, old; c = atomic_read(v); @@ -244,17 +237,17 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) -#define atomic_inc_return(v) (atomic_add_return(1,v)) -#define atomic_dec_return(v) (atomic_sub_return(1,v)) +#define atomic_inc_return(v) (atomic_add_return(1, v)) +#define atomic_dec_return(v) (atomic_sub_return(1, v)) /* These are x86-specific, used by some header files */ -#define atomic_clear_mask(mask, addr) \ -__asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \ -: : "r" (~(mask)),"m" (*addr) : "memory") +#define atomic_clear_mask(mask, addr) \ + asm volatile(LOCK_PREFIX "andl %0,%1" \ + : : "r" (~(mask)), "m" (*(addr)) : "memory") -#define atomic_set_mask(mask, addr) \ -__asm__ __volatile__(LOCK_PREFIX "orl %0,%1" \ -: : "r" (mask),"m" (*(addr)) : "memory") +#define atomic_set_mask(mask, addr) \ + asm volatile(LOCK_PREFIX "orl %0,%1" \ + : : "r" (mask), "m" (*(addr)) : "memory") /* Atomic operations are already serializing on x86 */ #define smp_mb__before_atomic_dec() barrier() diff --git a/include/asm-x86/atomic_64.h b/include/asm-x86/atomic_64.h index 2d20a7a19f6..3e0cd7d3833 100644 --- a/include/asm-x86/atomic_64.h +++ b/include/asm-x86/atomic_64.h @@ -22,140 +22,135 @@ * on us. We need to use _exactly_ the address the user gave us, * not some alias that contains the same information. */ -typedef struct { int counter; } atomic_t; +typedef struct { + int counter; +} atomic_t; #define ATOMIC_INIT(i) { (i) } /** * atomic_read - read atomic variable * @v: pointer of type atomic_t - * + * * Atomically reads the value of @v. - */ + */ #define atomic_read(v) ((v)->counter) /** * atomic_set - set atomic variable * @v: pointer of type atomic_t * @i: required value - * + * * Atomically sets the value of @v to @i. - */ -#define atomic_set(v,i) (((v)->counter) = (i)) + */ +#define atomic_set(v, i) (((v)->counter) = (i)) /** * atomic_add - add integer to atomic variable * @i: integer value to add * @v: pointer of type atomic_t - * + * * Atomically adds @i to @v. */ -static __inline__ void atomic_add(int i, atomic_t *v) +static inline void atomic_add(int i, atomic_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "addl %1,%0" - :"=m" (v->counter) - :"ir" (i), "m" (v->counter)); + asm volatile(LOCK_PREFIX "addl %1,%0" + : "=m" (v->counter) + : "ir" (i), "m" (v->counter)); } /** * atomic_sub - subtract the atomic variable * @i: integer value to subtract * @v: pointer of type atomic_t - * + * * Atomically subtracts @i from @v. */ -static __inline__ void atomic_sub(int i, atomic_t *v) +static inline void atomic_sub(int i, atomic_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "subl %1,%0" - :"=m" (v->counter) - :"ir" (i), "m" (v->counter)); + asm volatile(LOCK_PREFIX "subl %1,%0" + : "=m" (v->counter) + : "ir" (i), "m" (v->counter)); } /** * atomic_sub_and_test - subtract value from variable and test result * @i: integer value to subtract * @v: pointer of type atomic_t - * + * * Atomically subtracts @i from @v and returns * true if the result is zero, or false for all * other cases. */ -static __inline__ int atomic_sub_and_test(int i, atomic_t *v) +static inline int atomic_sub_and_test(int i, atomic_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "subl %2,%0; sete %1" - :"=m" (v->counter), "=qm" (c) - :"ir" (i), "m" (v->counter) : "memory"); + asm volatile(LOCK_PREFIX "subl %2,%0; sete %1" + : "=m" (v->counter), "=qm" (c) + : "ir" (i), "m" (v->counter) : "memory"); return c; } /** * atomic_inc - increment atomic variable * @v: pointer of type atomic_t - * + * * Atomically increments @v by 1. - */ -static __inline__ void atomic_inc(atomic_t *v) + */ +static inline void atomic_inc(atomic_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "incl %0" - :"=m" (v->counter) - :"m" (v->counter)); + asm volatile(LOCK_PREFIX "incl %0" + : "=m" (v->counter) + : "m" (v->counter)); } /** * atomic_dec - decrement atomic variable * @v: pointer of type atomic_t - * + * * Atomically decrements @v by 1. - */ -static __inline__ void atomic_dec(atomic_t *v) + */ +static inline void atomic_dec(atomic_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "decl %0" - :"=m" (v->counter) - :"m" (v->counter)); + asm volatile(LOCK_PREFIX "decl %0" + : "=m" (v->counter) + : "m" (v->counter)); } /** * atomic_dec_and_test - decrement and test * @v: pointer of type atomic_t - * + * * Atomically decrements @v by 1 and * returns true if the result is 0, or false for all other * cases. - */ -static __inline__ int atomic_dec_and_test(atomic_t *v) + */ +static inline int atomic_dec_and_test(atomic_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "decl %0; sete %1" - :"=m" (v->counter), "=qm" (c) - :"m" (v->counter) : "memory"); + asm volatile(LOCK_PREFIX "decl %0; sete %1" + : "=m" (v->counter), "=qm" (c) + : "m" (v->counter) : "memory"); return c != 0; } /** - * atomic_inc_and_test - increment and test + * atomic_inc_and_test - increment and test * @v: pointer of type atomic_t - * + * * Atomically increments @v by 1 * and returns true if the result is zero, or false for all * other cases. - */ -static __inline__ int atomic_inc_and_test(atomic_t *v) + */ +static inline int atomic_inc_and_test(atomic_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "incl %0; sete %1" - :"=m" (v->counter), "=qm" (c) - :"m" (v->counter) : "memory"); + asm volatile(LOCK_PREFIX "incl %0; sete %1" + : "=m" (v->counter), "=qm" (c) + : "m" (v->counter) : "memory"); return c != 0; } @@ -163,19 +158,18 @@ static __inline__ int atomic_inc_and_test(atomic_t *v) * atomic_add_negative - add and test if negative * @i: integer value to add * @v: pointer of type atomic_t - * + * * Atomically adds @i to @v and returns true * if the result is negative, or false when * result is greater than or equal to zero. - */ -static __inline__ int atomic_add_negative(int i, atomic_t *v) + */ +static inline int atomic_add_negative(int i, atomic_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "addl %2,%0; sets %1" - :"=m" (v->counter), "=qm" (c) - :"ir" (i), "m" (v->counter) : "memory"); + asm volatile(LOCK_PREFIX "addl %2,%0; sets %1" + : "=m" (v->counter), "=qm" (c) + : "ir" (i), "m" (v->counter) : "memory"); return c; } @@ -186,27 +180,28 @@ static __inline__ int atomic_add_negative(int i, atomic_t *v) * * Atomically adds @i to @v and returns @i + @v */ -static __inline__ int atomic_add_return(int i, atomic_t *v) +static inline int atomic_add_return(int i, atomic_t *v) { int __i = i; - __asm__ __volatile__( - LOCK_PREFIX "xaddl %0, %1" - :"+r" (i), "+m" (v->counter) - : : "memory"); + asm volatile(LOCK_PREFIX "xaddl %0, %1" + : "+r" (i), "+m" (v->counter) + : : "memory"); return i + __i; } -static __inline__ int atomic_sub_return(int i, atomic_t *v) +static inline int atomic_sub_return(int i, atomic_t *v) { - return atomic_add_return(-i,v); + return atomic_add_return(-i, v); } -#define atomic_inc_return(v) (atomic_add_return(1,v)) -#define atomic_dec_return(v) (atomic_sub_return(1,v)) +#define atomic_inc_return(v) (atomic_add_return(1, v)) +#define atomic_dec_return(v) (atomic_sub_return(1, v)) /* An 64bit atomic type */ -typedef struct { long counter; } atomic64_t; +typedef struct { + long counter; +} atomic64_t; #define ATOMIC64_INIT(i) { (i) } @@ -226,7 +221,7 @@ typedef struct { long counter; } atomic64_t; * * Atomically sets the value of @v to @i. */ -#define atomic64_set(v,i) (((v)->counter) = (i)) +#define atomic64_set(v, i) (((v)->counter) = (i)) /** * atomic64_add - add integer to atomic64 variable @@ -235,12 +230,11 @@ typedef struct { long counter; } atomic64_t; * * Atomically adds @i to @v. */ -static __inline__ void atomic64_add(long i, atomic64_t *v) +static inline void atomic64_add(long i, atomic64_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "addq %1,%0" - :"=m" (v->counter) - :"ir" (i), "m" (v->counter)); + asm volatile(LOCK_PREFIX "addq %1,%0" + : "=m" (v->counter) + : "ir" (i), "m" (v->counter)); } /** @@ -250,12 +244,11 @@ static __inline__ void atomic64_add(long i, atomic64_t *v) * * Atomically subtracts @i from @v. */ -static __inline__ void atomic64_sub(long i, atomic64_t *v) +static inline void atomic64_sub(long i, atomic64_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "subq %1,%0" - :"=m" (v->counter) - :"ir" (i), "m" (v->counter)); + asm volatile(LOCK_PREFIX "subq %1,%0" + : "=m" (v->counter) + : "ir" (i), "m" (v->counter)); } /** @@ -267,14 +260,13 @@ static __inline__ void atomic64_sub(long i, atomic64_t *v) * true if the result is zero, or false for all * other cases. */ -static __inline__ int atomic64_sub_and_test(long i, atomic64_t *v) +static inline int atomic64_sub_and_test(long i, atomic64_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "subq %2,%0; sete %1" - :"=m" (v->counter), "=qm" (c) - :"ir" (i), "m" (v->counter) : "memory"); + asm volatile(LOCK_PREFIX "subq %2,%0; sete %1" + : "=m" (v->counter), "=qm" (c) + : "ir" (i), "m" (v->counter) : "memory"); return c; } @@ -284,12 +276,11 @@ static __inline__ int atomic64_sub_and_test(long i, atomic64_t *v) * * Atomically increments @v by 1. */ -static __inline__ void atomic64_inc(atomic64_t *v) +static inline void atomic64_inc(atomic64_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "incq %0" - :"=m" (v->counter) - :"m" (v->counter)); + asm volatile(LOCK_PREFIX "incq %0" + : "=m" (v->counter) + : "m" (v->counter)); } /** @@ -298,12 +289,11 @@ static __inline__ void atomic64_inc(atomic64_t *v) * * Atomically decrements @v by 1. */ -static __inline__ void atomic64_dec(atomic64_t *v) +static inline void atomic64_dec(atomic64_t *v) { - __asm__ __volatile__( - LOCK_PREFIX "decq %0" - :"=m" (v->counter) - :"m" (v->counter)); + asm volatile(LOCK_PREFIX "decq %0" + : "=m" (v->counter) + : "m" (v->counter)); } /** @@ -314,14 +304,13 @@ static __inline__ void atomic64_dec(atomic64_t *v) * returns true if the result is 0, or false for all other * cases. */ -static __inline__ int atomic64_dec_and_test(atomic64_t *v) +static inline int atomic64_dec_and_test(atomic64_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "decq %0; sete %1" - :"=m" (v->counter), "=qm" (c) - :"m" (v->counter) : "memory"); + asm volatile(LOCK_PREFIX "decq %0; sete %1" + : "=m" (v->counter), "=qm" (c) + : "m" (v->counter) : "memory"); return c != 0; } @@ -333,14 +322,13 @@ static __inline__ int atomic64_dec_and_test(atomic64_t *v) * and returns true if the result is zero, or false for all * other cases. */ -static __inline__ int atomic64_inc_and_test(atomic64_t *v) +static inline int atomic64_inc_and_test(atomic64_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "incq %0; sete %1" - :"=m" (v->counter), "=qm" (c) - :"m" (v->counter) : "memory"); + asm volatile(LOCK_PREFIX "incq %0; sete %1" + : "=m" (v->counter), "=qm" (c) + : "m" (v->counter) : "memory"); return c != 0; } @@ -353,14 +341,13 @@ static __inline__ int atomic64_inc_and_test(atomic64_t *v) * if the result is negative, or false when * result is greater than or equal to zero. */ -static __inline__ int atomic64_add_negative(long i, atomic64_t *v) +static inline int atomic64_add_negative(long i, atomic64_t *v) { unsigned char c; - __asm__ __volatile__( - LOCK_PREFIX "addq %2,%0; sets %1" - :"=m" (v->counter), "=qm" (c) - :"ir" (i), "m" (v->counter) : "memory"); + asm volatile(LOCK_PREFIX "addq %2,%0; sets %1" + : "=m" (v->counter), "=qm" (c) + : "ir" (i), "m" (v->counter) : "memory"); return c; } @@ -371,29 +358,28 @@ static __inline__ int atomic64_add_negative(long i, atomic64_t *v) * * Atomically adds @i to @v and returns @i + @v */ -static __inline__ long atomic64_add_return(long i, atomic64_t *v) +static inline long atomic64_add_return(long i, atomic64_t *v) { long __i = i; - __asm__ __volatile__( - LOCK_PREFIX "xaddq %0, %1;" - :"+r" (i), "+m" (v->counter) - : : "memory"); + asm volatile(LOCK_PREFIX "xaddq %0, %1;" + : "+r" (i), "+m" (v->counter) + : : "memory"); return i + __i; } -static __inline__ long atomic64_sub_return(long i, atomic64_t *v) +static inline long atomic64_sub_return(long i, atomic64_t *v) { - return atomic64_add_return(-i,v); + return atomic64_add_return(-i, v); } -#define atomic64_inc_return(v) (atomic64_add_return(1,v)) -#define atomic64_dec_return(v) (atomic64_sub_return(1,v)) +#define atomic64_inc_return(v) (atomic64_add_return(1, (v))) +#define atomic64_dec_return(v) (atomic64_sub_return(1, (v))) -#define atomic64_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) +#define atomic64_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new))) #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new))) +#define atomic_xchg(v, new) (xchg(&((v)->counter), (new))) /** * atomic_add_unless - add unless the number is a given value @@ -404,7 +390,7 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) +static inline int atomic_add_unless(atomic_t *v, int a, int u) { int c, old; c = atomic_read(v); @@ -430,7 +416,7 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) * Atomically adds @a to @v, so long as it was not @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +static inline int atomic64_add_unless(atomic64_t *v, long a, long u) { long c, old; c = atomic64_read(v); @@ -448,13 +434,14 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) /* These are x86-specific, used by some header files */ -#define atomic_clear_mask(mask, addr) \ -__asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \ -: : "r" (~(mask)),"m" (*addr) : "memory") - -#define atomic_set_mask(mask, addr) \ -__asm__ __volatile__(LOCK_PREFIX "orl %0,%1" \ -: : "r" ((unsigned)mask),"m" (*(addr)) : "memory") +#define atomic_clear_mask(mask, addr) \ + asm volatile(LOCK_PREFIX "andl %0,%1" \ + : : "r" (~(mask)), "m" (*(addr)) : "memory") + +#define atomic_set_mask(mask, addr) \ + asm volatile(LOCK_PREFIX "orl %0,%1" \ + : : "r" ((unsigned)(mask)), "m" (*(addr)) \ + : "memory") /* Atomic operations are already serializing on x86 */ #define smp_mb__before_atomic_dec() barrier() diff --git a/include/asm-x86/mach-default/bios_ebda.h b/include/asm-x86/bios_ebda.h index 9cbd9a668af..b4a46b7be79 100644 --- a/include/asm-x86/mach-default/bios_ebda.h +++ b/include/asm-x86/bios_ebda.h @@ -1,6 +1,8 @@ #ifndef _MACH_BIOS_EBDA_H #define _MACH_BIOS_EBDA_H +#include <asm/io.h> + /* * there is a real-mode segmented pointer pointing to the * 4K EBDA area at 0x40E. diff --git a/include/asm-x86/bitops.h b/include/asm-x86/bitops.h index 1a23ce1a569..b81a4d4d333 100644 --- a/include/asm-x86/bitops.h +++ b/include/asm-x86/bitops.h @@ -23,10 +23,13 @@ #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1) /* Technically wrong, but this avoids compilation errors on some gcc versions. */ -#define ADDR "=m" (*(volatile long *) addr) +#define ADDR "=m" (*(volatile long *)addr) +#define BIT_ADDR "=m" (((volatile int *)addr)[nr >> 5]) #else #define ADDR "+m" (*(volatile long *) addr) +#define BIT_ADDR "+m" (((volatile int *)addr)[nr >> 5]) #endif +#define BASE_ADDR "m" (*(volatile int *)addr) /** * set_bit - Atomically set a bit in memory @@ -45,9 +48,7 @@ */ static inline void set_bit(int nr, volatile void *addr) { - asm volatile(LOCK_PREFIX "bts %1,%0" - : ADDR - : "Ir" (nr) : "memory"); + asm volatile(LOCK_PREFIX "bts %1,%0" : ADDR : "Ir" (nr) : "memory"); } /** @@ -61,12 +62,9 @@ static inline void set_bit(int nr, volatile void *addr) */ static inline void __set_bit(int nr, volatile void *addr) { - asm volatile("bts %1,%0" - : ADDR - : "Ir" (nr) : "memory"); + asm volatile("bts %1,%0" : ADDR : "Ir" (nr) : "memory"); } - /** * clear_bit - Clears a bit in memory * @nr: Bit to clear @@ -79,9 +77,7 @@ static inline void __set_bit(int nr, volatile void *addr) */ static inline void clear_bit(int nr, volatile void *addr) { - asm volatile(LOCK_PREFIX "btr %1,%0" - : ADDR - : "Ir" (nr)); + asm volatile(LOCK_PREFIX "btr %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR); } /* @@ -100,7 +96,7 @@ static inline void clear_bit_unlock(unsigned nr, volatile void *addr) static inline void __clear_bit(int nr, volatile void *addr) { - asm volatile("btr %1,%0" : ADDR : "Ir" (nr)); + asm volatile("btr %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR); } /* @@ -135,7 +131,7 @@ static inline void __clear_bit_unlock(unsigned nr, volatile void *addr) */ static inline void __change_bit(int nr, volatile void *addr) { - asm volatile("btc %1,%0" : ADDR : "Ir" (nr)); + asm volatile("btc %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR); } /** @@ -149,8 +145,7 @@ static inline void __change_bit(int nr, volatile void *addr) */ static inline void change_bit(int nr, volatile void *addr) { - asm volatile(LOCK_PREFIX "btc %1,%0" - : ADDR : "Ir" (nr)); + asm volatile(LOCK_PREFIX "btc %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR); } /** @@ -166,9 +161,7 @@ static inline int test_and_set_bit(int nr, volatile void *addr) int oldbit; asm volatile(LOCK_PREFIX "bts %2,%1\n\t" - "sbb %0,%0" - : "=r" (oldbit), ADDR - : "Ir" (nr) : "memory"); + "sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory"); return oldbit; } @@ -198,10 +191,9 @@ static inline int __test_and_set_bit(int nr, volatile void *addr) { int oldbit; - asm("bts %2,%1\n\t" - "sbb %0,%0" - : "=r" (oldbit), ADDR - : "Ir" (nr)); + asm volatile("bts %2,%3\n\t" + "sbb %0,%0" + : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR); return oldbit; } @@ -219,8 +211,7 @@ static inline int test_and_clear_bit(int nr, volatile void *addr) asm volatile(LOCK_PREFIX "btr %2,%1\n\t" "sbb %0,%0" - : "=r" (oldbit), ADDR - : "Ir" (nr) : "memory"); + : "=r" (oldbit), ADDR : "Ir" (nr) : "memory"); return oldbit; } @@ -238,10 +229,9 @@ static inline int __test_and_clear_bit(int nr, volatile void *addr) { int oldbit; - asm volatile("btr %2,%1\n\t" + asm volatile("btr %2,%3\n\t" "sbb %0,%0" - : "=r" (oldbit), ADDR - : "Ir" (nr)); + : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR); return oldbit; } @@ -250,10 +240,9 @@ static inline int __test_and_change_bit(int nr, volatile void *addr) { int oldbit; - asm volatile("btc %2,%1\n\t" + asm volatile("btc %2,%3\n\t" "sbb %0,%0" - : "=r" (oldbit), ADDR - : "Ir" (nr) : "memory"); + : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR); return oldbit; } @@ -272,8 +261,7 @@ static inline int test_and_change_bit(int nr, volatile void *addr) asm volatile(LOCK_PREFIX "btc %2,%1\n\t" "sbb %0,%0" - : "=r" (oldbit), ADDR - : "Ir" (nr) : "memory"); + : "=r" (oldbit), ADDR : "Ir" (nr) : "memory"); return oldbit; } @@ -288,10 +276,11 @@ static inline int variable_test_bit(int nr, volatile const void *addr) { int oldbit; - asm volatile("bt %2,%1\n\t" + asm volatile("bt %2,%3\n\t" "sbb %0,%0" : "=r" (oldbit) - : "m" (*(unsigned long *)addr), "Ir" (nr)); + : "m" (((volatile const int *)addr)[nr >> 5]), + "Ir" (nr), BASE_ADDR); return oldbit; } @@ -305,17 +294,145 @@ static inline int variable_test_bit(int nr, volatile const void *addr) static int test_bit(int nr, const volatile unsigned long *addr); #endif -#define test_bit(nr,addr) \ - (__builtin_constant_p(nr) ? \ - constant_test_bit((nr),(addr)) : \ - variable_test_bit((nr),(addr))) +#define test_bit(nr, addr) \ + (__builtin_constant_p((nr)) \ + ? constant_test_bit((nr), (addr)) \ + : variable_test_bit((nr), (addr))) -#undef ADDR +/** + * __ffs - find first set bit in word + * @word: The word to search + * + * Undefined if no bit exists, so code should check against 0 first. + */ +static inline unsigned long __ffs(unsigned long word) +{ + asm("bsf %1,%0" + : "=r" (word) + : "rm" (word)); + return word; +} + +/** + * ffz - find first zero bit in word + * @word: The word to search + * + * Undefined if no zero exists, so code should check against ~0UL first. + */ +static inline unsigned long ffz(unsigned long word) +{ + asm("bsf %1,%0" + : "=r" (word) + : "r" (~word)); + return word; +} -#ifdef CONFIG_X86_32 -# include "bitops_32.h" +/* + * __fls: find last set bit in word + * @word: The word to search + * + * Undefined if no zero exists, so code should check against ~0UL first. + */ +static inline unsigned long __fls(unsigned long word) +{ + asm("bsr %1,%0" + : "=r" (word) + : "rm" (word)); + return word; +} + +#ifdef __KERNEL__ +/** + * ffs - find first set bit in word + * @x: the word to search + * + * This is defined the same way as the libc and compiler builtin ffs + * routines, therefore differs in spirit from the other bitops. + * + * ffs(value) returns 0 if value is 0 or the position of the first + * set bit if value is nonzero. The first (least significant) bit + * is at position 1. + */ +static inline int ffs(int x) +{ + int r; +#ifdef CONFIG_X86_CMOV + asm("bsfl %1,%0\n\t" + "cmovzl %2,%0" + : "=r" (r) : "rm" (x), "r" (-1)); #else -# include "bitops_64.h" + asm("bsfl %1,%0\n\t" + "jnz 1f\n\t" + "movl $-1,%0\n" + "1:" : "=r" (r) : "rm" (x)); #endif + return r + 1; +} + +/** + * fls - find last set bit in word + * @x: the word to search + * + * This is defined in a similar way as the libc and compiler builtin + * ffs, but returns the position of the most significant set bit. + * + * fls(value) returns 0 if value is 0 or the position of the last + * set bit if value is nonzero. The last (most significant) bit is + * at position 32. + */ +static inline int fls(int x) +{ + int r; +#ifdef CONFIG_X86_CMOV + asm("bsrl %1,%0\n\t" + "cmovzl %2,%0" + : "=&r" (r) : "rm" (x), "rm" (-1)); +#else + asm("bsrl %1,%0\n\t" + "jnz 1f\n\t" + "movl $-1,%0\n" + "1:" : "=r" (r) : "rm" (x)); +#endif + return r + 1; +} +#endif /* __KERNEL__ */ + +#undef BASE_ADDR +#undef BIT_ADDR +#undef ADDR + +static inline void set_bit_string(unsigned long *bitmap, + unsigned long i, int len) +{ + unsigned long end = i + len; + while (i < end) { + __set_bit(i, bitmap); + i++; + } +} + +#ifdef __KERNEL__ + +#include <asm-generic/bitops/sched.h> + +#define ARCH_HAS_FAST_MULTIPLIER 1 + +#include <asm-generic/bitops/hweight.h> + +#endif /* __KERNEL__ */ + +#include <asm-generic/bitops/fls64.h> + +#ifdef __KERNEL__ + +#include <asm-generic/bitops/ext2-non-atomic.h> + +#define ext2_set_bit_atomic(lock, nr, addr) \ + test_and_set_bit((nr), (unsigned long *)(addr)) +#define ext2_clear_bit_atomic(lock, nr, addr) \ + test_and_clear_bit((nr), (unsigned long *)(addr)) + +#include <asm-generic/bitops/minix.h> +#endif /* __KERNEL__ */ #endif /* _ASM_X86_BITOPS_H */ diff --git a/include/asm-x86/bitops_32.h b/include/asm-x86/bitops_32.h deleted file mode 100644 index e4d75fcf9c0..00000000000 --- a/include/asm-x86/bitops_32.h +++ /dev/null @@ -1,164 +0,0 @@ -#ifndef _I386_BITOPS_H -#define _I386_BITOPS_H - -/* - * Copyright 1992, Linus Torvalds. - */ - -/** - * find_first_zero_bit - find the first zero bit in a memory region - * @addr: The address to start the search at - * @size: The maximum size to search - * - * Returns the bit number of the first zero bit, not the number of the byte - * containing a bit. - */ -static inline int find_first_zero_bit(const unsigned long *addr, unsigned size) -{ - int d0, d1, d2; - int res; - - if (!size) - return 0; - /* This looks at memory. Mark it volatile to tell gcc not to move it around */ - __asm__ __volatile__( - "movl $-1,%%eax\n\t" - "xorl %%edx,%%edx\n\t" - "repe; scasl\n\t" - "je 1f\n\t" - "xorl -4(%%edi),%%eax\n\t" - "subl $4,%%edi\n\t" - "bsfl %%eax,%%edx\n" - "1:\tsubl %%ebx,%%edi\n\t" - "shll $3,%%edi\n\t" - "addl %%edi,%%edx" - :"=d" (res), "=&c" (d0), "=&D" (d1), "=&a" (d2) - :"1" ((size + 31) >> 5), "2" (addr), "b" (addr) : "memory"); - return res; -} - -/** - * find_next_zero_bit - find the first zero bit in a memory region - * @addr: The address to base the search on - * @offset: The bit number to start searching at - * @size: The maximum size to search - */ -int find_next_zero_bit(const unsigned long *addr, int size, int offset); - -/** - * __ffs - find first bit in word. - * @word: The word to search - * - * Undefined if no bit exists, so code should check against 0 first. - */ -static inline unsigned long __ffs(unsigned long word) -{ - __asm__("bsfl %1,%0" - :"=r" (word) - :"rm" (word)); - return word; -} - -/** - * find_first_bit - find the first set bit in a memory region - * @addr: The address to start the search at - * @size: The maximum size to search - * - * Returns the bit number of the first set bit, not the number of the byte - * containing a bit. - */ -static inline unsigned find_first_bit(const unsigned long *addr, unsigned size) -{ - unsigned x = 0; - - while (x < size) { - unsigned long val = *addr++; - if (val) - return __ffs(val) + x; - x += (sizeof(*addr)<<3); - } - return x; -} - -/** - * find_next_bit - find the first set bit in a memory region - * @addr: The address to base the search on - * @offset: The bit number to start searching at - * @size: The maximum size to search - */ -int find_next_bit(const unsigned long *addr, int size, int offset); - -/** - * ffz - find first zero in word. - * @word: The word to search - * - * Undefined if no zero exists, so code should check against ~0UL first. - */ -static inline unsigned long ffz(unsigned long word) -{ - __asm__("bsfl %1,%0" - :"=r" (word) - :"r" (~word)); - return word; -} - -#ifdef __KERNEL__ - -#include <asm-generic/bitops/sched.h> - -/** - * ffs - find first bit set - * @x: the word to search - * - * This is defined the same way as - * the libc and compiler builtin ffs routines, therefore - * differs in spirit from the above ffz() (man ffs). - */ -static inline int ffs(int x) -{ - int r; - - __asm__("bsfl %1,%0\n\t" - "jnz 1f\n\t" - "movl $-1,%0\n" - "1:" : "=r" (r) : "rm" (x)); - return r+1; -} - -/** - * fls - find last bit set - * @x: the word to search - * - * This is defined the same way as ffs(). - */ -static inline int fls(int x) -{ - int r; - - __asm__("bsrl %1,%0\n\t" - "jnz 1f\n\t" - "movl $-1,%0\n" - "1:" : "=r" (r) : "rm" (x)); - return r+1; -} - -#include <asm-generic/bitops/hweight.h> - -#endif /* __KERNEL__ */ - -#include <asm-generic/bitops/fls64.h> - -#ifdef __KERNEL__ - -#include <asm-generic/bitops/ext2-non-atomic.h> - -#define ext2_set_bit_atomic(lock, nr, addr) \ - test_and_set_bit((nr), (unsigned long *)addr) -#define ext2_clear_bit_atomic(lock, nr, addr) \ - test_and_clear_bit((nr), (unsigned long *)addr) - -#include <asm-generic/bitops/minix.h> - -#endif /* __KERNEL__ */ - -#endif /* _I386_BITOPS_H */ diff --git a/include/asm-x86/bitops_64.h b/include/asm-x86/bitops_64.h deleted file mode 100644 index aaf15194d53..00000000000 --- a/include/asm-x86/bitops_64.h +++ /dev/null @@ -1,162 +0,0 @@ -#ifndef _X86_64_BITOPS_H -#define _X86_64_BITOPS_H - -/* - * Copyright 1992, Linus Torvalds. - */ - -extern long find_first_zero_bit(const unsigned long *addr, unsigned long size); -extern long find_next_zero_bit(const unsigned long *addr, long size, long offset); -extern long find_first_bit(const unsigned long *addr, unsigned long size); -extern long find_next_bit(const unsigned long *addr, long size, long offset); - -/* return index of first bet set in val or max when no bit is set */ -static inline long __scanbit(unsigned long val, unsigned long max) -{ - asm("bsfq %1,%0 ; cmovz %2,%0" : "=&r" (val) : "r" (val), "r" (max)); - return val; -} - -#define find_first_bit(addr,size) \ -((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \ - (__scanbit(*(unsigned long *)addr,(size))) : \ - find_first_bit(addr,size))) - -#define find_next_bit(addr,size,off) \ -((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \ - ((off) + (__scanbit((*(unsigned long *)addr) >> (off),(size)-(off)))) : \ - find_next_bit(addr,size,off))) - -#define find_first_zero_bit(addr,size) \ -((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \ - (__scanbit(~*(unsigned long *)addr,(size))) : \ - find_first_zero_bit(addr,size))) - -#define find_next_zero_bit(addr,size,off) \ -((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \ - ((off)+(__scanbit(~(((*(unsigned long *)addr)) >> (off)),(size)-(off)))) : \ - find_next_zero_bit(addr,size,off))) - -static inline void set_bit_string(unsigned long *bitmap, unsigned long i, - int len) -{ - unsigned long end = i + len; - while (i < end) { - __set_bit(i, bitmap); - i++; - } -} - -/** - * ffz - find first zero in word. - * @word: The word to search - * - * Undefined if no zero exists, so code should check against ~0UL first. - */ -static inline unsigned long ffz(unsigned long word) -{ - __asm__("bsfq %1,%0" - :"=r" (word) - :"r" (~word)); - return word; -} - -/** - * __ffs - find first bit in word. - * @word: The word to search - * - * Undefined if no bit exists, so code should check against 0 first. - */ -static inline unsigned long __ffs(unsigned long word) -{ - __asm__("bsfq %1,%0" - :"=r" (word) - :"rm" (word)); - return word; -} - -/* - * __fls: find last bit set. - * @word: The word to search - * - * Undefined if no zero exists, so code should check against ~0UL first. - */ -static inline unsigned long __fls(unsigned long word) -{ - __asm__("bsrq %1,%0" - :"=r" (word) - :"rm" (word)); - return word; -} - -#ifdef __KERNEL__ - -#include <asm-generic/bitops/sched.h> - -/** - * ffs - find first bit set - * @x: the word to search - * - * This is defined the same way as - * the libc and compiler builtin ffs routines, therefore - * differs in spirit from the above ffz (man ffs). - */ -static inline int ffs(int x) -{ - int r; - - __asm__("bsfl %1,%0\n\t" - "cmovzl %2,%0" - : "=r" (r) : "rm" (x), "r" (-1)); - return r+1; -} - -/** - * fls64 - find last bit set in 64 bit word - * @x: the word to search - * - * This is defined the same way as fls. - */ -static inline int fls64(__u64 x) -{ - if (x == 0) - return 0; - return __fls(x) + 1; -} - -/** - * fls - find last bit set - * @x: the word to search - * - * This is defined the same way as ffs. - */ -static inline int fls(int x) -{ - int r; - - __asm__("bsrl %1,%0\n\t" - "cmovzl %2,%0" - : "=&r" (r) : "rm" (x), "rm" (-1)); - return r+1; -} - -#define ARCH_HAS_FAST_MULTIPLIER 1 - -#include <asm-generic/bitops/hweight.h> - -#endif /* __KERNEL__ */ - -#ifdef __KERNEL__ - -#include <asm-generic/bitops/ext2-non-atomic.h> - -#define ext2_set_bit_atomic(lock,nr,addr) \ - test_and_set_bit((nr),(unsigned long*)addr) -#define ext2_clear_bit_atomic(lock,nr,addr) \ - test_and_clear_bit((nr),(unsigned long*)addr) - -#include <asm-generic/bitops/minix.h> - -#endif /* __KERNEL__ */ - -#endif /* _X86_64_BITOPS_H */ diff --git a/include/asm-x86/boot.h b/include/asm-x86/boot.h index ed8affbf96c..2faed7ecb09 100644 --- a/include/asm-x86/boot.h +++ b/include/asm-x86/boot.h @@ -17,4 +17,12 @@ + (CONFIG_PHYSICAL_ALIGN - 1)) \ & ~(CONFIG_PHYSICAL_ALIGN - 1)) +#ifdef CONFIG_X86_64 +#define BOOT_HEAP_SIZE 0x7000 +#define BOOT_STACK_SIZE 0x4000 +#else +#define BOOT_HEAP_SIZE 0x4000 +#define BOOT_STACK_SIZE 0x1000 +#endif + #endif /* _ASM_BOOT_H */ diff --git a/include/asm-x86/bootparam.h b/include/asm-x86/bootparam.h index 51151356840..e8659909e5f 100644 --- a/include/asm-x86/bootparam.h +++ b/include/asm-x86/bootparam.h @@ -9,6 +9,17 @@ #include <asm/ist.h> #include <video/edid.h> +/* setup data types */ +#define SETUP_NONE 0 + +/* extensible setup data list node */ +struct setup_data { + u64 next; + u32 type; + u32 len; + u8 data[0]; +}; + struct setup_header { __u8 setup_sects; __u16 root_flags; @@ -46,6 +57,9 @@ struct setup_header { __u32 cmdline_size; __u32 hardware_subarch; __u64 hardware_subarch_data; + __u32 payload_offset; + __u32 payload_length; + __u64 setup_data; } __attribute__((packed)); struct sys_desc_table { diff --git a/include/asm-x86/bug.h b/include/asm-x86/bug.h index 8d477a20139..b69aa64b82a 100644 --- a/include/asm-x86/bug.h +++ b/include/asm-x86/bug.h @@ -12,25 +12,25 @@ # define __BUG_C0 "2:\t.quad 1b, %c0\n" #endif -#define BUG() \ - do { \ - asm volatile("1:\tud2\n" \ - ".pushsection __bug_table,\"a\"\n" \ - __BUG_C0 \ - "\t.word %c1, 0\n" \ - "\t.org 2b+%c2\n" \ - ".popsection" \ - : : "i" (__FILE__), "i" (__LINE__), \ - "i" (sizeof(struct bug_entry))); \ - for(;;) ; \ - } while(0) +#define BUG() \ +do { \ + asm volatile("1:\tud2\n" \ + ".pushsection __bug_table,\"a\"\n" \ + __BUG_C0 \ + "\t.word %c1, 0\n" \ + "\t.org 2b+%c2\n" \ + ".popsection" \ + : : "i" (__FILE__), "i" (__LINE__), \ + "i" (sizeof(struct bug_entry))); \ + for (;;) ; \ +} while (0) #else -#define BUG() \ - do { \ - asm volatile("ud2"); \ - for(;;) ; \ - } while(0) +#define BUG() \ +do { \ + asm volatile("ud2"); \ + for (;;) ; \ +} while (0) #endif #endif /* !CONFIG_BUG */ diff --git a/include/asm-x86/byteorder.h b/include/asm-x86/byteorder.h index fe2f2e5d51b..e02ae2d89ac 100644 --- a/include/asm-x86/byteorder.h +++ b/include/asm-x86/byteorder.h @@ -8,50 +8,59 @@ #ifdef __i386__ -static __inline__ __attribute_const__ __u32 ___arch__swab32(__u32 x) +static inline __attribute_const__ __u32 ___arch__swab32(__u32 x) { #ifdef CONFIG_X86_BSWAP - __asm__("bswap %0" : "=r" (x) : "0" (x)); + asm("bswap %0" : "=r" (x) : "0" (x)); #else - __asm__("xchgb %b0,%h0\n\t" /* swap lower bytes */ - "rorl $16,%0\n\t" /* swap words */ - "xchgb %b0,%h0" /* swap higher bytes */ - :"=q" (x) - : "0" (x)); + asm("xchgb %b0,%h0\n\t" /* swap lower bytes */ + "rorl $16,%0\n\t" /* swap words */ + "xchgb %b0,%h0" /* swap higher bytes */ + : "=q" (x) + : "0" (x)); #endif return x; } -static __inline__ __attribute_const__ __u64 ___arch__swab64(__u64 val) +static inline __attribute_const__ __u64 ___arch__swab64(__u64 val) { union { - struct { __u32 a,b; } s; + struct { + __u32 a; + __u32 b; + } s; __u64 u; } v; v.u = val; #ifdef CONFIG_X86_BSWAP - __asm__("bswapl %0 ; bswapl %1 ; xchgl %0,%1" + asm("bswapl %0 ; bswapl %1 ; xchgl %0,%1" : "=r" (v.s.a), "=r" (v.s.b) : "0" (v.s.a), "1" (v.s.b)); #else v.s.a = ___arch__swab32(v.s.a); v.s.b = ___arch__swab32(v.s.b); - __asm__("xchgl %0,%1" : "=r" (v.s.a), "=r" (v.s.b) : "0" (v.s.a), "1" (v.s.b)); + asm("xchgl %0,%1" + : "=r" (v.s.a), "=r" (v.s.b) + : "0" (v.s.a), "1" (v.s.b)); #endif return v.u; } #else /* __i386__ */ -static __inline__ __attribute_const__ __u64 ___arch__swab64(__u64 x) +static inline __attribute_const__ __u64 ___arch__swab64(__u64 x) { - __asm__("bswapq %0" : "=r" (x) : "0" (x)); + asm("bswapq %0" + : "=r" (x) + : "0" (x)); return x; } -static __inline__ __attribute_const__ __u32 ___arch__swab32(__u32 x) +static inline __attribute_const__ __u32 ___arch__swab32(__u32 x) { - __asm__("bswapl %0" : "=r" (x) : "0" (x)); + asm("bswapl %0" + : "=r" (x) + : "0" (x)); return x; } diff --git a/include/asm-x86/cacheflush.h b/include/asm-x86/cacheflush.h index 5396c212d8c..f4c0ab50d2c 100644 --- a/include/asm-x86/cacheflush.h +++ b/include/asm-x86/cacheflush.h @@ -14,33 +14,85 @@ #define flush_dcache_mmap_lock(mapping) do { } while (0) #define flush_dcache_mmap_unlock(mapping) do { } while (0) #define flush_icache_range(start, end) do { } while (0) -#define flush_icache_page(vma,pg) do { } while (0) -#define flush_icache_user_range(vma,pg,adr,len) do { } while (0) +#define flush_icache_page(vma, pg) do { } while (0) +#define flush_icache_user_range(vma, pg, adr, len) do { } while (0) #define flush_cache_vmap(start, end) do { } while (0) #define flush_cache_vunmap(start, end) do { } while (0) -#define copy_to_user_page(vma, page, vaddr, dst, src, len) \ - memcpy(dst, src, len) -#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ - memcpy(dst, src, len) +#define copy_to_user_page(vma, page, vaddr, dst, src, len) \ + memcpy((dst), (src), (len)) +#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ + memcpy((dst), (src), (len)) -int __deprecated_for_modules change_page_attr(struct page *page, int numpages, - pgprot_t prot); -int set_pages_uc(struct page *page, int numpages); -int set_pages_wb(struct page *page, int numpages); -int set_pages_x(struct page *page, int numpages); -int set_pages_nx(struct page *page, int numpages); -int set_pages_ro(struct page *page, int numpages); -int set_pages_rw(struct page *page, int numpages); +/* + * The set_memory_* API can be used to change various attributes of a virtual + * address range. The attributes include: + * Cachability : UnCached, WriteCombining, WriteBack + * Executability : eXeutable, NoteXecutable + * Read/Write : ReadOnly, ReadWrite + * Presence : NotPresent + * + * Within a catagory, the attributes are mutually exclusive. + * + * The implementation of this API will take care of various aspects that + * are associated with changing such attributes, such as: + * - Flushing TLBs + * - Flushing CPU caches + * - Making sure aliases of the memory behind the mapping don't violate + * coherency rules as defined by the CPU in the system. + * + * What this API does not do: + * - Provide exclusion between various callers - including callers that + * operation on other mappings of the same physical page + * - Restore default attributes when a page is freed + * - Guarantee that mappings other than the requested one are + * in any state, other than that these do not violate rules for + * the CPU you have. Do not depend on any effects on other mappings, + * CPUs other than the one you have may have more relaxed rules. + * The caller is required to take care of these. + */ +int _set_memory_uc(unsigned long addr, int numpages); +int _set_memory_wc(unsigned long addr, int numpages); +int _set_memory_wb(unsigned long addr, int numpages); int set_memory_uc(unsigned long addr, int numpages); +int set_memory_wc(unsigned long addr, int numpages); int set_memory_wb(unsigned long addr, int numpages); int set_memory_x(unsigned long addr, int numpages); int set_memory_nx(unsigned long addr, int numpages); int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); int set_memory_np(unsigned long addr, int numpages); +int set_memory_4k(unsigned long addr, int numpages); + +/* + * For legacy compatibility with the old APIs, a few functions + * are provided that work on a "struct page". + * These functions operate ONLY on the 1:1 kernel mapping of the + * memory that the struct page represents, and internally just + * call the set_memory_* function. See the description of the + * set_memory_* function for more details on conventions. + * + * These APIs should be considered *deprecated* and are likely going to + * be removed in the future. + * The reason for this is the implicit operation on the 1:1 mapping only, + * making this not a generally useful API. + * + * Specifically, many users of the old APIs had a virtual address, + * called virt_to_page() or vmalloc_to_page() on that address to + * get a struct page* that the old API required. + * To convert these cases, use set_memory_*() on the original + * virtual address, do not use these functions. + */ + +int set_pages_uc(struct page *page, int numpages); +int set_pages_wb(struct page *page, int numpages); +int set_pages_x(struct page *page, int numpages); +int set_pages_nx(struct page *page, int numpages); +int set_pages_ro(struct page *page, int numpages); +int set_pages_rw(struct page *page, int numpages); + void clflush_cache_range(void *addr, unsigned int size); diff --git a/include/asm-x86/checksum_32.h b/include/asm-x86/checksum_32.h index 75194abbe8e..52bbb0d8c4c 100644 --- a/include/asm-x86/checksum_32.h +++ b/include/asm-x86/checksum_32.h @@ -28,7 +28,8 @@ asmlinkage __wsum csum_partial(const void *buff, int len, __wsum sum); */ asmlinkage __wsum csum_partial_copy_generic(const void *src, void *dst, - int len, __wsum sum, int *src_err_ptr, int *dst_err_ptr); + int len, __wsum sum, + int *src_err_ptr, int *dst_err_ptr); /* * Note: when you get a NULL pointer exception here this means someone @@ -37,20 +38,20 @@ asmlinkage __wsum csum_partial_copy_generic(const void *src, void *dst, * If you use these functions directly please don't forget the * access_ok(). */ -static __inline__ -__wsum csum_partial_copy_nocheck (const void *src, void *dst, - int len, __wsum sum) +static inline __wsum csum_partial_copy_nocheck(const void *src, void *dst, + int len, __wsum sum) { - return csum_partial_copy_generic ( src, dst, len, sum, NULL, NULL); + return csum_partial_copy_generic(src, dst, len, sum, NULL, NULL); } -static __inline__ -__wsum csum_partial_copy_from_user(const void __user *src, void *dst, - int len, __wsum sum, int *err_ptr) +static inline __wsum csum_partial_copy_from_user(const void __user *src, + void *dst, + int len, __wsum sum, + int *err_ptr) { might_sleep(); return csum_partial_copy_generic((__force void *)src, dst, - len, sum, err_ptr, NULL); + len, sum, err_ptr, NULL); } /* @@ -64,30 +65,29 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) { unsigned int sum; - __asm__ __volatile__( - "movl (%1), %0 ;\n" - "subl $4, %2 ;\n" - "jbe 2f ;\n" - "addl 4(%1), %0 ;\n" - "adcl 8(%1), %0 ;\n" - "adcl 12(%1), %0 ;\n" -"1: adcl 16(%1), %0 ;\n" - "lea 4(%1), %1 ;\n" - "decl %2 ;\n" - "jne 1b ;\n" - "adcl $0, %0 ;\n" - "movl %0, %2 ;\n" - "shrl $16, %0 ;\n" - "addw %w2, %w0 ;\n" - "adcl $0, %0 ;\n" - "notl %0 ;\n" -"2: ;\n" + asm volatile("movl (%1), %0 ;\n" + "subl $4, %2 ;\n" + "jbe 2f ;\n" + "addl 4(%1), %0 ;\n" + "adcl 8(%1), %0 ;\n" + "adcl 12(%1), %0;\n" + "1: adcl 16(%1), %0 ;\n" + "lea 4(%1), %1 ;\n" + "decl %2 ;\n" + "jne 1b ;\n" + "adcl $0, %0 ;\n" + "movl %0, %2 ;\n" + "shrl $16, %0 ;\n" + "addw %w2, %w0 ;\n" + "adcl $0, %0 ;\n" + "notl %0 ;\n" + "2: ;\n" /* Since the input registers which are loaded with iph and ihl are modified, we must also specify them as outputs, or gcc will assume they contain their original values. */ - : "=r" (sum), "=r" (iph), "=r" (ihl) - : "1" (iph), "2" (ihl) - : "memory"); + : "=r" (sum), "=r" (iph), "=r" (ihl) + : "1" (iph), "2" (ihl) + : "memory"); return (__force __sum16)sum; } @@ -97,29 +97,27 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) static inline __sum16 csum_fold(__wsum sum) { - __asm__( - "addl %1, %0 ;\n" - "adcl $0xffff, %0 ;\n" - : "=r" (sum) - : "r" ((__force u32)sum << 16), - "0" ((__force u32)sum & 0xffff0000) - ); + asm("addl %1, %0 ;\n" + "adcl $0xffff, %0 ;\n" + : "=r" (sum) + : "r" ((__force u32)sum << 16), + "0" ((__force u32)sum & 0xffff0000)); return (__force __sum16)(~(__force u32)sum >> 16); } static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, - unsigned short len, - unsigned short proto, - __wsum sum) + unsigned short len, + unsigned short proto, + __wsum sum) { - __asm__( - "addl %1, %0 ;\n" - "adcl %2, %0 ;\n" - "adcl %3, %0 ;\n" - "adcl $0, %0 ;\n" - : "=r" (sum) - : "g" (daddr), "g"(saddr), "g"((len + proto) << 8), "0"(sum)); - return sum; + asm("addl %1, %0 ;\n" + "adcl %2, %0 ;\n" + "adcl %3, %0 ;\n" + "adcl $0, %0 ;\n" + : "=r" (sum) + : "g" (daddr), "g"(saddr), + "g" ((len + proto) << 8), "0" (sum)); + return sum; } /* @@ -127,11 +125,11 @@ static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, * returns a 16-bit checksum, already complemented */ static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, - unsigned short len, - unsigned short proto, - __wsum sum) + unsigned short len, + unsigned short proto, + __wsum sum) { - return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum)); + return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); } /* @@ -141,30 +139,29 @@ static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, static inline __sum16 ip_compute_csum(const void *buff, int len) { - return csum_fold (csum_partial(buff, len, 0)); + return csum_fold(csum_partial(buff, len, 0)); } #define _HAVE_ARCH_IPV6_CSUM -static __inline__ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, - const struct in6_addr *daddr, - __u32 len, unsigned short proto, - __wsum sum) +static inline __sum16 csum_ipv6_magic(const struct in6_addr *saddr, + const struct in6_addr *daddr, + __u32 len, unsigned short proto, + __wsum sum) { - __asm__( - "addl 0(%1), %0 ;\n" - "adcl 4(%1), %0 ;\n" - "adcl 8(%1), %0 ;\n" - "adcl 12(%1), %0 ;\n" - "adcl 0(%2), %0 ;\n" - "adcl 4(%2), %0 ;\n" - "adcl 8(%2), %0 ;\n" - "adcl 12(%2), %0 ;\n" - "adcl %3, %0 ;\n" - "adcl %4, %0 ;\n" - "adcl $0, %0 ;\n" - : "=&r" (sum) - : "r" (saddr), "r" (daddr), - "r"(htonl(len)), "r"(htonl(proto)), "0"(sum)); + asm("addl 0(%1), %0 ;\n" + "adcl 4(%1), %0 ;\n" + "adcl 8(%1), %0 ;\n" + "adcl 12(%1), %0 ;\n" + "adcl 0(%2), %0 ;\n" + "adcl 4(%2), %0 ;\n" + "adcl 8(%2), %0 ;\n" + "adcl 12(%2), %0 ;\n" + "adcl %3, %0 ;\n" + "adcl %4, %0 ;\n" + "adcl $0, %0 ;\n" + : "=&r" (sum) + : "r" (saddr), "r" (daddr), + "r" (htonl(len)), "r" (htonl(proto)), "0" (sum)); return csum_fold(sum); } @@ -173,14 +170,15 @@ static __inline__ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, * Copy and checksum to user */ #define HAVE_CSUM_COPY_USER -static __inline__ __wsum csum_and_copy_to_user(const void *src, - void __user *dst, - int len, __wsum sum, - int *err_ptr) +static inline __wsum csum_and_copy_to_user(const void *src, + void __user *dst, + int len, __wsum sum, + int *err_ptr) { might_sleep(); if (access_ok(VERIFY_WRITE, dst, len)) - return csum_partial_copy_generic(src, (__force void *)dst, len, sum, NULL, err_ptr); + return csum_partial_copy_generic(src, (__force void *)dst, + len, sum, NULL, err_ptr); if (len) *err_ptr = -EFAULT; diff --git a/include/asm-x86/checksum_64.h b/include/asm-x86/checksum_64.h index e5f79997dec..8bd861cc526 100644 --- a/include/asm-x86/checksum_64.h +++ b/include/asm-x86/checksum_64.h @@ -1,33 +1,31 @@ #ifndef _X86_64_CHECKSUM_H #define _X86_64_CHECKSUM_H -/* - * Checksums for x86-64 - * Copyright 2002 by Andi Kleen, SuSE Labs +/* + * Checksums for x86-64 + * Copyright 2002 by Andi Kleen, SuSE Labs * with some code from asm-x86/checksum.h - */ + */ #include <linux/compiler.h> #include <asm/uaccess.h> #include <asm/byteorder.h> -/** +/** * csum_fold - Fold and invert a 32bit checksum. * sum: 32bit unfolded sum - * + * * Fold a 32bit running checksum to 16bit and invert it. This is usually * the last step before putting a checksum into a packet. * Make sure not to mix with 64bit checksums. */ static inline __sum16 csum_fold(__wsum sum) { - __asm__( - " addl %1,%0\n" - " adcl $0xffff,%0" - : "=r" (sum) - : "r" ((__force u32)sum << 16), - "0" ((__force u32)sum & 0xffff0000) - ); + asm(" addl %1,%0\n" + " adcl $0xffff,%0" + : "=r" (sum) + : "r" ((__force u32)sum << 16), + "0" ((__force u32)sum & 0xffff0000)); return (__force __sum16)(~(__force u32)sum >> 16); } @@ -43,46 +41,46 @@ static inline __sum16 csum_fold(__wsum sum) * ip_fast_csum - Compute the IPv4 header checksum efficiently. * iph: ipv4 header * ihl: length of header / 4 - */ + */ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) { unsigned int sum; - asm( " movl (%1), %0\n" - " subl $4, %2\n" - " jbe 2f\n" - " addl 4(%1), %0\n" - " adcl 8(%1), %0\n" - " adcl 12(%1), %0\n" - "1: adcl 16(%1), %0\n" - " lea 4(%1), %1\n" - " decl %2\n" - " jne 1b\n" - " adcl $0, %0\n" - " movl %0, %2\n" - " shrl $16, %0\n" - " addw %w2, %w0\n" - " adcl $0, %0\n" - " notl %0\n" - "2:" + asm(" movl (%1), %0\n" + " subl $4, %2\n" + " jbe 2f\n" + " addl 4(%1), %0\n" + " adcl 8(%1), %0\n" + " adcl 12(%1), %0\n" + "1: adcl 16(%1), %0\n" + " lea 4(%1), %1\n" + " decl %2\n" + " jne 1b\n" + " adcl $0, %0\n" + " movl %0, %2\n" + " shrl $16, %0\n" + " addw %w2, %w0\n" + " adcl $0, %0\n" + " notl %0\n" + "2:" /* Since the input registers which are loaded with iph and ihl are modified, we must also specify them as outputs, or gcc will assume they contain their original values. */ - : "=r" (sum), "=r" (iph), "=r" (ihl) - : "1" (iph), "2" (ihl) - : "memory"); + : "=r" (sum), "=r" (iph), "=r" (ihl) + : "1" (iph), "2" (ihl) + : "memory"); return (__force __sum16)sum; } -/** +/** * csum_tcpup_nofold - Compute an IPv4 pseudo header checksum. * @saddr: source address * @daddr: destination address * @len: length of packet * @proto: ip protocol of packet - * @sum: initial sum to be added in (32bit unfolded) - * - * Returns the pseudo header checksum the input data. Result is + * @sum: initial sum to be added in (32bit unfolded) + * + * Returns the pseudo header checksum the input data. Result is * 32bit unfolded. */ static inline __wsum @@ -93,32 +91,32 @@ csum_tcpudp_nofold(__be32 saddr, __be32 daddr, unsigned short len, " adcl %2, %0\n" " adcl %3, %0\n" " adcl $0, %0\n" - : "=r" (sum) + : "=r" (sum) : "g" (daddr), "g" (saddr), "g" ((len + proto)<<8), "0" (sum)); - return sum; + return sum; } -/** +/** * csum_tcpup_magic - Compute an IPv4 pseudo header checksum. * @saddr: source address * @daddr: destination address * @len: length of packet * @proto: ip protocol of packet - * @sum: initial sum to be added in (32bit unfolded) - * + * @sum: initial sum to be added in (32bit unfolded) + * * Returns the 16bit pseudo header checksum the input data already * complemented and ready to be filled in. */ -static inline __sum16 -csum_tcpudp_magic(__be32 saddr, __be32 daddr, - unsigned short len, unsigned short proto, __wsum sum) +static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, + unsigned short len, + unsigned short proto, __wsum sum) { - return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum)); + return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); } -/** +/** * csum_partial - Compute an internet checksum. * @buff: buffer to be checksummed * @len: length of buffer. @@ -127,7 +125,7 @@ csum_tcpudp_magic(__be32 saddr, __be32 daddr, * Returns the 32bit unfolded internet checksum of the buffer. * Before filling it in it needs to be csum_fold()'ed. * buff should be aligned to a 64bit boundary if possible. - */ + */ extern __wsum csum_partial(const void *buff, int len, __wsum sum); #define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER 1 @@ -136,23 +134,22 @@ extern __wsum csum_partial(const void *buff, int len, __wsum sum); /* Do not call this directly. Use the wrappers below */ extern __wsum csum_partial_copy_generic(const void *src, const void *dst, - int len, - __wsum sum, - int *src_err_ptr, int *dst_err_ptr); + int len, __wsum sum, + int *src_err_ptr, int *dst_err_ptr); extern __wsum csum_partial_copy_from_user(const void __user *src, void *dst, - int len, __wsum isum, int *errp); + int len, __wsum isum, int *errp); extern __wsum csum_partial_copy_to_user(const void *src, void __user *dst, - int len, __wsum isum, int *errp); -extern __wsum csum_partial_copy_nocheck(const void *src, void *dst, int len, - __wsum sum); + int len, __wsum isum, int *errp); +extern __wsum csum_partial_copy_nocheck(const void *src, void *dst, + int len, __wsum sum); /* Old names. To be removed. */ #define csum_and_copy_to_user csum_partial_copy_to_user #define csum_and_copy_from_user csum_partial_copy_from_user -/** +/** * ip_compute_csum - Compute an 16bit IP checksum. * @buff: buffer address. * @len: length of buffer. @@ -170,7 +167,7 @@ extern __sum16 ip_compute_csum(const void *buff, int len); * @proto: protocol of packet * @sum: initial sum (32bit unfolded) to be added in * - * Computes an IPv6 pseudo header checksum. This sum is added the checksum + * Computes an IPv6 pseudo header checksum. This sum is added the checksum * into UDP/TCP packets and contains some link layer information. * Returns the unfolded 32bit checksum. */ @@ -185,11 +182,10 @@ csum_ipv6_magic(const struct in6_addr *saddr, const struct in6_addr *daddr, static inline unsigned add32_with_carry(unsigned a, unsigned b) { asm("addl %2,%0\n\t" - "adcl $0,%0" - : "=r" (a) + "adcl $0,%0" + : "=r" (a) : "0" (a), "r" (b)); return a; } #endif - diff --git a/include/asm-x86/cmpxchg_32.h b/include/asm-x86/cmpxchg_32.h index 959fad00dff..bf5a69d1329 100644 --- a/include/asm-x86/cmpxchg_32.h +++ b/include/asm-x86/cmpxchg_32.h @@ -8,9 +8,12 @@ * you need to test for the feature in boot_cpu_data. */ -#define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr)))) +#define xchg(ptr, v) \ + ((__typeof__(*(ptr)))__xchg((unsigned long)(v), (ptr), sizeof(*(ptr)))) -struct __xchg_dummy { unsigned long a[100]; }; +struct __xchg_dummy { + unsigned long a[100]; +}; #define __xg(x) ((struct __xchg_dummy *)(x)) /* @@ -27,72 +30,74 @@ struct __xchg_dummy { unsigned long a[100]; }; * of the instruction set reference 24319102.pdf. We need * the reader side to see the coherent 64bit value. */ -static inline void __set_64bit (unsigned long long * ptr, - unsigned int low, unsigned int high) +static inline void __set_64bit(unsigned long long *ptr, + unsigned int low, unsigned int high) { - __asm__ __volatile__ ( - "\n1:\t" - "movl (%0), %%eax\n\t" - "movl 4(%0), %%edx\n\t" - LOCK_PREFIX "cmpxchg8b (%0)\n\t" - "jnz 1b" - : /* no outputs */ - : "D"(ptr), - "b"(low), - "c"(high) - : "ax","dx","memory"); + asm volatile("\n1:\t" + "movl (%0), %%eax\n\t" + "movl 4(%0), %%edx\n\t" + LOCK_PREFIX "cmpxchg8b (%0)\n\t" + "jnz 1b" + : /* no outputs */ + : "D"(ptr), + "b"(low), + "c"(high) + : "ax", "dx", "memory"); } -static inline void __set_64bit_constant (unsigned long long *ptr, - unsigned long long value) +static inline void __set_64bit_constant(unsigned long long *ptr, + unsigned long long value) { - __set_64bit(ptr,(unsigned int)(value), (unsigned int)((value)>>32ULL)); + __set_64bit(ptr, (unsigned int)value, (unsigned int)(value >> 32)); } -#define ll_low(x) *(((unsigned int*)&(x))+0) -#define ll_high(x) *(((unsigned int*)&(x))+1) -static inline void __set_64bit_var (unsigned long long *ptr, - unsigned long long value) +#define ll_low(x) *(((unsigned int *)&(x)) + 0) +#define ll_high(x) *(((unsigned int *)&(x)) + 1) + +static inline void __set_64bit_var(unsigned long long *ptr, + unsigned long long value) { - __set_64bit(ptr,ll_low(value), ll_high(value)); + __set_64bit(ptr, ll_low(value), ll_high(value)); } -#define set_64bit(ptr,value) \ -(__builtin_constant_p(value) ? \ - __set_64bit_constant(ptr, value) : \ - __set_64bit_var(ptr, value) ) +#define set_64bit(ptr, value) \ + (__builtin_constant_p((value)) \ + ? __set_64bit_constant((ptr), (value)) \ + : __set_64bit_var((ptr), (value))) -#define _set_64bit(ptr,value) \ -(__builtin_constant_p(value) ? \ - __set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \ - __set_64bit(ptr, ll_low(value), ll_high(value)) ) +#define _set_64bit(ptr, value) \ + (__builtin_constant_p(value) \ + ? __set_64bit(ptr, (unsigned int)(value), \ + (unsigned int)((value) >> 32)) \ + : __set_64bit(ptr, ll_low((value)), ll_high((value)))) /* * Note: no "lock" prefix even on SMP: xchg always implies lock anyway * Note 2: xchg has side effect, so that attribute volatile is necessary, * but generally the primitive is invalid, *ptr is output argument. --ANK */ -static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) +static inline unsigned long __xchg(unsigned long x, volatile void *ptr, + int size) { switch (size) { - case 1: - __asm__ __volatile__("xchgb %b0,%1" - :"=q" (x) - :"m" (*__xg(ptr)), "0" (x) - :"memory"); - break; - case 2: - __asm__ __volatile__("xchgw %w0,%1" - :"=r" (x) - :"m" (*__xg(ptr)), "0" (x) - :"memory"); - break; - case 4: - __asm__ __volatile__("xchgl %0,%1" - :"=r" (x) - :"m" (*__xg(ptr)), "0" (x) - :"memory"); - break; + case 1: + asm volatile("xchgb %b0,%1" + : "=q" (x) + : "m" (*__xg(ptr)), "0" (x) + : "memory"); + break; + case 2: + asm volatile("xchgw %w0,%1" + : "=r" (x) + : "m" (*__xg(ptr)), "0" (x) + : "memory"); + break; + case 4: + asm volatile("xchgl %0,%1" + : "=r" (x) + : "m" (*__xg(ptr)), "0" (x) + : "memory"); + break; } return x; } @@ -105,24 +110,27 @@ static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int siz #ifdef CONFIG_X86_CMPXCHG #define __HAVE_ARCH_CMPXCHG 1 -#define cmpxchg(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ - (unsigned long)(n), sizeof(*(ptr)))) -#define sync_cmpxchg(ptr, o, n) \ - ((__typeof__(*(ptr)))__sync_cmpxchg((ptr), (unsigned long)(o), \ - (unsigned long)(n), sizeof(*(ptr)))) -#define cmpxchg_local(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ - (unsigned long)(n), sizeof(*(ptr)))) +#define cmpxchg(ptr, o, n) \ + ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ + (unsigned long)(n), \ + sizeof(*(ptr)))) +#define sync_cmpxchg(ptr, o, n) \ + ((__typeof__(*(ptr)))__sync_cmpxchg((ptr), (unsigned long)(o), \ + (unsigned long)(n), \ + sizeof(*(ptr)))) +#define cmpxchg_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ + (unsigned long)(n), \ + sizeof(*(ptr)))) #endif #ifdef CONFIG_X86_CMPXCHG64 -#define cmpxchg64(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ - (unsigned long long)(n))) -#define cmpxchg64_local(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o),\ - (unsigned long long)(n))) +#define cmpxchg64(ptr, o, n) \ + ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ + (unsigned long long)(n))) +#define cmpxchg64_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ + (unsigned long long)(n))) #endif static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, @@ -131,22 +139,22 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, unsigned long prev; switch (size) { case 1: - __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2" - : "=a"(prev) - : "q"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile(LOCK_PREFIX "cmpxchgb %b1,%2" + : "=a"(prev) + : "q"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 2: - __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile(LOCK_PREFIX "cmpxchgw %w1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 4: - __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile(LOCK_PREFIX "cmpxchgl %1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; } return old; @@ -158,85 +166,88 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, * isn't. */ static inline unsigned long __sync_cmpxchg(volatile void *ptr, - unsigned long old, - unsigned long new, int size) + unsigned long old, + unsigned long new, int size) { unsigned long prev; switch (size) { case 1: - __asm__ __volatile__("lock; cmpxchgb %b1,%2" - : "=a"(prev) - : "q"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile("lock; cmpxchgb %b1,%2" + : "=a"(prev) + : "q"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 2: - __asm__ __volatile__("lock; cmpxchgw %w1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile("lock; cmpxchgw %w1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 4: - __asm__ __volatile__("lock; cmpxchgl %1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile("lock; cmpxchgl %1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; } return old; } static inline unsigned long __cmpxchg_local(volatile void *ptr, - unsigned long old, unsigned long new, int size) + unsigned long old, + unsigned long new, int size) { unsigned long prev; switch (size) { case 1: - __asm__ __volatile__("cmpxchgb %b1,%2" - : "=a"(prev) - : "q"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile("cmpxchgb %b1,%2" + : "=a"(prev) + : "q"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 2: - __asm__ __volatile__("cmpxchgw %w1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile("cmpxchgw %w1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 4: - __asm__ __volatile__("cmpxchgl %1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile("cmpxchgl %1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; } return old; } static inline unsigned long long __cmpxchg64(volatile void *ptr, - unsigned long long old, unsigned long long new) + unsigned long long old, + unsigned long long new) { unsigned long long prev; - __asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3" - : "=A"(prev) - : "b"((unsigned long)new), - "c"((unsigned long)(new >> 32)), - "m"(*__xg(ptr)), - "0"(old) - : "memory"); + asm volatile(LOCK_PREFIX "cmpxchg8b %3" + : "=A"(prev) + : "b"((unsigned long)new), + "c"((unsigned long)(new >> 32)), + "m"(*__xg(ptr)), + "0"(old) + : "memory"); return prev; } static inline unsigned long long __cmpxchg64_local(volatile void *ptr, - unsigned long long old, unsigned long long new) + unsigned long long old, + unsigned long long new) { unsigned long long prev; - __asm__ __volatile__("cmpxchg8b %3" - : "=A"(prev) - : "b"((unsigned long)new), - "c"((unsigned long)(new >> 32)), - "m"(*__xg(ptr)), - "0"(old) - : "memory"); + asm volatile("cmpxchg8b %3" + : "=A"(prev) + : "b"((unsigned long)new), + "c"((unsigned long)(new >> 32)), + "m"(*__xg(ptr)), + "0"(old) + : "memory"); return prev; } @@ -252,7 +263,7 @@ extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16); extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32); static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old, - unsigned long new, int size) + unsigned long new, int size) { switch (size) { case 1: diff --git a/include/asm-x86/cmpxchg_64.h b/include/asm-x86/cmpxchg_64.h index 56f5b41e071..d9b26b9a28c 100644 --- a/include/asm-x86/cmpxchg_64.h +++ b/include/asm-x86/cmpxchg_64.h @@ -3,7 +3,8 @@ #include <asm/alternative.h> /* Provides LOCK_PREFIX */ -#define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr)))) +#define xchg(ptr, v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v), \ + (ptr), sizeof(*(ptr)))) #define __xg(x) ((volatile long *)(x)) @@ -19,33 +20,34 @@ static inline void set_64bit(volatile unsigned long *ptr, unsigned long val) * Note 2: xchg has side effect, so that attribute volatile is necessary, * but generally the primitive is invalid, *ptr is output argument. --ANK */ -static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) +static inline unsigned long __xchg(unsigned long x, volatile void *ptr, + int size) { switch (size) { - case 1: - __asm__ __volatile__("xchgb %b0,%1" - :"=q" (x) - :"m" (*__xg(ptr)), "0" (x) - :"memory"); - break; - case 2: - __asm__ __volatile__("xchgw %w0,%1" - :"=r" (x) - :"m" (*__xg(ptr)), "0" (x) - :"memory"); - break; - case 4: - __asm__ __volatile__("xchgl %k0,%1" - :"=r" (x) - :"m" (*__xg(ptr)), "0" (x) - :"memory"); - break; - case 8: - __asm__ __volatile__("xchgq %0,%1" - :"=r" (x) - :"m" (*__xg(ptr)), "0" (x) - :"memory"); - break; + case 1: + asm volatile("xchgb %b0,%1" + : "=q" (x) + : "m" (*__xg(ptr)), "0" (x) + : "memory"); + break; + case 2: + asm volatile("xchgw %w0,%1" + : "=r" (x) + : "m" (*__xg(ptr)), "0" (x) + : "memory"); + break; + case 4: + asm volatile("xchgl %k0,%1" + : "=r" (x) + : "m" (*__xg(ptr)), "0" (x) + : "memory"); + break; + case 8: + asm volatile("xchgq %0,%1" + : "=r" (x) + : "m" (*__xg(ptr)), "0" (x) + : "memory"); + break; } return x; } @@ -64,61 +66,62 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, unsigned long prev; switch (size) { case 1: - __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2" - : "=a"(prev) - : "q"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile(LOCK_PREFIX "cmpxchgb %b1,%2" + : "=a"(prev) + : "q"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 2: - __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile(LOCK_PREFIX "cmpxchgw %w1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 4: - __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %k1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile(LOCK_PREFIX "cmpxchgl %k1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 8: - __asm__ __volatile__(LOCK_PREFIX "cmpxchgq %1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile(LOCK_PREFIX "cmpxchgq %1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; } return old; } static inline unsigned long __cmpxchg_local(volatile void *ptr, - unsigned long old, unsigned long new, int size) + unsigned long old, + unsigned long new, int size) { unsigned long prev; switch (size) { case 1: - __asm__ __volatile__("cmpxchgb %b1,%2" - : "=a"(prev) - : "q"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile("cmpxchgb %b1,%2" + : "=a"(prev) + : "q"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 2: - __asm__ __volatile__("cmpxchgw %w1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile("cmpxchgw %w1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 4: - __asm__ __volatile__("cmpxchgl %k1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile("cmpxchgl %k1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; case 8: - __asm__ __volatile__("cmpxchgq %1,%2" - : "=a"(prev) - : "r"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); + asm volatile("cmpxchgq %1,%2" + : "=a"(prev) + : "r"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); return prev; } return old; @@ -126,19 +129,20 @@ static inline unsigned long __cmpxchg_local(volatile void *ptr, #define cmpxchg(ptr, o, n) \ ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ - (unsigned long)(n), sizeof(*(ptr)))) + (unsigned long)(n), sizeof(*(ptr)))) #define cmpxchg64(ptr, o, n) \ - ({ \ +({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ cmpxchg((ptr), (o), (n)); \ - }) +}) #define cmpxchg_local(ptr, o, n) \ ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ - (unsigned long)(n), sizeof(*(ptr)))) + (unsigned long)(n), \ + sizeof(*(ptr)))) #define cmpxchg64_local(ptr, o, n) \ - ({ \ +({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ cmpxchg_local((ptr), (o), (n)); \ - }) +}) #endif diff --git a/include/asm-x86/compat.h b/include/asm-x86/compat.h index d3e8f3e87ee..1793ac317a3 100644 --- a/include/asm-x86/compat.h +++ b/include/asm-x86/compat.h @@ -204,7 +204,7 @@ static inline compat_uptr_t ptr_to_compat(void __user *uptr) return (u32)(unsigned long)uptr; } -static __inline__ void __user *compat_alloc_user_space(long len) +static inline void __user *compat_alloc_user_space(long len) { struct pt_regs *regs = task_pt_regs(current); return (void __user *)regs->sp - len; diff --git a/include/asm-x86/cpufeature.h b/include/asm-x86/cpufeature.h index 065e92966c7..0d609c837a4 100644 --- a/include/asm-x86/cpufeature.h +++ b/include/asm-x86/cpufeature.h @@ -120,6 +120,9 @@ extern const char * const x86_cap_flags[NCAPINTS*32]; extern const char * const x86_power_flags[32]; +#define test_cpu_cap(c, bit) \ + test_bit(bit, (unsigned long *)((c)->x86_capability)) + #define cpu_has(c, bit) \ (__builtin_constant_p(bit) && \ ( (((bit)>>5)==0 && (1UL<<((bit)&31) & REQUIRED_MASK0)) || \ @@ -131,7 +134,8 @@ extern const char * const x86_power_flags[32]; (((bit)>>5)==6 && (1UL<<((bit)&31) & REQUIRED_MASK6)) || \ (((bit)>>5)==7 && (1UL<<((bit)&31) & REQUIRED_MASK7)) ) \ ? 1 : \ - test_bit(bit, (unsigned long *)((c)->x86_capability))) + test_cpu_cap(c, bit)) + #define boot_cpu_has(bit) cpu_has(&boot_cpu_data, bit) #define set_cpu_cap(c, bit) set_bit(bit, (unsigned long *)((c)->x86_capability)) @@ -181,6 +185,8 @@ extern const char * const x86_power_flags[32]; #define cpu_has_clflush boot_cpu_has(X86_FEATURE_CLFLSH) #define cpu_has_bts boot_cpu_has(X86_FEATURE_BTS) #define cpu_has_gbpages boot_cpu_has(X86_FEATURE_GBPAGES) +#define cpu_has_arch_perfmon boot_cpu_has(X86_FEATURE_ARCH_PERFMON) +#define cpu_has_pat boot_cpu_has(X86_FEATURE_PAT) #if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64) # define cpu_has_invlpg 1 diff --git a/include/asm-x86/current_32.h b/include/asm-x86/current_32.h index d3524853991..5af9bdb97a1 100644 --- a/include/asm-x86/current_32.h +++ b/include/asm-x86/current_32.h @@ -11,7 +11,7 @@ static __always_inline struct task_struct *get_current(void) { return x86_read_percpu(current_task); } - + #define current get_current() #endif /* !(_I386_CURRENT_H) */ diff --git a/include/asm-x86/current_64.h b/include/asm-x86/current_64.h index bc8adecee66..2d368ede2fc 100644 --- a/include/asm-x86/current_64.h +++ b/include/asm-x86/current_64.h @@ -1,23 +1,23 @@ #ifndef _X86_64_CURRENT_H #define _X86_64_CURRENT_H -#if !defined(__ASSEMBLY__) +#if !defined(__ASSEMBLY__) struct task_struct; #include <asm/pda.h> -static inline struct task_struct *get_current(void) -{ - struct task_struct *t = read_pda(pcurrent); +static inline struct task_struct *get_current(void) +{ + struct task_struct *t = read_pda(pcurrent); return t; -} +} #define current get_current() #else #ifndef ASM_OFFSET_H -#include <asm/asm-offsets.h> +#include <asm/asm-offsets.h> #endif #define GET_CURRENT(reg) movq %gs:(pda_pcurrent),reg diff --git a/include/asm-x86/desc.h b/include/asm-x86/desc.h index 5b6a05d3a77..268a012bcd7 100644 --- a/include/asm-x86/desc.h +++ b/include/asm-x86/desc.h @@ -62,8 +62,8 @@ static inline struct desc_struct *get_cpu_gdt_table(unsigned int cpu) } static inline void pack_gate(gate_desc *gate, unsigned char type, - unsigned long base, unsigned dpl, unsigned flags, unsigned short seg) - + unsigned long base, unsigned dpl, unsigned flags, + unsigned short seg) { gate->a = (seg << 16) | (base & 0xffff); gate->b = (base & 0xffff0000) | @@ -84,22 +84,23 @@ static inline int desc_empty(const void *ptr) #define load_TR_desc() native_load_tr_desc() #define load_gdt(dtr) native_load_gdt(dtr) #define load_idt(dtr) native_load_idt(dtr) -#define load_tr(tr) __asm__ __volatile("ltr %0"::"m" (tr)) -#define load_ldt(ldt) __asm__ __volatile("lldt %0"::"m" (ldt)) +#define load_tr(tr) asm volatile("ltr %0"::"m" (tr)) +#define load_ldt(ldt) asm volatile("lldt %0"::"m" (ldt)) #define store_gdt(dtr) native_store_gdt(dtr) #define store_idt(dtr) native_store_idt(dtr) #define store_tr(tr) (tr = native_store_tr()) -#define store_ldt(ldt) __asm__ ("sldt %0":"=m" (ldt)) +#define store_ldt(ldt) asm("sldt %0":"=m" (ldt)) #define load_TLS(t, cpu) native_load_tls(t, cpu) #define set_ldt native_set_ldt -#define write_ldt_entry(dt, entry, desc) \ - native_write_ldt_entry(dt, entry, desc) -#define write_gdt_entry(dt, entry, desc, type) \ - native_write_gdt_entry(dt, entry, desc, type) -#define write_idt_entry(dt, entry, g) native_write_idt_entry(dt, entry, g) +#define write_ldt_entry(dt, entry, desc) \ + native_write_ldt_entry(dt, entry, desc) +#define write_gdt_entry(dt, entry, desc, type) \ + native_write_gdt_entry(dt, entry, desc, type) +#define write_idt_entry(dt, entry, g) \ + native_write_idt_entry(dt, entry, g) #endif static inline void native_write_idt_entry(gate_desc *idt, int entry, @@ -138,8 +139,8 @@ static inline void pack_descriptor(struct desc_struct *desc, unsigned long base, { desc->a = ((base & 0xffff) << 16) | (limit & 0xffff); desc->b = (base & 0xff000000) | ((base & 0xff0000) >> 16) | - (limit & 0x000f0000) | ((type & 0xff) << 8) | - ((flags & 0xf) << 20); + (limit & 0x000f0000) | ((type & 0xff) << 8) | + ((flags & 0xf) << 20); desc->p = 1; } @@ -159,7 +160,6 @@ static inline void set_tssldt_descriptor(void *d, unsigned long addr, desc->base2 = (PTR_MIDDLE(addr) >> 8) & 0xFF; desc->base3 = PTR_HIGH(addr); #else - pack_descriptor((struct desc_struct *)d, addr, size, 0x80 | type, 0); #endif } @@ -177,7 +177,8 @@ static inline void __set_tss_desc(unsigned cpu, unsigned int entry, void *addr) * last valid byte */ set_tssldt_descriptor(&tss, (unsigned long)addr, DESC_TSS, - IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1); + IO_BITMAP_OFFSET + IO_BITMAP_BYTES + + sizeof(unsigned long) - 1); write_gdt_entry(d, entry, &tss, DESC_TSS); } @@ -186,7 +187,7 @@ static inline void __set_tss_desc(unsigned cpu, unsigned int entry, void *addr) static inline void native_set_ldt(const void *addr, unsigned int entries) { if (likely(entries == 0)) - __asm__ __volatile__("lldt %w0"::"q" (0)); + asm volatile("lldt %w0"::"q" (0)); else { unsigned cpu = smp_processor_id(); ldt_desc ldt; @@ -195,7 +196,7 @@ static inline void native_set_ldt(const void *addr, unsigned int entries) DESC_LDT, entries * sizeof(ldt) - 1); write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_LDT, &ldt, DESC_LDT); - __asm__ __volatile__("lldt %w0"::"q" (GDT_ENTRY_LDT*8)); + asm volatile("lldt %w0"::"q" (GDT_ENTRY_LDT*8)); } } @@ -240,15 +241,15 @@ static inline void native_load_tls(struct thread_struct *t, unsigned int cpu) gdt[GDT_ENTRY_TLS_MIN + i] = t->tls_array[i]; } -#define _LDT_empty(info) (\ - (info)->base_addr == 0 && \ - (info)->limit == 0 && \ - (info)->contents == 0 && \ - (info)->read_exec_only == 1 && \ - (info)->seg_32bit == 0 && \ - (info)->limit_in_pages == 0 && \ - (info)->seg_not_present == 1 && \ - (info)->useable == 0) +#define _LDT_empty(info) \ + ((info)->base_addr == 0 && \ + (info)->limit == 0 && \ + (info)->contents == 0 && \ + (info)->read_exec_only == 1 && \ + (info)->seg_32bit == 0 && \ + (info)->limit_in_pages == 0 && \ + (info)->seg_not_present == 1 && \ + (info)->useable == 0) #ifdef CONFIG_X86_64 #define LDT_empty(info) (_LDT_empty(info) && ((info)->lm == 0)) @@ -287,7 +288,7 @@ static inline unsigned long get_desc_limit(const struct desc_struct *desc) } static inline void _set_gate(int gate, unsigned type, void *addr, - unsigned dpl, unsigned ist, unsigned seg) + unsigned dpl, unsigned ist, unsigned seg) { gate_desc s; pack_gate(&s, type, (unsigned long)addr, dpl, ist, seg); @@ -370,10 +371,10 @@ static inline void set_system_gate_ist(int n, void *addr, unsigned ist) * Will read the base address of GDT_ENTRY_ESPFIX_SS and put it into %eax. */ #define GET_DESC_BASE(idx, gdt, base, lo_w, lo_b, hi_b) \ - movb idx*8+4(gdt), lo_b; \ - movb idx*8+7(gdt), hi_b; \ - shll $16, base; \ - movw idx*8+2(gdt), lo_w; + movb idx * 8 + 4(gdt), lo_b; \ + movb idx * 8 + 7(gdt), hi_b; \ + shll $16, base; \ + movw idx * 8 + 2(gdt), lo_w; #endif /* __ASSEMBLY__ */ diff --git a/include/asm-x86/desc_defs.h b/include/asm-x86/desc_defs.h index e33f078b3e5..eccb4ea1f91 100644 --- a/include/asm-x86/desc_defs.h +++ b/include/asm-x86/desc_defs.h @@ -18,17 +18,19 @@ * incrementally. We keep the signature as a struct, rather than an union, * so we can get rid of it transparently in the future -- glommer */ -// 8 byte segment descriptor +/* 8 byte segment descriptor */ struct desc_struct { union { - struct { unsigned int a, b; }; + struct { + unsigned int a; + unsigned int b; + }; struct { u16 limit0; u16 base0; unsigned base1: 8, type: 4, s: 1, dpl: 2, p: 1; unsigned limit: 4, avl: 1, l: 1, d: 1, g: 1, base2: 8; }; - }; } __attribute__((packed)); @@ -39,7 +41,7 @@ enum { GATE_TASK = 0x5, }; -// 16byte gate +/* 16byte gate */ struct gate_struct64 { u16 offset_low; u16 segment; @@ -56,10 +58,10 @@ struct gate_struct64 { enum { DESC_TSS = 0x9, DESC_LDT = 0x2, - DESCTYPE_S = 0x10, /* !system */ + DESCTYPE_S = 0x10, /* !system */ }; -// LDT or TSS descriptor in the GDT. 16 bytes. +/* LDT or TSS descriptor in the GDT. 16 bytes. */ struct ldttss_desc64 { u16 limit0; u16 base0; @@ -84,7 +86,6 @@ struct desc_ptr { unsigned long address; } __attribute__((packed)) ; - #endif /* !__ASSEMBLY__ */ #endif diff --git a/include/asm-x86/div64.h b/include/asm-x86/div64.h index e98d16e7a37..9a2d644c08e 100644 --- a/include/asm-x86/div64.h +++ b/include/asm-x86/div64.h @@ -17,40 +17,41 @@ * This ends up being the most efficient "calling * convention" on x86. */ -#define do_div(n,base) ({ \ - unsigned long __upper, __low, __high, __mod, __base; \ - __base = (base); \ - asm("":"=a" (__low), "=d" (__high):"A" (n)); \ - __upper = __high; \ - if (__high) { \ - __upper = __high % (__base); \ - __high = __high / (__base); \ - } \ - asm("divl %2":"=a" (__low), "=d" (__mod):"rm" (__base), "0" (__low), "1" (__upper)); \ - asm("":"=A" (n):"a" (__low),"d" (__high)); \ - __mod; \ +#define do_div(n, base) \ +({ \ + unsigned long __upper, __low, __high, __mod, __base; \ + __base = (base); \ + asm("":"=a" (__low), "=d" (__high) : "A" (n)); \ + __upper = __high; \ + if (__high) { \ + __upper = __high % (__base); \ + __high = __high / (__base); \ + } \ + asm("divl %2":"=a" (__low), "=d" (__mod) \ + : "rm" (__base), "0" (__low), "1" (__upper)); \ + asm("":"=A" (n) : "a" (__low), "d" (__high)); \ + __mod; \ }) -/* - * (long)X = ((long long)divs) / (long)div - * (long)rem = ((long long)divs) % (long)div - * - * Warning, this will do an exception if X overflows. - */ -#define div_long_long_rem(a,b,c) div_ll_X_l_rem(a,b,c) - -static inline long -div_ll_X_l_rem(long long divs, long div, long *rem) +static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) { - long dum2; - __asm__("divl %2":"=a"(dum2), "=d"(*rem) - : "rm"(div), "A"(divs)); - - return dum2; - + union { + u64 v64; + u32 v32[2]; + } d = { dividend }; + u32 upper; + + upper = d.v32[1]; + d.v32[1] = 0; + if (upper >= divisor) { + d.v32[1] = upper / divisor; + upper %= divisor; + } + asm ("divl %2" : "=a" (d.v32[0]), "=d" (*remainder) : + "rm" (divisor), "0" (d.v32[0]), "1" (upper)); + return d.v64; } - -extern uint64_t div64_64(uint64_t dividend, uint64_t divisor); +#define div_u64_rem div_u64_rem #else # include <asm-generic/div64.h> diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 58f790f4df5..a1a4dc7fe6e 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -1,5 +1,237 @@ +#ifndef _ASM_DMA_MAPPING_H_ +#define _ASM_DMA_MAPPING_H_ + +/* + * IOMMU interface. See Documentation/DMA-mapping.txt and DMA-API.txt for + * documentation. + */ + +#include <linux/scatterlist.h> +#include <asm/io.h> +#include <asm/swiotlb.h> + +extern dma_addr_t bad_dma_address; +extern int iommu_merge; +extern struct device fallback_dev; +extern int panic_on_overflow; +extern int forbid_dac; +extern int force_iommu; + +struct dma_mapping_ops { + int (*mapping_error)(dma_addr_t dma_addr); + void* (*alloc_coherent)(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp); + void (*free_coherent)(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle); + dma_addr_t (*map_single)(struct device *hwdev, phys_addr_t ptr, + size_t size, int direction); + /* like map_single, but doesn't check the device mask */ + dma_addr_t (*map_simple)(struct device *hwdev, phys_addr_t ptr, + size_t size, int direction); + void (*unmap_single)(struct device *dev, dma_addr_t addr, + size_t size, int direction); + void (*sync_single_for_cpu)(struct device *hwdev, + dma_addr_t dma_handle, size_t size, + int direction); + void (*sync_single_for_device)(struct device *hwdev, + dma_addr_t dma_handle, size_t size, + int direction); + void (*sync_single_range_for_cpu)(struct device *hwdev, + dma_addr_t dma_handle, unsigned long offset, + size_t size, int direction); + void (*sync_single_range_for_device)(struct device *hwdev, + dma_addr_t dma_handle, unsigned long offset, + size_t size, int direction); + void (*sync_sg_for_cpu)(struct device *hwdev, + struct scatterlist *sg, int nelems, + int direction); + void (*sync_sg_for_device)(struct device *hwdev, + struct scatterlist *sg, int nelems, + int direction); + int (*map_sg)(struct device *hwdev, struct scatterlist *sg, + int nents, int direction); + void (*unmap_sg)(struct device *hwdev, + struct scatterlist *sg, int nents, + int direction); + int (*dma_supported)(struct device *hwdev, u64 mask); + int is_phys; +}; + +extern const struct dma_mapping_ops *dma_ops; + +static inline int dma_mapping_error(dma_addr_t dma_addr) +{ + if (dma_ops->mapping_error) + return dma_ops->mapping_error(dma_addr); + + return (dma_addr == bad_dma_address); +} + +#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) +#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) + +void *dma_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag); + +void dma_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle); + + +extern int dma_supported(struct device *hwdev, u64 mask); +extern int dma_set_mask(struct device *dev, u64 mask); + +static inline dma_addr_t +dma_map_single(struct device *hwdev, void *ptr, size_t size, + int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + return dma_ops->map_single(hwdev, virt_to_phys(ptr), size, direction); +} + +static inline void +dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size, + int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->unmap_single) + dma_ops->unmap_single(dev, addr, size, direction); +} + +static inline int +dma_map_sg(struct device *hwdev, struct scatterlist *sg, + int nents, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + return dma_ops->map_sg(hwdev, sg, nents, direction); +} + +static inline void +dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents, + int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->unmap_sg) + dma_ops->unmap_sg(hwdev, sg, nents, direction); +} + +static inline void +dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t dma_handle, + size_t size, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_single_for_cpu) + dma_ops->sync_single_for_cpu(hwdev, dma_handle, size, + direction); + flush_write_buffers(); +} + +static inline void +dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle, + size_t size, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_single_for_device) + dma_ops->sync_single_for_device(hwdev, dma_handle, size, + direction); + flush_write_buffers(); +} + +static inline void +dma_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dma_handle, + unsigned long offset, size_t size, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_single_range_for_cpu) + dma_ops->sync_single_range_for_cpu(hwdev, dma_handle, offset, + size, direction); + + flush_write_buffers(); +} + +static inline void +dma_sync_single_range_for_device(struct device *hwdev, dma_addr_t dma_handle, + unsigned long offset, size_t size, + int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_single_range_for_device) + dma_ops->sync_single_range_for_device(hwdev, dma_handle, + offset, size, direction); + + flush_write_buffers(); +} + +static inline void +dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, + int nelems, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_sg_for_cpu) + dma_ops->sync_sg_for_cpu(hwdev, sg, nelems, direction); + flush_write_buffers(); +} + +static inline void +dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, + int nelems, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_sg_for_device) + dma_ops->sync_sg_for_device(hwdev, sg, nelems, direction); + + flush_write_buffers(); +} + +static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, + size_t offset, size_t size, + int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + return dma_ops->map_single(dev, page_to_phys(page)+offset, + size, direction); +} + +static inline void dma_unmap_page(struct device *dev, dma_addr_t addr, + size_t size, int direction) +{ + dma_unmap_single(dev, addr, size, direction); +} + +static inline void +dma_cache_sync(struct device *dev, void *vaddr, size_t size, + enum dma_data_direction dir) +{ + flush_write_buffers(); +} + +static inline int dma_get_cache_alignment(void) +{ + /* no easy way to get cache size on all x86, so return the + * maximum possible, to be safe */ + return boot_cpu_data.x86_clflush_size; +} + +#define dma_is_consistent(d, h) (1) + #ifdef CONFIG_X86_32 -# include "dma-mapping_32.h" -#else -# include "dma-mapping_64.h" +# define ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY +struct dma_coherent_mem { + void *virt_base; + u32 device_base; + int size; + int flags; + unsigned long *bitmap; +}; + +extern int +dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, + dma_addr_t device_addr, size_t size, int flags); + +extern void +dma_release_declared_memory(struct device *dev); + +extern void * +dma_mark_declared_memory_occupied(struct device *dev, + dma_addr_t device_addr, size_t size); +#endif /* CONFIG_X86_32 */ #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h deleted file mode 100644 index 55f01bd9e55..00000000000 --- a/include/asm-x86/dma-mapping_32.h +++ /dev/null @@ -1,187 +0,0 @@ -#ifndef _ASM_I386_DMA_MAPPING_H -#define _ASM_I386_DMA_MAPPING_H - -#include <linux/mm.h> -#include <linux/scatterlist.h> - -#include <asm/cache.h> -#include <asm/io.h> -#include <asm/bug.h> - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - -void *dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag); - -void dma_free_coherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle); - -static inline dma_addr_t -dma_map_single(struct device *dev, void *ptr, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(!valid_dma_direction(direction)); - WARN_ON(size == 0); - flush_write_buffers(); - return virt_to_phys(ptr); -} - -static inline void -dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(!valid_dma_direction(direction)); -} - -static inline int -dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents, - enum dma_data_direction direction) -{ - struct scatterlist *sg; - int i; - - BUG_ON(!valid_dma_direction(direction)); - WARN_ON(nents == 0 || sglist[0].length == 0); - - for_each_sg(sglist, sg, nents, i) { - BUG_ON(!sg_page(sg)); - - sg->dma_address = sg_phys(sg); - } - - flush_write_buffers(); - return nents; -} - -static inline dma_addr_t -dma_map_page(struct device *dev, struct page *page, unsigned long offset, - size_t size, enum dma_data_direction direction) -{ - BUG_ON(!valid_dma_direction(direction)); - return page_to_phys(page) + offset; -} - -static inline void -dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(!valid_dma_direction(direction)); -} - - -static inline void -dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries, - enum dma_data_direction direction) -{ - BUG_ON(!valid_dma_direction(direction)); -} - -static inline void -dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size, - enum dma_data_direction direction) -{ -} - -static inline void -dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size, - enum dma_data_direction direction) -{ - flush_write_buffers(); -} - -static inline void -dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle, - unsigned long offset, size_t size, - enum dma_data_direction direction) -{ -} - -static inline void -dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle, - unsigned long offset, size_t size, - enum dma_data_direction direction) -{ - flush_write_buffers(); -} - -static inline void -dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, - enum dma_data_direction direction) -{ -} - -static inline void -dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems, - enum dma_data_direction direction) -{ - flush_write_buffers(); -} - -static inline int -dma_mapping_error(dma_addr_t dma_addr) -{ - return 0; -} - -extern int forbid_dac; - -static inline int -dma_supported(struct device *dev, u64 mask) -{ - /* - * we fall back to GFP_DMA when the mask isn't all 1s, - * so we can't guarantee allocations that must be - * within a tighter range than GFP_DMA.. - */ - if(mask < 0x00ffffff) - return 0; - - /* Work around chipset bugs */ - if (forbid_dac > 0 && mask > 0xffffffffULL) - return 0; - - return 1; -} - -static inline int -dma_set_mask(struct device *dev, u64 mask) -{ - if(!dev->dma_mask || !dma_supported(dev, mask)) - return -EIO; - - *dev->dma_mask = mask; - - return 0; -} - -static inline int -dma_get_cache_alignment(void) -{ - /* no easy way to get cache size on all x86, so return the - * maximum possible, to be safe */ - return (1 << INTERNODE_CACHE_SHIFT); -} - -#define dma_is_consistent(d, h) (1) - -static inline void -dma_cache_sync(struct device *dev, void *vaddr, size_t size, - enum dma_data_direction direction) -{ - flush_write_buffers(); -} - -#define ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY -extern int -dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, - dma_addr_t device_addr, size_t size, int flags); - -extern void -dma_release_declared_memory(struct device *dev); - -extern void * -dma_mark_declared_memory_occupied(struct device *dev, - dma_addr_t device_addr, size_t size); - -#endif diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h deleted file mode 100644 index ecd0f6125ba..00000000000 --- a/include/asm-x86/dma-mapping_64.h +++ /dev/null @@ -1,202 +0,0 @@ -#ifndef _X8664_DMA_MAPPING_H -#define _X8664_DMA_MAPPING_H 1 - -/* - * IOMMU interface. See Documentation/DMA-mapping.txt and DMA-API.txt for - * documentation. - */ - -#include <linux/scatterlist.h> -#include <asm/io.h> -#include <asm/swiotlb.h> - -struct dma_mapping_ops { - int (*mapping_error)(dma_addr_t dma_addr); - void* (*alloc_coherent)(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp); - void (*free_coherent)(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle); - dma_addr_t (*map_single)(struct device *hwdev, void *ptr, - size_t size, int direction); - /* like map_single, but doesn't check the device mask */ - dma_addr_t (*map_simple)(struct device *hwdev, char *ptr, - size_t size, int direction); - void (*unmap_single)(struct device *dev, dma_addr_t addr, - size_t size, int direction); - void (*sync_single_for_cpu)(struct device *hwdev, - dma_addr_t dma_handle, size_t size, - int direction); - void (*sync_single_for_device)(struct device *hwdev, - dma_addr_t dma_handle, size_t size, - int direction); - void (*sync_single_range_for_cpu)(struct device *hwdev, - dma_addr_t dma_handle, unsigned long offset, - size_t size, int direction); - void (*sync_single_range_for_device)(struct device *hwdev, - dma_addr_t dma_handle, unsigned long offset, - size_t size, int direction); - void (*sync_sg_for_cpu)(struct device *hwdev, - struct scatterlist *sg, int nelems, - int direction); - void (*sync_sg_for_device)(struct device *hwdev, - struct scatterlist *sg, int nelems, - int direction); - int (*map_sg)(struct device *hwdev, struct scatterlist *sg, - int nents, int direction); - void (*unmap_sg)(struct device *hwdev, - struct scatterlist *sg, int nents, - int direction); - int (*dma_supported)(struct device *hwdev, u64 mask); - int is_phys; -}; - -extern dma_addr_t bad_dma_address; -extern const struct dma_mapping_ops* dma_ops; -extern int iommu_merge; - -static inline int dma_mapping_error(dma_addr_t dma_addr) -{ - if (dma_ops->mapping_error) - return dma_ops->mapping_error(dma_addr); - - return (dma_addr == bad_dma_address); -} - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - -extern void *dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp); -extern void dma_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle); - -static inline dma_addr_t -dma_map_single(struct device *hwdev, void *ptr, size_t size, - int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - return dma_ops->map_single(hwdev, ptr, size, direction); -} - -static inline void -dma_unmap_single(struct device *dev, dma_addr_t addr,size_t size, - int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - dma_ops->unmap_single(dev, addr, size, direction); -} - -#define dma_map_page(dev,page,offset,size,dir) \ - dma_map_single((dev), page_address(page)+(offset), (size), (dir)) - -#define dma_unmap_page dma_unmap_single - -static inline void -dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t dma_handle, - size_t size, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_single_for_cpu) - dma_ops->sync_single_for_cpu(hwdev, dma_handle, size, - direction); - flush_write_buffers(); -} - -static inline void -dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle, - size_t size, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_single_for_device) - dma_ops->sync_single_for_device(hwdev, dma_handle, size, - direction); - flush_write_buffers(); -} - -static inline void -dma_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dma_handle, - unsigned long offset, size_t size, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_single_range_for_cpu) { - dma_ops->sync_single_range_for_cpu(hwdev, dma_handle, offset, size, direction); - } - - flush_write_buffers(); -} - -static inline void -dma_sync_single_range_for_device(struct device *hwdev, dma_addr_t dma_handle, - unsigned long offset, size_t size, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_single_range_for_device) - dma_ops->sync_single_range_for_device(hwdev, dma_handle, - offset, size, direction); - - flush_write_buffers(); -} - -static inline void -dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, - int nelems, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_sg_for_cpu) - dma_ops->sync_sg_for_cpu(hwdev, sg, nelems, direction); - flush_write_buffers(); -} - -static inline void -dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, - int nelems, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_sg_for_device) { - dma_ops->sync_sg_for_device(hwdev, sg, nelems, direction); - } - - flush_write_buffers(); -} - -static inline int -dma_map_sg(struct device *hwdev, struct scatterlist *sg, int nents, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - return dma_ops->map_sg(hwdev, sg, nents, direction); -} - -static inline void -dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents, - int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - dma_ops->unmap_sg(hwdev, sg, nents, direction); -} - -extern int dma_supported(struct device *hwdev, u64 mask); - -/* same for gart, swiotlb, and nommu */ -static inline int dma_get_cache_alignment(void) -{ - return boot_cpu_data.x86_clflush_size; -} - -#define dma_is_consistent(d, h) 1 - -extern int dma_set_mask(struct device *dev, u64 mask); - -static inline void -dma_cache_sync(struct device *dev, void *vaddr, size_t size, - enum dma_data_direction dir) -{ - flush_write_buffers(); -} - -extern struct device fallback_dev; -extern int panic_on_overflow; - -#endif /* _X8664_DMA_MAPPING_H */ diff --git a/include/asm-x86/dma.h b/include/asm-x86/dma.h index e9733ce8988..ca1098a7e58 100644 --- a/include/asm-x86/dma.h +++ b/include/asm-x86/dma.h @@ -12,7 +12,6 @@ #include <asm/io.h> /* need byte IO */ #include <linux/delay.h> - #ifdef HAVE_REALLY_SLOW_DMA_CONTROLLER #define dma_outb outb_p #else @@ -74,15 +73,15 @@ #ifdef CONFIG_X86_32 /* The maximum address that we can perform a DMA transfer to on this platform */ -#define MAX_DMA_ADDRESS (PAGE_OFFSET+0x1000000) +#define MAX_DMA_ADDRESS (PAGE_OFFSET + 0x1000000) #else /* 16MB ISA DMA zone */ -#define MAX_DMA_PFN ((16*1024*1024) >> PAGE_SHIFT) +#define MAX_DMA_PFN ((16 * 1024 * 1024) >> PAGE_SHIFT) /* 4GB broken PCI/AGP hardware bus master zone */ -#define MAX_DMA32_PFN ((4UL*1024*1024*1024) >> PAGE_SHIFT) +#define MAX_DMA32_PFN ((4UL * 1024 * 1024 * 1024) >> PAGE_SHIFT) /* Compat define for old dma zone */ #define MAX_DMA_ADDRESS ((unsigned long)__va(MAX_DMA_PFN << PAGE_SHIFT)) @@ -154,20 +153,20 @@ extern spinlock_t dma_spin_lock; -static __inline__ unsigned long claim_dma_lock(void) +static inline unsigned long claim_dma_lock(void) { unsigned long flags; spin_lock_irqsave(&dma_spin_lock, flags); return flags; } -static __inline__ void release_dma_lock(unsigned long flags) +static inline void release_dma_lock(unsigned long flags) { spin_unlock_irqrestore(&dma_spin_lock, flags); } /* enable/disable a specific DMA channel */ -static __inline__ void enable_dma(unsigned int dmanr) +static inline void enable_dma(unsigned int dmanr) { if (dmanr <= 3) dma_outb(dmanr, DMA1_MASK_REG); @@ -175,7 +174,7 @@ static __inline__ void enable_dma(unsigned int dmanr) dma_outb(dmanr & 3, DMA2_MASK_REG); } -static __inline__ void disable_dma(unsigned int dmanr) +static inline void disable_dma(unsigned int dmanr) { if (dmanr <= 3) dma_outb(dmanr | 4, DMA1_MASK_REG); @@ -190,7 +189,7 @@ static __inline__ void disable_dma(unsigned int dmanr) * --- In order to do that, the DMA routines below should --- * --- only be used while holding the DMA lock ! --- */ -static __inline__ void clear_dma_ff(unsigned int dmanr) +static inline void clear_dma_ff(unsigned int dmanr) { if (dmanr <= 3) dma_outb(0, DMA1_CLEAR_FF_REG); @@ -199,7 +198,7 @@ static __inline__ void clear_dma_ff(unsigned int dmanr) } /* set mode (above) for a specific DMA channel */ -static __inline__ void set_dma_mode(unsigned int dmanr, char mode) +static inline void set_dma_mode(unsigned int dmanr, char mode) { if (dmanr <= 3) dma_outb(mode | dmanr, DMA1_MODE_REG); @@ -212,7 +211,7 @@ static __inline__ void set_dma_mode(unsigned int dmanr, char mode) * the lower 16 bits of the DMA current address register, but a 64k boundary * may have been crossed. */ -static __inline__ void set_dma_page(unsigned int dmanr, char pagenr) +static inline void set_dma_page(unsigned int dmanr, char pagenr) { switch (dmanr) { case 0: @@ -243,15 +242,15 @@ static __inline__ void set_dma_page(unsigned int dmanr, char pagenr) /* Set transfer address & page bits for specific DMA channel. * Assumes dma flipflop is clear. */ -static __inline__ void set_dma_addr(unsigned int dmanr, unsigned int a) +static inline void set_dma_addr(unsigned int dmanr, unsigned int a) { set_dma_page(dmanr, a>>16); if (dmanr <= 3) { dma_outb(a & 0xff, ((dmanr & 3) << 1) + IO_DMA1_BASE); dma_outb((a >> 8) & 0xff, ((dmanr & 3) << 1) + IO_DMA1_BASE); } else { - dma_outb((a >> 1) & 0xff, ((dmanr & 3) << 2) + IO_DMA2_BASE); - dma_outb((a >> 9) & 0xff, ((dmanr & 3) << 2) + IO_DMA2_BASE); + dma_outb((a >> 1) & 0xff, ((dmanr & 3) << 2) + IO_DMA2_BASE); + dma_outb((a >> 9) & 0xff, ((dmanr & 3) << 2) + IO_DMA2_BASE); } } @@ -264,18 +263,18 @@ static __inline__ void set_dma_addr(unsigned int dmanr, unsigned int a) * Assumes dma flip-flop is clear. * NOTE 2: "count" represents _bytes_ and must be even for channels 5-7. */ -static __inline__ void set_dma_count(unsigned int dmanr, unsigned int count) +static inline void set_dma_count(unsigned int dmanr, unsigned int count) { count--; if (dmanr <= 3) { - dma_outb(count & 0xff, ((dmanr & 3) << 1) + 1 + IO_DMA1_BASE); - dma_outb((count >> 8) & 0xff, - ((dmanr & 3) << 1) + 1 + IO_DMA1_BASE); + dma_outb(count & 0xff, ((dmanr & 3) << 1) + 1 + IO_DMA1_BASE); + dma_outb((count >> 8) & 0xff, + ((dmanr & 3) << 1) + 1 + IO_DMA1_BASE); } else { - dma_outb((count >> 1) & 0xff, - ((dmanr & 3) << 2) + 2 + IO_DMA2_BASE); - dma_outb((count >> 9) & 0xff, - ((dmanr & 3) << 2) + 2 + IO_DMA2_BASE); + dma_outb((count >> 1) & 0xff, + ((dmanr & 3) << 2) + 2 + IO_DMA2_BASE); + dma_outb((count >> 9) & 0xff, + ((dmanr & 3) << 2) + 2 + IO_DMA2_BASE); } } @@ -288,7 +287,7 @@ static __inline__ void set_dma_count(unsigned int dmanr, unsigned int count) * * Assumes DMA flip-flop is clear. */ -static __inline__ int get_dma_residue(unsigned int dmanr) +static inline int get_dma_residue(unsigned int dmanr) { unsigned int io_port; /* using short to get 16-bit wrap around */ diff --git a/include/asm-x86/dmi.h b/include/asm-x86/dmi.h index 1241e6ad193..4edf7514a75 100644 --- a/include/asm-x86/dmi.h +++ b/include/asm-x86/dmi.h @@ -27,6 +27,7 @@ static inline void *dmi_alloc(unsigned len) #endif +/* Use early IO mappings for DMI because it's initialized early */ #define dmi_ioremap early_ioremap #define dmi_iounmap early_iounmap diff --git a/include/asm-x86/dwarf2_64.h b/include/asm-x86/dwarf2_64.h index eedc08526b0..c950519a264 100644 --- a/include/asm-x86/dwarf2_64.h +++ b/include/asm-x86/dwarf2_64.h @@ -1,16 +1,15 @@ #ifndef _DWARF2_H #define _DWARF2_H 1 - #ifndef __ASSEMBLY__ #warning "asm/dwarf2.h should be only included in pure assembly files" #endif -/* +/* Macros for dwarf2 CFI unwind table entries. - See "as.info" for details on these pseudo ops. Unfortunately - they are only supported in very new binutils, so define them - away for older version. + See "as.info" for details on these pseudo ops. Unfortunately + they are only supported in very new binutils, so define them + away for older version. */ #ifdef CONFIG_AS_CFI diff --git a/include/asm-x86/e820_32.h b/include/asm-x86/e820_32.h index e7207a6de3e..a9f7c6ec32b 100644 --- a/include/asm-x86/e820_32.h +++ b/include/asm-x86/e820_32.h @@ -24,7 +24,7 @@ extern void update_e820(void); extern int e820_all_mapped(unsigned long start, unsigned long end, unsigned type); extern int e820_any_mapped(u64 start, u64 end, unsigned type); -extern void find_max_pfn(void); +extern void propagate_e820_map(void); extern void register_bootmem_low_pages(unsigned long max_low_pfn); extern void add_memory_region(unsigned long long start, unsigned long long size, int type); @@ -34,8 +34,8 @@ extern void e820_register_memory(void); extern void limit_regions(unsigned long long size); extern void print_memory_map(char *who); extern void init_iomem_resources(struct resource *code_resource, - struct resource *data_resource, - struct resource *bss_resource); + struct resource *data_resource, + struct resource *bss_resource); #if defined(CONFIG_PM) && defined(CONFIG_HIBERNATION) extern void e820_mark_nosave_regions(void); diff --git a/include/asm-x86/e820_64.h b/include/asm-x86/e820_64.h index 22ede73ae72..71c4d685d30 100644 --- a/include/asm-x86/e820_64.h +++ b/include/asm-x86/e820_64.h @@ -14,20 +14,24 @@ #include <linux/ioport.h> #ifndef __ASSEMBLY__ -extern unsigned long find_e820_area(unsigned long start, unsigned long end, - unsigned size, unsigned long align); -extern void add_memory_region(unsigned long start, unsigned long size, +extern unsigned long find_e820_area(unsigned long start, unsigned long end, + unsigned long size, unsigned long align); +extern unsigned long find_e820_area_size(unsigned long start, + unsigned long *sizep, + unsigned long align); +extern void add_memory_region(unsigned long start, unsigned long size, int type); extern void update_memory_range(u64 start, u64 size, unsigned old_type, unsigned new_type); extern void setup_memory_region(void); -extern void contig_e820_setup(void); +extern void contig_e820_setup(void); extern unsigned long e820_end_of_ram(void); -extern void e820_reserve_resources(struct resource *code_resource, - struct resource *data_resource, struct resource *bss_resource); +extern void e820_reserve_resources(void); extern void e820_mark_nosave_regions(void); -extern int e820_any_mapped(unsigned long start, unsigned long end, unsigned type); -extern int e820_all_mapped(unsigned long start, unsigned long end, unsigned type); +extern int e820_any_mapped(unsigned long start, unsigned long end, + unsigned type); +extern int e820_all_mapped(unsigned long start, unsigned long end, + unsigned type); extern int e820_any_non_reserved(unsigned long start, unsigned long end); extern int is_memory_any_valid(unsigned long start, unsigned long end); extern int e820_all_non_reserved(unsigned long start, unsigned long end); @@ -35,8 +39,8 @@ extern int is_memory_all_valid(unsigned long start, unsigned long end); extern unsigned long e820_hole_size(unsigned long start, unsigned long end); extern void e820_setup_gap(void); -extern void e820_register_active_regions(int nid, - unsigned long start_pfn, unsigned long end_pfn); +extern void e820_register_active_regions(int nid, unsigned long start_pfn, + unsigned long end_pfn); extern void finish_e820_parsing(void); @@ -44,7 +48,8 @@ extern struct e820map e820; extern void update_e820(void); extern void reserve_early(unsigned long start, unsigned long end, char *name); -extern void early_res_to_bootmem(void); +extern void free_early(unsigned long start, unsigned long end); +extern void early_res_to_bootmem(unsigned long start, unsigned long end); #endif/*!__ASSEMBLY__*/ diff --git a/include/asm-x86/edac.h b/include/asm-x86/edac.h index cf3200a745a..a8088f63a30 100644 --- a/include/asm-x86/edac.h +++ b/include/asm-x86/edac.h @@ -3,7 +3,7 @@ /* ECC atomic, DMA, SMP and interrupt safe scrub function */ -static __inline__ void atomic_scrub(void *va, u32 size) +static inline void atomic_scrub(void *va, u32 size) { u32 i, *virt_addr = va; @@ -12,7 +12,7 @@ static __inline__ void atomic_scrub(void *va, u32 size) * are interrupt, DMA and SMP safe. */ for (i = 0; i < size / 4; i++, virt_addr++) - __asm__ __volatile__("lock; addl $0, %0"::"m"(*virt_addr)); + asm volatile("lock; addl $0, %0"::"m" (*virt_addr)); } #endif diff --git a/include/asm-x86/efi.h b/include/asm-x86/efi.h index ea9734b74ac..d53004b855c 100644 --- a/include/asm-x86/efi.h +++ b/include/asm-x86/efi.h @@ -20,7 +20,7 @@ extern unsigned long asmlinkage efi_call_phys(void *, ...); */ #define efi_call_virt(f, args...) \ - ((efi_##f##_t __attribute__((regparm(0)))*)efi.systab->runtime->f)(args) + ((efi_##f##_t __attribute__((regparm(0)))*)efi.systab->runtime->f)(args) #define efi_call_virt0(f) efi_call_virt(f) #define efi_call_virt1(f, a1) efi_call_virt(f, a1) diff --git a/include/asm-x86/elf.h b/include/asm-x86/elf.h index fb62f9941e3..8f232dc5b5f 100644 --- a/include/asm-x86/elf.h +++ b/include/asm-x86/elf.h @@ -11,7 +11,7 @@ typedef unsigned long elf_greg_t; -#define ELF_NGREG (sizeof (struct user_regs_struct) / sizeof(elf_greg_t)) +#define ELF_NGREG (sizeof(struct user_regs_struct) / sizeof(elf_greg_t)) typedef elf_greg_t elf_gregset_t[ELF_NGREG]; typedef struct user_i387_struct elf_fpregset_t; @@ -82,8 +82,9 @@ extern unsigned int vdso_enabled; #define elf_check_arch_ia32(x) \ (((x)->e_machine == EM_386) || ((x)->e_machine == EM_486)) -#ifdef CONFIG_X86_32 #include <asm/processor.h> + +#ifdef CONFIG_X86_32 #include <asm/system.h> /* for savesegment */ #include <asm/desc.h> @@ -99,10 +100,11 @@ extern unsigned int vdso_enabled; We might as well make sure everything else is cleared too (except for %esp), just to make things more deterministic. */ -#define ELF_PLAT_INIT(_r, load_addr) do { \ - _r->bx = 0; _r->cx = 0; _r->dx = 0; \ - _r->si = 0; _r->di = 0; _r->bp = 0; \ - _r->ax = 0; \ +#define ELF_PLAT_INIT(_r, load_addr) \ + do { \ + _r->bx = 0; _r->cx = 0; _r->dx = 0; \ + _r->si = 0; _r->di = 0; _r->bp = 0; \ + _r->ax = 0; \ } while (0) /* @@ -110,24 +112,25 @@ extern unsigned int vdso_enabled; * now struct_user_regs, they are different) */ -#define ELF_CORE_COPY_REGS(pr_reg, regs) do { \ - pr_reg[0] = regs->bx; \ - pr_reg[1] = regs->cx; \ - pr_reg[2] = regs->dx; \ - pr_reg[3] = regs->si; \ - pr_reg[4] = regs->di; \ - pr_reg[5] = regs->bp; \ - pr_reg[6] = regs->ax; \ - pr_reg[7] = regs->ds & 0xffff; \ - pr_reg[8] = regs->es & 0xffff; \ - pr_reg[9] = regs->fs & 0xffff; \ - savesegment(gs, pr_reg[10]); \ - pr_reg[11] = regs->orig_ax; \ - pr_reg[12] = regs->ip; \ - pr_reg[13] = regs->cs & 0xffff; \ - pr_reg[14] = regs->flags; \ - pr_reg[15] = regs->sp; \ - pr_reg[16] = regs->ss & 0xffff; \ +#define ELF_CORE_COPY_REGS(pr_reg, regs) \ +do { \ + pr_reg[0] = regs->bx; \ + pr_reg[1] = regs->cx; \ + pr_reg[2] = regs->dx; \ + pr_reg[3] = regs->si; \ + pr_reg[4] = regs->di; \ + pr_reg[5] = regs->bp; \ + pr_reg[6] = regs->ax; \ + pr_reg[7] = regs->ds & 0xffff; \ + pr_reg[8] = regs->es & 0xffff; \ + pr_reg[9] = regs->fs & 0xffff; \ + savesegment(gs, pr_reg[10]); \ + pr_reg[11] = regs->orig_ax; \ + pr_reg[12] = regs->ip; \ + pr_reg[13] = regs->cs & 0xffff; \ + pr_reg[14] = regs->flags; \ + pr_reg[15] = regs->sp; \ + pr_reg[16] = regs->ss & 0xffff; \ } while (0); #define ELF_PLATFORM (utsname()->machine) @@ -135,12 +138,10 @@ extern unsigned int vdso_enabled; #else /* CONFIG_X86_32 */ -#include <asm/processor.h> - /* * This is used to ensure we don't load something for the wrong architecture. */ -#define elf_check_arch(x) \ +#define elf_check_arch(x) \ ((x)->e_machine == EM_X86_64) #define compat_elf_check_arch(x) elf_check_arch_ia32(x) @@ -169,24 +170,30 @@ static inline void elf_common_init(struct thread_struct *t, t->ds = t->es = ds; } -#define ELF_PLAT_INIT(_r, load_addr) do { \ - elf_common_init(¤t->thread, _r, 0); \ - clear_thread_flag(TIF_IA32); \ +#define ELF_PLAT_INIT(_r, load_addr) \ +do { \ + elf_common_init(¤t->thread, _r, 0); \ + clear_thread_flag(TIF_IA32); \ } while (0) -#define COMPAT_ELF_PLAT_INIT(regs, load_addr) \ +#define COMPAT_ELF_PLAT_INIT(regs, load_addr) \ elf_common_init(¤t->thread, regs, __USER_DS) -#define compat_start_thread(regs, ip, sp) do { \ - start_ia32_thread(regs, ip, sp); \ - set_fs(USER_DS); \ - } while (0) -#define COMPAT_SET_PERSONALITY(ex, ibcs2) do { \ - if (test_thread_flag(TIF_IA32)) \ - clear_thread_flag(TIF_ABI_PENDING); \ - else \ - set_thread_flag(TIF_ABI_PENDING); \ - current->personality |= force_personality32; \ - } while (0) + +#define compat_start_thread(regs, ip, sp) \ +do { \ + start_ia32_thread(regs, ip, sp); \ + set_fs(USER_DS); \ +} while (0) + +#define COMPAT_SET_PERSONALITY(ex, ibcs2) \ +do { \ + if (test_thread_flag(TIF_IA32)) \ + clear_thread_flag(TIF_ABI_PENDING); \ + else \ + set_thread_flag(TIF_ABI_PENDING); \ + current->personality |= force_personality32; \ +} while (0) + #define COMPAT_ELF_PLATFORM ("i686") /* @@ -195,7 +202,8 @@ static inline void elf_common_init(struct thread_struct *t, * getting dumped. */ -#define ELF_CORE_COPY_REGS(pr_reg, regs) do { \ +#define ELF_CORE_COPY_REGS(pr_reg, regs) \ +do { \ unsigned v; \ (pr_reg)[0] = (regs)->r15; \ (pr_reg)[1] = (regs)->r14; \ @@ -269,10 +277,12 @@ extern int force_personality32; struct task_struct; -#define ARCH_DLINFO_IA32(vdso_enabled) \ -do if (vdso_enabled) { \ +#define ARCH_DLINFO_IA32(vdso_enabled) \ +do { \ + if (vdso_enabled) { \ NEW_AUX_ENT(AT_SYSINFO, VDSO_ENTRY); \ NEW_AUX_ENT(AT_SYSINFO_EHDR, VDSO_CURRENT_BASE); \ + } \ } while (0) #ifdef CONFIG_X86_32 @@ -290,9 +300,11 @@ do if (vdso_enabled) { \ /* 1GB for 64bit, 8MB for 32bit */ #define STACK_RND_MASK (test_thread_flag(TIF_IA32) ? 0x7ff : 0x3fffff) -#define ARCH_DLINFO \ -do if (vdso_enabled) { \ - NEW_AUX_ENT(AT_SYSINFO_EHDR,(unsigned long)current->mm->context.vdso);\ +#define ARCH_DLINFO \ +do { \ + if (vdso_enabled) \ + NEW_AUX_ENT(AT_SYSINFO_EHDR, \ + (unsigned long)current->mm->context.vdso); \ } while (0) #define AT_SYSINFO 32 @@ -305,8 +317,8 @@ do if (vdso_enabled) { \ #define VDSO_CURRENT_BASE ((unsigned long)current->mm->context.vdso) -#define VDSO_ENTRY \ - ((unsigned long) VDSO32_SYMBOL(VDSO_CURRENT_BASE, vsyscall)) +#define VDSO_ENTRY \ + ((unsigned long)VDSO32_SYMBOL(VDSO_CURRENT_BASE, vsyscall)) struct linux_binprm; diff --git a/include/asm-x86/fixmap.h b/include/asm-x86/fixmap.h index 382eb271a89..5bd206973dc 100644 --- a/include/asm-x86/fixmap.h +++ b/include/asm-x86/fixmap.h @@ -1,5 +1,13 @@ +#ifndef _ASM_FIXMAP_H +#define _ASM_FIXMAP_H + #ifdef CONFIG_X86_32 # include "fixmap_32.h" #else # include "fixmap_64.h" #endif + +#define clear_fixmap(idx) \ + __set_fixmap(idx, 0, __pgprot(0)) + +#endif diff --git a/include/asm-x86/fixmap_32.h b/include/asm-x86/fixmap_32.h index a7404d50686..4b96148e90c 100644 --- a/include/asm-x86/fixmap_32.h +++ b/include/asm-x86/fixmap_32.h @@ -10,8 +10,8 @@ * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 */ -#ifndef _ASM_FIXMAP_H -#define _ASM_FIXMAP_H +#ifndef _ASM_FIXMAP_32_H +#define _ASM_FIXMAP_32_H /* used by vmalloc.c, vsyscall.lds.S. @@ -99,8 +99,7 @@ enum fixed_addresses { */ #define NR_FIX_BTMAPS 64 #define FIX_BTMAPS_NESTING 4 - FIX_BTMAP_END = - __end_of_permanent_fixed_addresses + 512 - + FIX_BTMAP_END = __end_of_permanent_fixed_addresses + 512 - (__end_of_permanent_fixed_addresses & 511), FIX_BTMAP_BEGIN = FIX_BTMAP_END + NR_FIX_BTMAPS*FIX_BTMAPS_NESTING - 1, FIX_WP_TEST, @@ -110,20 +109,17 @@ enum fixed_addresses { __end_of_fixed_addresses }; -extern void __set_fixmap (enum fixed_addresses idx, - unsigned long phys, pgprot_t flags); +extern void __set_fixmap(enum fixed_addresses idx, + unsigned long phys, pgprot_t flags); extern void reserve_top_address(unsigned long reserve); -#define set_fixmap(idx, phys) \ - __set_fixmap(idx, phys, PAGE_KERNEL) +#define set_fixmap(idx, phys) \ + __set_fixmap(idx, phys, PAGE_KERNEL) /* * Some hardware wants to get fixmapped without caching. */ -#define set_fixmap_nocache(idx, phys) \ - __set_fixmap(idx, phys, PAGE_KERNEL_NOCACHE) - -#define clear_fixmap(idx) \ - __set_fixmap(idx, 0, __pgprot(0)) +#define set_fixmap_nocache(idx, phys) \ + __set_fixmap(idx, phys, PAGE_KERNEL_NOCACHE) #define FIXADDR_TOP ((unsigned long)__FIXADDR_TOP) @@ -156,7 +152,7 @@ static __always_inline unsigned long fix_to_virt(const unsigned int idx) if (idx >= __end_of_fixed_addresses) __this_fixmap_does_not_exist(); - return __fix_to_virt(idx); + return __fix_to_virt(idx); } static inline unsigned long virt_to_fix(const unsigned long vaddr) diff --git a/include/asm-x86/fixmap_64.h b/include/asm-x86/fixmap_64.h index 70ddb21e645..355d26a75a8 100644 --- a/include/asm-x86/fixmap_64.h +++ b/include/asm-x86/fixmap_64.h @@ -8,8 +8,8 @@ * Copyright (C) 1998 Ingo Molnar */ -#ifndef _ASM_FIXMAP_H -#define _ASM_FIXMAP_H +#ifndef _ASM_FIXMAP_64_H +#define _ASM_FIXMAP_64_H #include <linux/kernel.h> #include <asm/apicdef.h> @@ -34,32 +34,34 @@ enum fixed_addresses { VSYSCALL_LAST_PAGE, - VSYSCALL_FIRST_PAGE = VSYSCALL_LAST_PAGE + ((VSYSCALL_END-VSYSCALL_START) >> PAGE_SHIFT) - 1, + VSYSCALL_FIRST_PAGE = VSYSCALL_LAST_PAGE + + ((VSYSCALL_END-VSYSCALL_START) >> PAGE_SHIFT) - 1, VSYSCALL_HPET, FIX_DBGP_BASE, FIX_EARLYCON_MEM_BASE, FIX_HPET_BASE, FIX_APIC_BASE, /* local (CPU) APIC) -- required for SMP or not */ FIX_IO_APIC_BASE_0, - FIX_IO_APIC_BASE_END = FIX_IO_APIC_BASE_0 + MAX_IO_APICS-1, + FIX_IO_APIC_BASE_END = FIX_IO_APIC_BASE_0 + MAX_IO_APICS - 1, FIX_EFI_IO_MAP_LAST_PAGE, - FIX_EFI_IO_MAP_FIRST_PAGE = FIX_EFI_IO_MAP_LAST_PAGE+MAX_EFI_IO_PAGES-1, + FIX_EFI_IO_MAP_FIRST_PAGE = FIX_EFI_IO_MAP_LAST_PAGE + + MAX_EFI_IO_PAGES - 1, #ifdef CONFIG_PROVIDE_OHCI1394_DMA_INIT FIX_OHCI1394_BASE, #endif __end_of_fixed_addresses }; -extern void __set_fixmap (enum fixed_addresses idx, - unsigned long phys, pgprot_t flags); +extern void __set_fixmap(enum fixed_addresses idx, + unsigned long phys, pgprot_t flags); -#define set_fixmap(idx, phys) \ - __set_fixmap(idx, phys, PAGE_KERNEL) +#define set_fixmap(idx, phys) \ + __set_fixmap(idx, phys, PAGE_KERNEL) /* * Some hardware wants to get fixmapped without caching. */ -#define set_fixmap_nocache(idx, phys) \ - __set_fixmap(idx, phys, PAGE_KERNEL_NOCACHE) +#define set_fixmap_nocache(idx, phys) \ + __set_fixmap(idx, phys, PAGE_KERNEL_NOCACHE) #define FIXADDR_TOP (VSYSCALL_END-PAGE_SIZE) #define FIXADDR_SIZE (__end_of_fixed_addresses << PAGE_SHIFT) diff --git a/include/asm-x86/floppy.h b/include/asm-x86/floppy.h index a48d7153c09..dbe82a5c5ea 100644 --- a/include/asm-x86/floppy.h +++ b/include/asm-x86/floppy.h @@ -20,20 +20,21 @@ * driver otherwise. It doesn't matter much for performance anyway, as most * floppy accesses go through the track buffer. */ -#define _CROSS_64KB(a,s,vdma) \ -(!(vdma) && ((unsigned long)(a)/K_64 != ((unsigned long)(a) + (s) - 1) / K_64)) +#define _CROSS_64KB(a, s, vdma) \ + (!(vdma) && \ + ((unsigned long)(a)/K_64 != ((unsigned long)(a) + (s) - 1) / K_64)) -#define CROSS_64KB(a,s) _CROSS_64KB(a,s,use_virtual_dma & 1) +#define CROSS_64KB(a, s) _CROSS_64KB(a, s, use_virtual_dma & 1) -#define SW fd_routine[use_virtual_dma&1] +#define SW fd_routine[use_virtual_dma & 1] #define CSW fd_routine[can_use_virtual_dma & 1] #define fd_inb(port) inb_p(port) -#define fd_outb(value,port) outb_p(value,port) +#define fd_outb(value, port) outb_p(value, port) -#define fd_request_dma() CSW._request_dma(FLOPPY_DMA,"floppy") +#define fd_request_dma() CSW._request_dma(FLOPPY_DMA, "floppy") #define fd_free_dma() CSW._free_dma(FLOPPY_DMA) #define fd_enable_irq() enable_irq(FLOPPY_IRQ) #define fd_disable_irq() disable_irq(FLOPPY_IRQ) @@ -52,64 +53,64 @@ static int doing_pdma; static irqreturn_t floppy_hardint(int irq, void *dev_id) { - register unsigned char st; + unsigned char st; #undef TRACE_FLPY_INT #ifdef TRACE_FLPY_INT - static int calls=0; - static int bytes=0; - static int dma_wait=0; + static int calls; + static int bytes; + static int dma_wait; #endif if (!doing_pdma) return floppy_interrupt(irq, dev_id); #ifdef TRACE_FLPY_INT - if(!calls) + if (!calls) bytes = virtual_dma_count; #endif { - register int lcount; - register char *lptr; + int lcount; + char *lptr; st = 1; - for(lcount=virtual_dma_count, lptr=virtual_dma_addr; - lcount; lcount--, lptr++) { - st=inb(virtual_dma_port+4) & 0xa0 ; - if(st != 0xa0) + for (lcount = virtual_dma_count, lptr = virtual_dma_addr; + lcount; lcount--, lptr++) { + st = inb(virtual_dma_port + 4) & 0xa0; + if (st != 0xa0) break; - if(virtual_dma_mode) - outb_p(*lptr, virtual_dma_port+5); + if (virtual_dma_mode) + outb_p(*lptr, virtual_dma_port + 5); else - *lptr = inb_p(virtual_dma_port+5); + *lptr = inb_p(virtual_dma_port + 5); } virtual_dma_count = lcount; virtual_dma_addr = lptr; - st = inb(virtual_dma_port+4); + st = inb(virtual_dma_port + 4); } #ifdef TRACE_FLPY_INT calls++; #endif - if(st == 0x20) + if (st == 0x20) return IRQ_HANDLED; - if(!(st & 0x20)) { + if (!(st & 0x20)) { virtual_dma_residue += virtual_dma_count; - virtual_dma_count=0; + virtual_dma_count = 0; #ifdef TRACE_FLPY_INT printk("count=%x, residue=%x calls=%d bytes=%d dma_wait=%d\n", virtual_dma_count, virtual_dma_residue, calls, bytes, dma_wait); calls = 0; - dma_wait=0; + dma_wait = 0; #endif doing_pdma = 0; floppy_interrupt(irq, dev_id); return IRQ_HANDLED; } #ifdef TRACE_FLPY_INT - if(!virtual_dma_count) + if (!virtual_dma_count) dma_wait++; #endif return IRQ_HANDLED; @@ -117,14 +118,14 @@ static irqreturn_t floppy_hardint(int irq, void *dev_id) static void fd_disable_dma(void) { - if(! (can_use_virtual_dma & 1)) + if (!(can_use_virtual_dma & 1)) disable_dma(FLOPPY_DMA); doing_pdma = 0; virtual_dma_residue += virtual_dma_count; - virtual_dma_count=0; + virtual_dma_count = 0; } -static int vdma_request_dma(unsigned int dmanr, const char * device_id) +static int vdma_request_dma(unsigned int dmanr, const char *device_id) { return 0; } @@ -142,7 +143,7 @@ static int vdma_get_dma_residue(unsigned int dummy) static int fd_request_irq(void) { - if(can_use_virtual_dma) + if (can_use_virtual_dma) return request_irq(FLOPPY_IRQ, floppy_hardint, IRQF_DISABLED, "floppy", NULL); else @@ -152,13 +153,13 @@ static int fd_request_irq(void) static unsigned long dma_mem_alloc(unsigned long size) { - return __get_dma_pages(GFP_KERNEL|__GFP_NORETRY,get_order(size)); + return __get_dma_pages(GFP_KERNEL|__GFP_NORETRY, get_order(size)); } static unsigned long vdma_mem_alloc(unsigned long size) { - return (unsigned long) vmalloc(size); + return (unsigned long)vmalloc(size); } @@ -166,7 +167,7 @@ static unsigned long vdma_mem_alloc(unsigned long size) static void _fd_dma_mem_free(unsigned long addr, unsigned long size) { - if((unsigned long) addr >= (unsigned long) high_memory) + if ((unsigned long)addr >= (unsigned long)high_memory) vfree((void *)addr); else free_pages(addr, get_order(size)); @@ -176,10 +177,10 @@ static void _fd_dma_mem_free(unsigned long addr, unsigned long size) static void _fd_chose_dma_mode(char *addr, unsigned long size) { - if(can_use_virtual_dma == 2) { - if((unsigned long) addr >= (unsigned long) high_memory || - isa_virt_to_bus(addr) >= 0x1000000 || - _CROSS_64KB(addr, size, 0)) + if (can_use_virtual_dma == 2) { + if ((unsigned long)addr >= (unsigned long)high_memory || + isa_virt_to_bus(addr) >= 0x1000000 || + _CROSS_64KB(addr, size, 0)) use_virtual_dma = 1; else use_virtual_dma = 0; @@ -195,7 +196,7 @@ static int vdma_dma_setup(char *addr, unsigned long size, int mode, int io) { doing_pdma = 1; virtual_dma_port = io; - virtual_dma_mode = (mode == DMA_MODE_WRITE); + virtual_dma_mode = (mode == DMA_MODE_WRITE); virtual_dma_addr = addr; virtual_dma_count = size; virtual_dma_residue = 0; @@ -213,18 +214,18 @@ static int hard_dma_setup(char *addr, unsigned long size, int mode, int io) /* actual, physical DMA */ doing_pdma = 0; clear_dma_ff(FLOPPY_DMA); - set_dma_mode(FLOPPY_DMA,mode); - set_dma_addr(FLOPPY_DMA,isa_virt_to_bus(addr)); - set_dma_count(FLOPPY_DMA,size); + set_dma_mode(FLOPPY_DMA, mode); + set_dma_addr(FLOPPY_DMA, isa_virt_to_bus(addr)); + set_dma_count(FLOPPY_DMA, size); enable_dma(FLOPPY_DMA); return 0; } static struct fd_routine_l { - int (*_request_dma)(unsigned int dmanr, const char * device_id); + int (*_request_dma)(unsigned int dmanr, const char *device_id); void (*_free_dma)(unsigned int dmanr); int (*_get_dma_residue)(unsigned int dummy); - unsigned long (*_dma_mem_alloc) (unsigned long size); + unsigned long (*_dma_mem_alloc)(unsigned long size); int (*_dma_setup)(char *addr, unsigned long size, int mode, int io); } fd_routine[] = { { @@ -252,7 +253,8 @@ static int FDC2 = -1; * is needed to prevent corrupted CMOS RAM in case "insmod floppy" * coincides with another rtc CMOS user. Paul G. */ -#define FLOPPY0_TYPE ({ \ +#define FLOPPY0_TYPE \ +({ \ unsigned long flags; \ unsigned char val; \ spin_lock_irqsave(&rtc_lock, flags); \ @@ -261,7 +263,8 @@ static int FDC2 = -1; val; \ }) -#define FLOPPY1_TYPE ({ \ +#define FLOPPY1_TYPE \ +({ \ unsigned long flags; \ unsigned char val; \ spin_lock_irqsave(&rtc_lock, flags); \ diff --git a/include/asm-x86/futex.h b/include/asm-x86/futex.h index c9952ea9f69..e7a76b37b33 100644 --- a/include/asm-x86/futex.h +++ b/include/asm-x86/futex.h @@ -4,43 +4,40 @@ #ifdef __KERNEL__ #include <linux/futex.h> +#include <linux/uaccess.h> #include <asm/asm.h> #include <asm/errno.h> #include <asm/processor.h> #include <asm/system.h> -#include <asm/uaccess.h> #define __futex_atomic_op1(insn, ret, oldval, uaddr, oparg) \ - __asm__ __volatile( \ -"1: " insn "\n" \ -"2: .section .fixup,\"ax\"\n \ -3: mov %3, %1\n \ - jmp 2b\n \ - .previous\n" \ - _ASM_EXTABLE(1b,3b) \ - : "=r" (oldval), "=r" (ret), "+m" (*uaddr) \ - : "i" (-EFAULT), "0" (oparg), "1" (0)) + asm volatile("1:\t" insn "\n" \ + "2:\t.section .fixup,\"ax\"\n" \ + "3:\tmov\t%3, %1\n" \ + "\tjmp\t2b\n" \ + "\t.previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r" (oldval), "=r" (ret), "+m" (*uaddr) \ + : "i" (-EFAULT), "0" (oparg), "1" (0)) #define __futex_atomic_op2(insn, ret, oldval, uaddr, oparg) \ - __asm__ __volatile( \ -"1: movl %2, %0\n \ - movl %0, %3\n" \ - insn "\n" \ -"2: lock; cmpxchgl %3, %2\n \ - jnz 1b\n \ -3: .section .fixup,\"ax\"\n \ -4: mov %5, %1\n \ - jmp 3b\n \ - .previous\n" \ - _ASM_EXTABLE(1b,4b) \ - _ASM_EXTABLE(2b,4b) \ - : "=&a" (oldval), "=&r" (ret), "+m" (*uaddr), \ - "=&r" (tem) \ - : "r" (oparg), "i" (-EFAULT), "1" (0)) - -static inline int -futex_atomic_op_inuser(int encoded_op, int __user *uaddr) + asm volatile("1:\tmovl %2, %0\n" \ + "\tmovl\t%0, %3\n" \ + "\t" insn "\n" \ + "2:\tlock; cmpxchgl %3, %2\n" \ + "\tjnz\t1b\n" \ + "3:\t.section .fixup,\"ax\"\n" \ + "4:\tmov\t%5, %1\n" \ + "\tjmp\t3b\n" \ + "\t.previous\n" \ + _ASM_EXTABLE(1b, 4b) \ + _ASM_EXTABLE(2b, 4b) \ + : "=&a" (oldval), "=&r" (ret), \ + "+m" (*uaddr), "=&r" (tem) \ + : "r" (oparg), "i" (-EFAULT), "1" (0)) + +static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr) { int op = (encoded_op >> 28) & 7; int cmp = (encoded_op >> 24) & 15; @@ -87,20 +84,33 @@ futex_atomic_op_inuser(int encoded_op, int __user *uaddr) if (!ret) { switch (cmp) { - case FUTEX_OP_CMP_EQ: ret = (oldval == cmparg); break; - case FUTEX_OP_CMP_NE: ret = (oldval != cmparg); break; - case FUTEX_OP_CMP_LT: ret = (oldval < cmparg); break; - case FUTEX_OP_CMP_GE: ret = (oldval >= cmparg); break; - case FUTEX_OP_CMP_LE: ret = (oldval <= cmparg); break; - case FUTEX_OP_CMP_GT: ret = (oldval > cmparg); break; - default: ret = -ENOSYS; + case FUTEX_OP_CMP_EQ: + ret = (oldval == cmparg); + break; + case FUTEX_OP_CMP_NE: + ret = (oldval != cmparg); + break; + case FUTEX_OP_CMP_LT: + ret = (oldval < cmparg); + break; + case FUTEX_OP_CMP_GE: + ret = (oldval >= cmparg); + break; + case FUTEX_OP_CMP_LE: + ret = (oldval <= cmparg); + break; + case FUTEX_OP_CMP_GT: + ret = (oldval > cmparg); + break; + default: + ret = -ENOSYS; } } return ret; } -static inline int -futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) +static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, + int newval) { #if defined(CONFIG_X86_32) && !defined(CONFIG_X86_BSWAP) @@ -112,16 +122,15 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; - __asm__ __volatile__( - "1: lock; cmpxchgl %3, %1 \n" - "2: .section .fixup, \"ax\" \n" - "3: mov %2, %0 \n" - " jmp 2b \n" - " .previous \n" - _ASM_EXTABLE(1b,3b) - : "=a" (oldval), "+m" (*uaddr) - : "i" (-EFAULT), "r" (newval), "0" (oldval) - : "memory" + asm volatile("1:\tlock; cmpxchgl %3, %1\n" + "2:\t.section .fixup, \"ax\"\n" + "3:\tmov %2, %0\n" + "\tjmp 2b\n" + "\t.previous\n" + _ASM_EXTABLE(1b, 3b) + : "=a" (oldval), "+m" (*uaddr) + : "i" (-EFAULT), "r" (newval), "0" (oldval) + : "memory" ); return oldval; diff --git a/include/asm-x86/genapic_32.h b/include/asm-x86/genapic_32.h index 33e3ffe1766..b02ea6e17de 100644 --- a/include/asm-x86/genapic_32.h +++ b/include/asm-x86/genapic_32.h @@ -14,23 +14,22 @@ * Copyright 2003 Andi Kleen, SuSE Labs. */ -struct mpc_config_translation; struct mpc_config_bus; struct mp_config_table; struct mpc_config_processor; -struct genapic { - char *name; - int (*probe)(void); +struct genapic { + char *name; + int (*probe)(void); int (*apic_id_registered)(void); cpumask_t (*target_cpus)(void); int int_delivery_mode; - int int_dest_mode; + int int_dest_mode; int ESR_DISABLE; int apic_destination_logical; unsigned long (*check_apicid_used)(physid_mask_t bitmap, int apicid); - unsigned long (*check_apicid_present)(int apicid); + unsigned long (*check_apicid_present)(int apicid); int no_balance_irq; int no_ioapic_check; void (*init_apic_ldr)(void); @@ -38,28 +37,21 @@ struct genapic { void (*setup_apic_routing)(void); int (*multi_timer_check)(int apic, int irq); - int (*apicid_to_node)(int logical_apicid); + int (*apicid_to_node)(int logical_apicid); int (*cpu_to_logical_apicid)(int cpu); int (*cpu_present_to_apicid)(int mps_cpu); physid_mask_t (*apicid_to_cpu_present)(int phys_apicid); - int (*mpc_apic_id)(struct mpc_config_processor *m, - struct mpc_config_translation *t); - void (*setup_portio_remap)(void); + void (*setup_portio_remap)(void); int (*check_phys_apicid_present)(int boot_cpu_physical_apicid); void (*enable_apic_mode)(void); u32 (*phys_pkg_id)(u32 cpuid_apic, int index_msb); /* mpparse */ - void (*mpc_oem_bus_info)(struct mpc_config_bus *, char *, - struct mpc_config_translation *); - void (*mpc_oem_pci_bus)(struct mpc_config_bus *, - struct mpc_config_translation *); - /* When one of the next two hooks returns 1 the genapic - is switched to this. Essentially they are additional probe + is switched to this. Essentially they are additional probe functions. */ - int (*mps_oem_check)(struct mp_config_table *mpc, char *oem, - char *productid); + int (*mps_oem_check)(struct mp_config_table *mpc, char *oem, + char *productid); int (*acpi_madt_oem_check)(char *oem_id, char *oem_table_id); unsigned (*get_apic_id)(unsigned long x); @@ -72,7 +64,7 @@ struct genapic { void (*send_IPI_allbutself)(int vector); void (*send_IPI_all)(int vector); #endif -}; +}; #define APICFUNC(x) .x = x, @@ -85,43 +77,47 @@ struct genapic { #define IPIFUNC(x) #endif -#define APIC_INIT(aname, aprobe) { \ - .name = aname, \ - .probe = aprobe, \ - .int_delivery_mode = INT_DELIVERY_MODE, \ - .int_dest_mode = INT_DEST_MODE, \ - .no_balance_irq = NO_BALANCE_IRQ, \ - .ESR_DISABLE = esr_disable, \ - .apic_destination_logical = APIC_DEST_LOGICAL, \ - APICFUNC(apic_id_registered) \ - APICFUNC(target_cpus) \ - APICFUNC(check_apicid_used) \ - APICFUNC(check_apicid_present) \ - APICFUNC(init_apic_ldr) \ - APICFUNC(ioapic_phys_id_map) \ - APICFUNC(setup_apic_routing) \ - APICFUNC(multi_timer_check) \ - APICFUNC(apicid_to_node) \ - APICFUNC(cpu_to_logical_apicid) \ - APICFUNC(cpu_present_to_apicid) \ - APICFUNC(apicid_to_cpu_present) \ - APICFUNC(mpc_apic_id) \ - APICFUNC(setup_portio_remap) \ - APICFUNC(check_phys_apicid_present) \ - APICFUNC(mpc_oem_bus_info) \ - APICFUNC(mpc_oem_pci_bus) \ - APICFUNC(mps_oem_check) \ - APICFUNC(get_apic_id) \ - .apic_id_mask = APIC_ID_MASK, \ - APICFUNC(cpu_mask_to_apicid) \ - APICFUNC(acpi_madt_oem_check) \ - IPIFUNC(send_IPI_mask) \ - IPIFUNC(send_IPI_allbutself) \ - IPIFUNC(send_IPI_all) \ - APICFUNC(enable_apic_mode) \ - APICFUNC(phys_pkg_id) \ - } +#define APIC_INIT(aname, aprobe) \ +{ \ + .name = aname, \ + .probe = aprobe, \ + .int_delivery_mode = INT_DELIVERY_MODE, \ + .int_dest_mode = INT_DEST_MODE, \ + .no_balance_irq = NO_BALANCE_IRQ, \ + .ESR_DISABLE = esr_disable, \ + .apic_destination_logical = APIC_DEST_LOGICAL, \ + APICFUNC(apic_id_registered) \ + APICFUNC(target_cpus) \ + APICFUNC(check_apicid_used) \ + APICFUNC(check_apicid_present) \ + APICFUNC(init_apic_ldr) \ + APICFUNC(ioapic_phys_id_map) \ + APICFUNC(setup_apic_routing) \ + APICFUNC(multi_timer_check) \ + APICFUNC(apicid_to_node) \ + APICFUNC(cpu_to_logical_apicid) \ + APICFUNC(cpu_present_to_apicid) \ + APICFUNC(apicid_to_cpu_present) \ + APICFUNC(setup_portio_remap) \ + APICFUNC(check_phys_apicid_present) \ + APICFUNC(mps_oem_check) \ + APICFUNC(get_apic_id) \ + .apic_id_mask = APIC_ID_MASK, \ + APICFUNC(cpu_mask_to_apicid) \ + APICFUNC(acpi_madt_oem_check) \ + IPIFUNC(send_IPI_mask) \ + IPIFUNC(send_IPI_allbutself) \ + IPIFUNC(send_IPI_all) \ + APICFUNC(enable_apic_mode) \ + APICFUNC(phys_pkg_id) \ +} extern struct genapic *genapic; +enum uv_system_type {UV_NONE, UV_LEGACY_APIC, UV_X2APIC, UV_NON_UNIQUE_APIC}; +#define get_uv_system_type() UV_NONE +#define is_uv_system() 0 +#define uv_wakeup_secondary(a, b) 1 + + #endif diff --git a/include/asm-x86/genapic_64.h b/include/asm-x86/genapic_64.h index d7e516ccbaa..1de931b263c 100644 --- a/include/asm-x86/genapic_64.h +++ b/include/asm-x86/genapic_64.h @@ -33,5 +33,15 @@ extern struct genapic *genapic; extern struct genapic apic_flat; extern struct genapic apic_physflat; +extern int acpi_madt_oem_check(char *, char *); + +enum uv_system_type {UV_NONE, UV_LEGACY_APIC, UV_X2APIC, UV_NON_UNIQUE_APIC}; +extern enum uv_system_type get_uv_system_type(void); +extern int is_uv_system(void); + +extern struct genapic apic_x2apic_uv_x; +DECLARE_PER_CPU(int, x2apic_extra_bits); +extern void uv_cpu_init(void); +extern int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip); #endif diff --git a/include/asm-x86/geode.h b/include/asm-x86/geode.h index 9e7280092a4..7154dc4de95 100644 --- a/include/asm-x86/geode.h +++ b/include/asm-x86/geode.h @@ -30,7 +30,13 @@ extern int geode_get_dev_base(unsigned int dev); /* MSRS */ -#define GX_GLCP_SYS_RSTPLL 0x4C000014 +#define MSR_GLIU_P2D_RO0 0x10000029 + +#define MSR_LX_GLD_MSR_CONFIG 0x48002001 +#define MSR_LX_MSR_PADSEL 0x48002011 /* NOT 0x48000011; the data + * sheet has the wrong value */ +#define MSR_GLCP_SYS_RSTPLL 0x4C000014 +#define MSR_GLCP_DOTPLL 0x4C000015 #define MSR_LBAR_SMB 0x5140000B #define MSR_LBAR_GPIO 0x5140000C @@ -45,8 +51,14 @@ extern int geode_get_dev_base(unsigned int dev); #define MSR_PIC_ZSEL_LOW 0x51400022 #define MSR_PIC_ZSEL_HIGH 0x51400023 -#define MFGPT_IRQ_MSR 0x51400028 -#define MFGPT_NR_MSR 0x51400029 +#define MSR_MFGPT_IRQ 0x51400028 +#define MSR_MFGPT_NR 0x51400029 +#define MSR_MFGPT_SETUP 0x5140002B + +#define MSR_LX_SPARE_MSR 0x80000011 /* DC-specific */ + +#define MSR_GX_GLD_MSR_CONFIG 0xC0002001 +#define MSR_GX_MSR_PADSEL 0xC0002011 /* Resource Sizes */ @@ -93,6 +105,15 @@ extern int geode_get_dev_base(unsigned int dev); #define PM_AWKD 0x50 #define PM_SSC 0x54 +/* VSA2 magic values */ + +#define VSA_VRC_INDEX 0xAC1C +#define VSA_VRC_DATA 0xAC1E +#define VSA_VR_UNLOCK 0xFC53 /* unlock virtual register */ +#define VSA_VR_SIGNATURE 0x0003 +#define VSA_VR_MEM_SIZE 0x0200 +#define VSA_SIG 0x4132 /* signature is ascii 'VSA2' */ + /* GPIO */ #define GPIO_OUTPUT_VAL 0x00 @@ -164,10 +185,21 @@ static inline int is_geode(void) return (is_geode_gx() || is_geode_lx()); } +/* + * The VSA has virtual registers that we can query for a signature. + */ +static inline int geode_has_vsa2(void) +{ + outw(VSA_VR_UNLOCK, VSA_VRC_INDEX); + outw(VSA_VR_SIGNATURE, VSA_VRC_INDEX); + + return (inw(VSA_VRC_DATA) == VSA_SIG); +} + /* MFGPTs */ #define MFGPT_MAX_TIMERS 8 -#define MFGPT_TIMER_ANY -1 +#define MFGPT_TIMER_ANY (-1) #define MFGPT_DOMAIN_WORKING 1 #define MFGPT_DOMAIN_STANDBY 2 diff --git a/include/asm-x86/highmem.h b/include/asm-x86/highmem.h index 479767c9195..e153f3b4477 100644 --- a/include/asm-x86/highmem.h +++ b/include/asm-x86/highmem.h @@ -8,7 +8,7 @@ * Gerhard.Wichert@pdb.siemens.de * * - * Redesigned the x86 32-bit VM architecture to deal with + * Redesigned the x86 32-bit VM architecture to deal with * up to 16 Terabyte physical memory. With current x86 CPUs * we now support up to 64 Gigabytes physical RAM. * diff --git a/include/asm-x86/hugetlb.h b/include/asm-x86/hugetlb.h new file mode 100644 index 00000000000..14171a4924f --- /dev/null +++ b/include/asm-x86/hugetlb.h @@ -0,0 +1,91 @@ +#ifndef _ASM_X86_HUGETLB_H +#define _ASM_X86_HUGETLB_H + +#include <asm/page.h> + + +static inline int is_hugepage_only_range(struct mm_struct *mm, + unsigned long addr, + unsigned long len) { + return 0; +} + +/* + * If the arch doesn't supply something else, assume that hugepage + * size aligned regions are ok without further preparation. + */ +static inline int prepare_hugepage_range(unsigned long addr, unsigned long len) +{ + if (len & ~HPAGE_MASK) + return -EINVAL; + if (addr & ~HPAGE_MASK) + return -EINVAL; + return 0; +} + +static inline void hugetlb_prefault_arch_hook(struct mm_struct *mm) { +} + +static inline void hugetlb_free_pgd_range(struct mmu_gather **tlb, + unsigned long addr, unsigned long end, + unsigned long floor, + unsigned long ceiling) +{ + free_pgd_range(tlb, addr, end, floor, ceiling); +} + +static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + set_pte_at(mm, addr, ptep, pte); +} + +static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + return ptep_get_and_clear(mm, addr, ptep); +} + +static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ +} + +static inline int huge_pte_none(pte_t pte) +{ + return pte_none(pte); +} + +static inline pte_t huge_pte_wrprotect(pte_t pte) +{ + return pte_wrprotect(pte); +} + +static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + ptep_set_wrprotect(mm, addr, ptep); +} + +static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, int dirty) +{ + return ptep_set_access_flags(vma, addr, ptep, pte, dirty); +} + +static inline pte_t huge_ptep_get(pte_t *ptep) +{ + return *ptep; +} + +static inline int arch_prepare_hugepage(struct page *page) +{ + return 0; +} + +static inline void arch_release_hugepage(struct page *page) +{ +} + +#endif /* _ASM_X86_HUGETLB_H */ diff --git a/include/asm-x86/hw_irq_64.h b/include/asm-x86/hw_irq_64.h index 312a58d6dac..0062ef390f6 100644 --- a/include/asm-x86/hw_irq_64.h +++ b/include/asm-x86/hw_irq_64.h @@ -36,7 +36,7 @@ * cleanup after irq migration. */ #define IRQ_MOVE_CLEANUP_VECTOR FIRST_EXTERNAL_VECTOR - + /* * Vectors 0x30-0x3f are used for ISA interrupts. */ @@ -159,13 +159,12 @@ extern atomic_t irq_mis_count; * SMP has a few special interrupts for IPI messages */ -#define BUILD_IRQ(nr) \ -asmlinkage void IRQ_NAME(nr); \ -__asm__( \ -"\n.p2align\n" \ -"IRQ" #nr "_interrupt:\n\t" \ - "push $~(" #nr ") ; " \ - "jmp common_interrupt"); +#define BUILD_IRQ(nr) \ + asmlinkage void IRQ_NAME(nr); \ + asm("\n.p2align\n" \ + "IRQ" #nr "_interrupt:\n\t" \ + "push $~(" #nr ") ; " \ + "jmp common_interrupt"); #define platform_legacy_irq(irq) ((irq) < 16) diff --git a/include/asm-x86/hypertransport.h b/include/asm-x86/hypertransport.h index c16c6ff4bdd..d2bbd238b3e 100644 --- a/include/asm-x86/hypertransport.h +++ b/include/asm-x86/hypertransport.h @@ -8,12 +8,14 @@ #define HT_IRQ_LOW_BASE 0xf8000000 #define HT_IRQ_LOW_VECTOR_SHIFT 16 -#define HT_IRQ_LOW_VECTOR_MASK 0x00ff0000 -#define HT_IRQ_LOW_VECTOR(v) (((v) << HT_IRQ_LOW_VECTOR_SHIFT) & HT_IRQ_LOW_VECTOR_MASK) +#define HT_IRQ_LOW_VECTOR_MASK 0x00ff0000 +#define HT_IRQ_LOW_VECTOR(v) \ + (((v) << HT_IRQ_LOW_VECTOR_SHIFT) & HT_IRQ_LOW_VECTOR_MASK) #define HT_IRQ_LOW_DEST_ID_SHIFT 8 -#define HT_IRQ_LOW_DEST_ID_MASK 0x0000ff00 -#define HT_IRQ_LOW_DEST_ID(v) (((v) << HT_IRQ_LOW_DEST_ID_SHIFT) & HT_IRQ_LOW_DEST_ID_MASK) +#define HT_IRQ_LOW_DEST_ID_MASK 0x0000ff00 +#define HT_IRQ_LOW_DEST_ID(v) \ + (((v) << HT_IRQ_LOW_DEST_ID_SHIFT) & HT_IRQ_LOW_DEST_ID_MASK) #define HT_IRQ_LOW_DM_PHYSICAL 0x0000000 #define HT_IRQ_LOW_DM_LOGICAL 0x0000040 @@ -36,7 +38,8 @@ #define HT_IRQ_HIGH_DEST_ID_SHIFT 0 -#define HT_IRQ_HIGH_DEST_ID_MASK 0x00ffffff -#define HT_IRQ_HIGH_DEST_ID(v) ((((v) >> 8) << HT_IRQ_HIGH_DEST_ID_SHIFT) & HT_IRQ_HIGH_DEST_ID_MASK) +#define HT_IRQ_HIGH_DEST_ID_MASK 0x00ffffff +#define HT_IRQ_HIGH_DEST_ID(v) \ + ((((v) >> 8) << HT_IRQ_HIGH_DEST_ID_SHIFT) & HT_IRQ_HIGH_DEST_ID_MASK) #endif /* ASM_HYPERTRANSPORT_H */ diff --git a/include/asm-x86/i387.h b/include/asm-x86/i387.h index f377b76b2f3..da2adb45f6e 100644 --- a/include/asm-x86/i387.h +++ b/include/asm-x86/i387.h @@ -21,8 +21,9 @@ extern void fpu_init(void); extern void mxcsr_feature_mask_init(void); -extern void init_fpu(struct task_struct *child); +extern int init_fpu(struct task_struct *child); extern asmlinkage void math_state_restore(void); +extern void init_thread_xstate(void); extern user_regset_active_fn fpregs_active, xfpregs_active; extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get; @@ -41,7 +42,7 @@ static inline void tolerant_fwait(void) { asm volatile("1: fwait\n" "2:\n" - _ASM_EXTABLE(1b,2b)); + _ASM_EXTABLE(1b, 2b)); } static inline int restore_fpu_checking(struct i387_fxsave_struct *fx) @@ -54,7 +55,7 @@ static inline int restore_fpu_checking(struct i387_fxsave_struct *fx) "3: movl $-1,%[err]\n" " jmp 2b\n" ".previous\n" - _ASM_EXTABLE(1b,3b) + _ASM_EXTABLE(1b, 3b) : [err] "=r" (err) #if 0 /* See comment in __save_init_fpu() below. */ : [fx] "r" (fx), "m" (*fx), "0" (0)); @@ -76,11 +77,11 @@ static inline int restore_fpu_checking(struct i387_fxsave_struct *fx) static inline void clear_fpu_state(struct i387_fxsave_struct *fx) { if (unlikely(fx->swd & X87_FSW_ES)) - asm volatile("fnclex"); + asm volatile("fnclex"); alternative_input(ASM_NOP8 ASM_NOP2, - " emms\n" /* clear stack tags */ - " fildl %%gs:0", /* load to clear state */ - X86_FEATURE_FXSAVE_LEAK); + " emms\n" /* clear stack tags */ + " fildl %%gs:0", /* load to clear state */ + X86_FEATURE_FXSAVE_LEAK); } static inline int save_i387_checking(struct i387_fxsave_struct __user *fx) @@ -93,14 +94,15 @@ static inline int save_i387_checking(struct i387_fxsave_struct __user *fx) "3: movl $-1,%[err]\n" " jmp 2b\n" ".previous\n" - _ASM_EXTABLE(1b,3b) + _ASM_EXTABLE(1b, 3b) : [err] "=r" (err), "=m" (*fx) #if 0 /* See comment in __fxsave_clear() below. */ : [fx] "r" (fx), "0" (0)); #else : [fx] "cdaSDb" (fx), "0" (0)); #endif - if (unlikely(err) && __clear_user(fx, sizeof(struct i387_fxsave_struct))) + if (unlikely(err) && + __clear_user(fx, sizeof(struct i387_fxsave_struct))) err = -EFAULT; /* No need to clear here because the caller clears USED_MATH */ return err; @@ -116,24 +118,22 @@ static inline void __save_init_fpu(struct task_struct *tsk) /* Using "fxsaveq %0" would be the ideal choice, but is only supported starting with gas 2.16. */ __asm__ __volatile__("fxsaveq %0" - : "=m" (tsk->thread.i387.fxsave)); + : "=m" (tsk->thread.xstate->fxsave)); #elif 0 /* Using, as a workaround, the properly prefixed form below isn't accepted by any binutils version so far released, complaining that the same type of prefix is used twice if an extended register is needed for addressing (fix submitted to mainline 2005-11-21). */ __asm__ __volatile__("rex64/fxsave %0" - : "=m" (tsk->thread.i387.fxsave)); + : "=m" (tsk->thread.xstate->fxsave)); #else /* This, however, we can work around by forcing the compiler to select an addressing mode that doesn't require extended registers. */ - __asm__ __volatile__("rex64/fxsave %P2(%1)" - : "=m" (tsk->thread.i387.fxsave) - : "cdaSDb" (tsk), - "i" (offsetof(__typeof__(*tsk), - thread.i387.fxsave))); + __asm__ __volatile__("rex64/fxsave (%1)" + : "=m" (tsk->thread.xstate->fxsave) + : "cdaSDb" (&tsk->thread.xstate->fxsave)); #endif - clear_fpu_state(&tsk->thread.i387.fxsave); + clear_fpu_state(&tsk->thread.xstate->fxsave); task_thread_info(tsk)->status &= ~TS_USEDFPU; } @@ -147,7 +147,7 @@ static inline int save_i387(struct _fpstate __user *buf) int err = 0; BUILD_BUG_ON(sizeof(struct user_i387_struct) != - sizeof(tsk->thread.i387.fxsave)); + sizeof(tsk->thread.xstate->fxsave)); if ((unsigned long)buf % 16) printk("save_i387: bad fpstate %p\n", buf); @@ -156,12 +156,14 @@ static inline int save_i387(struct _fpstate __user *buf) return 0; clear_used_math(); /* trigger finit */ if (task_thread_info(tsk)->status & TS_USEDFPU) { - err = save_i387_checking((struct i387_fxsave_struct __user *)buf); - if (err) return err; + err = save_i387_checking((struct i387_fxsave_struct __user *) + buf); + if (err) + return err; task_thread_info(tsk)->status &= ~TS_USEDFPU; stts(); } else { - if (__copy_to_user(buf, &tsk->thread.i387.fxsave, + if (__copy_to_user(buf, &tsk->thread.xstate->fxsave, sizeof(struct i387_fxsave_struct))) return -1; } @@ -198,7 +200,7 @@ static inline void restore_fpu(struct task_struct *tsk) "nop ; frstor %1", "fxrstor %1", X86_FEATURE_FXSR, - "m" ((tsk)->thread.i387.fxsave)); + "m" (tsk->thread.xstate->fxsave)); } /* We need a safe address that is cheap to find and that is already @@ -222,8 +224,8 @@ static inline void __save_init_fpu(struct task_struct *tsk) "fxsave %[fx]\n" "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:", X86_FEATURE_FXSR, - [fx] "m" (tsk->thread.i387.fxsave), - [fsw] "m" (tsk->thread.i387.fxsave.swd) : "memory"); + [fx] "m" (tsk->thread.xstate->fxsave), + [fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory"); /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception is pending. Clear the x87 state here by setting it to fixed values. safe_address is a random variable that should be in L1 */ @@ -324,25 +326,25 @@ static inline void clear_fpu(struct task_struct *tsk) static inline unsigned short get_fpu_cwd(struct task_struct *tsk) { if (cpu_has_fxsr) { - return tsk->thread.i387.fxsave.cwd; + return tsk->thread.xstate->fxsave.cwd; } else { - return (unsigned short)tsk->thread.i387.fsave.cwd; + return (unsigned short)tsk->thread.xstate->fsave.cwd; } } static inline unsigned short get_fpu_swd(struct task_struct *tsk) { if (cpu_has_fxsr) { - return tsk->thread.i387.fxsave.swd; + return tsk->thread.xstate->fxsave.swd; } else { - return (unsigned short)tsk->thread.i387.fsave.swd; + return (unsigned short)tsk->thread.xstate->fsave.swd; } } static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) { if (cpu_has_xmm) { - return tsk->thread.i387.fxsave.mxcsr; + return tsk->thread.xstate->fxsave.mxcsr; } else { return MXCSR_DEFAULT; } diff --git a/include/asm-x86/i8259.h b/include/asm-x86/i8259.h index 67c319e0efc..45d4df3e51e 100644 --- a/include/asm-x86/i8259.h +++ b/include/asm-x86/i8259.h @@ -1,9 +1,11 @@ #ifndef __ASM_I8259_H__ #define __ASM_I8259_H__ +#include <linux/delay.h> + extern unsigned int cached_irq_mask; -#define __byte(x,y) (((unsigned char *) &(y))[x]) +#define __byte(x, y) (((unsigned char *)&(y))[x]) #define cached_master_mask (__byte(0, cached_irq_mask)) #define cached_slave_mask (__byte(1, cached_irq_mask)) @@ -29,7 +31,28 @@ extern void enable_8259A_irq(unsigned int irq); extern void disable_8259A_irq(unsigned int irq); extern unsigned int startup_8259A_irq(unsigned int irq); -#define inb_pic inb_p -#define outb_pic outb_p +/* the PIC may need a careful delay on some platforms, hence specific calls */ +static inline unsigned char inb_pic(unsigned int port) +{ + unsigned char value = inb(port); + + /* + * delay for some accesses to PIC on motherboard or in chipset + * must be at least one microsecond, so be safe here: + */ + udelay(2); + + return value; +} + +static inline void outb_pic(unsigned char value, unsigned int port) +{ + outb(value, port); + /* + * delay for some accesses to PIC on motherboard or in chipset + * must be at least one microsecond, so be safe here: + */ + udelay(2); +} #endif /* __ASM_I8259_H__ */ diff --git a/include/asm-x86/ia32.h b/include/asm-x86/ia32.h index aa9733206e2..55d3abe5276 100644 --- a/include/asm-x86/ia32.h +++ b/include/asm-x86/ia32.h @@ -14,19 +14,19 @@ /* signal.h */ struct sigaction32 { - unsigned int sa_handler; /* Really a pointer, but need to deal - with 32 bits */ - unsigned int sa_flags; - unsigned int sa_restorer; /* Another 32 bit pointer */ - compat_sigset_t sa_mask; /* A 32 bit mask */ + unsigned int sa_handler; /* Really a pointer, but need to deal + with 32 bits */ + unsigned int sa_flags; + unsigned int sa_restorer; /* Another 32 bit pointer */ + compat_sigset_t sa_mask; /* A 32 bit mask */ }; struct old_sigaction32 { - unsigned int sa_handler; /* Really a pointer, but need to deal - with 32 bits */ - compat_old_sigset_t sa_mask; /* A 32 bit mask */ - unsigned int sa_flags; - unsigned int sa_restorer; /* Another 32 bit pointer */ + unsigned int sa_handler; /* Really a pointer, but need to deal + with 32 bits */ + compat_old_sigset_t sa_mask; /* A 32 bit mask */ + unsigned int sa_flags; + unsigned int sa_restorer; /* Another 32 bit pointer */ }; typedef struct sigaltstack_ia32 { @@ -65,7 +65,7 @@ struct stat64 { long long st_size; unsigned int st_blksize; - long long st_blocks;/* Number 512-byte blocks allocated. */ + long long st_blocks;/* Number 512-byte blocks allocated */ unsigned st_atime; unsigned st_atime_nsec; @@ -77,13 +77,13 @@ struct stat64 { unsigned long long st_ino; } __attribute__((packed)); -typedef struct compat_siginfo{ +typedef struct compat_siginfo { int si_signo; int si_errno; int si_code; union { - int _pad[((128/sizeof(int)) - 3)]; + int _pad[((128 / sizeof(int)) - 3)]; /* kill() */ struct { @@ -129,28 +129,26 @@ typedef struct compat_siginfo{ } _sifields; } compat_siginfo_t; -struct sigframe32 -{ - u32 pretcode; - int sig; - struct sigcontext_ia32 sc; - struct _fpstate_ia32 fpstate; - unsigned int extramask[_COMPAT_NSIG_WORDS-1]; +struct sigframe32 { + u32 pretcode; + int sig; + struct sigcontext_ia32 sc; + struct _fpstate_ia32 fpstate; + unsigned int extramask[_COMPAT_NSIG_WORDS-1]; }; -struct rt_sigframe32 -{ - u32 pretcode; - int sig; - u32 pinfo; - u32 puc; - compat_siginfo_t info; - struct ucontext_ia32 uc; - struct _fpstate_ia32 fpstate; +struct rt_sigframe32 { + u32 pretcode; + int sig; + u32 pinfo; + u32 puc; + compat_siginfo_t info; + struct ucontext_ia32 uc; + struct _fpstate_ia32 fpstate; }; struct ustat32 { - __u32 f_tfree; + __u32 f_tfree; compat_ino_t f_tinode; char f_fname[6]; char f_fpack[6]; @@ -168,5 +166,5 @@ extern void ia32_pick_mmap_layout(struct mm_struct *mm); #endif #endif /* !CONFIG_IA32_SUPPORT */ - -#endif + +#endif diff --git a/include/asm-x86/ide.h b/include/asm-x86/ide.h index c2552d8bebf..cf9c98e5bdb 100644 --- a/include/asm-x86/ide.h +++ b/include/asm-x86/ide.h @@ -20,8 +20,6 @@ # endif #endif -#define IDE_ARCH_OBSOLETE_DEFAULTS - static __inline__ int ide_default_irq(unsigned long base) { switch (base) { @@ -60,14 +58,6 @@ static __inline__ unsigned long ide_default_io_base(int index) } } -#define ide_default_io_ctl(base) ((base) + 0x206) /* obsolete */ - -#ifdef CONFIG_BLK_DEV_IDEPCI -#define ide_init_default_irq(base) (0) -#else -#define ide_init_default_irq(base) ide_default_irq(base) -#endif - #include <asm-generic/ide_iops.h> #endif /* __KERNEL__ */ diff --git a/include/asm-x86/io.h b/include/asm-x86/io.h index 5a58b176dd6..d5b11f60dbd 100644 --- a/include/asm-x86/io.h +++ b/include/asm-x86/io.h @@ -1,5 +1,19 @@ +#ifndef _ASM_X86_IO_H +#define _ASM_X86_IO_H + +#define ARCH_HAS_IOREMAP_WC + #ifdef CONFIG_X86_32 # include "io_32.h" #else # include "io_64.h" #endif + +extern void *xlate_dev_mem_ptr(unsigned long phys); +extern void unxlate_dev_mem_ptr(unsigned long phys, void *addr); + +extern int ioremap_change_attr(unsigned long vaddr, unsigned long size, + unsigned long prot_val); +extern void __iomem *ioremap_wc(unsigned long offset, unsigned long size); + +#endif /* _ASM_X86_IO_H */ diff --git a/include/asm-x86/io_32.h b/include/asm-x86/io_32.h index d4d8fbd9378..049e81e797a 100644 --- a/include/asm-x86/io_32.h +++ b/include/asm-x86/io_32.h @@ -49,12 +49,6 @@ #include <linux/vmalloc.h> /* - * Convert a physical pointer to a virtual kernel pointer for /dev/mem - * access - */ -#define xlate_dev_mem_ptr(p) __va(p) - -/* * Convert a virtual cached pointer to an uncached pointer */ #define xlate_dev_kmem_ptr(p) p @@ -65,14 +59,14 @@ * * The returned physical address is the physical (CPU) mapping for * the memory address given. It is only valid to use this function on - * addresses directly mapped or allocated via kmalloc. + * addresses directly mapped or allocated via kmalloc. * * This function does not give bus mappings for DMA transfers. In * almost all conceivable cases a device driver should not be using * this function */ - -static inline unsigned long virt_to_phys(volatile void * address) + +static inline unsigned long virt_to_phys(volatile void *address) { return __pa(address); } @@ -90,7 +84,7 @@ static inline unsigned long virt_to_phys(volatile void * address) * this function */ -static inline void * phys_to_virt(unsigned long address) +static inline void *phys_to_virt(unsigned long address) { return __va(address); } @@ -139,11 +133,6 @@ extern void *early_ioremap(unsigned long offset, unsigned long size); extern void early_iounmap(void *addr, unsigned long size); extern void __iomem *fix_ioremap(unsigned idx, unsigned long phys); -/* Use early IO mappings for DMI because it's initialized early */ -#define dmi_ioremap early_ioremap -#define dmi_iounmap early_iounmap -#define dmi_alloc alloc_bootmem - /* * ISA I/O bus memory addresses are 1:1 with the physical address. */ @@ -169,16 +158,19 @@ extern void __iomem *fix_ioremap(unsigned idx, unsigned long phys); static inline unsigned char readb(const volatile void __iomem *addr) { - return *(volatile unsigned char __force *) addr; + return *(volatile unsigned char __force *)addr; } + static inline unsigned short readw(const volatile void __iomem *addr) { - return *(volatile unsigned short __force *) addr; + return *(volatile unsigned short __force *)addr; } + static inline unsigned int readl(const volatile void __iomem *addr) { return *(volatile unsigned int __force *) addr; } + #define readb_relaxed(addr) readb(addr) #define readw_relaxed(addr) readw(addr) #define readl_relaxed(addr) readl(addr) @@ -188,15 +180,17 @@ static inline unsigned int readl(const volatile void __iomem *addr) static inline void writeb(unsigned char b, volatile void __iomem *addr) { - *(volatile unsigned char __force *) addr = b; + *(volatile unsigned char __force *)addr = b; } + static inline void writew(unsigned short b, volatile void __iomem *addr) { - *(volatile unsigned short __force *) addr = b; + *(volatile unsigned short __force *)addr = b; } + static inline void writel(unsigned int b, volatile void __iomem *addr) { - *(volatile unsigned int __force *) addr = b; + *(volatile unsigned int __force *)addr = b; } #define __raw_writeb writeb #define __raw_writew writew @@ -239,12 +233,12 @@ memcpy_toio(volatile void __iomem *dst, const void *src, int count) * 1. Out of order aware processors * 2. Accidentally out of order processors (PPro errata #51) */ - + #if defined(CONFIG_X86_OOSTORE) || defined(CONFIG_X86_PPRO_FENCE) static inline void flush_write_buffers(void) { - __asm__ __volatile__ ("lock; addl $0,0(%%esp)": : :"memory"); + asm volatile("lock; addl $0,0(%%esp)": : :"memory"); } #else @@ -264,7 +258,8 @@ extern void io_delay_init(void); #include <asm/paravirt.h> #else -static inline void slow_down_io(void) { +static inline void slow_down_io(void) +{ native_io_delay(); #ifdef REALLY_SLOW_IO native_io_delay(); @@ -275,51 +270,74 @@ static inline void slow_down_io(void) { #endif -#define __BUILDIO(bwl,bw,type) \ -static inline void out##bwl(unsigned type value, int port) { \ - out##bwl##_local(value, port); \ -} \ -static inline unsigned type in##bwl(int port) { \ - return in##bwl##_local(port); \ +#define __BUILDIO(bwl, bw, type) \ +static inline void out##bwl(unsigned type value, int port) \ +{ \ + out##bwl##_local(value, port); \ +} \ + \ +static inline unsigned type in##bwl(int port) \ +{ \ + return in##bwl##_local(port); \ } -#define BUILDIO(bwl,bw,type) \ -static inline void out##bwl##_local(unsigned type value, int port) { \ - __asm__ __volatile__("out" #bwl " %" #bw "0, %w1" : : "a"(value), "Nd"(port)); \ -} \ -static inline unsigned type in##bwl##_local(int port) { \ - unsigned type value; \ - __asm__ __volatile__("in" #bwl " %w1, %" #bw "0" : "=a"(value) : "Nd"(port)); \ - return value; \ -} \ -static inline void out##bwl##_local_p(unsigned type value, int port) { \ - out##bwl##_local(value, port); \ - slow_down_io(); \ -} \ -static inline unsigned type in##bwl##_local_p(int port) { \ - unsigned type value = in##bwl##_local(port); \ - slow_down_io(); \ - return value; \ -} \ -__BUILDIO(bwl,bw,type) \ -static inline void out##bwl##_p(unsigned type value, int port) { \ - out##bwl(value, port); \ - slow_down_io(); \ -} \ -static inline unsigned type in##bwl##_p(int port) { \ - unsigned type value = in##bwl(port); \ - slow_down_io(); \ - return value; \ -} \ -static inline void outs##bwl(int port, const void *addr, unsigned long count) { \ - __asm__ __volatile__("rep; outs" #bwl : "+S"(addr), "+c"(count) : "d"(port)); \ -} \ -static inline void ins##bwl(int port, void *addr, unsigned long count) { \ - __asm__ __volatile__("rep; ins" #bwl : "+D"(addr), "+c"(count) : "d"(port)); \ +#define BUILDIO(bwl, bw, type) \ +static inline void out##bwl##_local(unsigned type value, int port) \ +{ \ + asm volatile("out" #bwl " %" #bw "0, %w1" \ + : : "a"(value), "Nd"(port)); \ +} \ + \ +static inline unsigned type in##bwl##_local(int port) \ +{ \ + unsigned type value; \ + asm volatile("in" #bwl " %w1, %" #bw "0" \ + : "=a"(value) : "Nd"(port)); \ + return value; \ +} \ + \ +static inline void out##bwl##_local_p(unsigned type value, int port) \ +{ \ + out##bwl##_local(value, port); \ + slow_down_io(); \ +} \ + \ +static inline unsigned type in##bwl##_local_p(int port) \ +{ \ + unsigned type value = in##bwl##_local(port); \ + slow_down_io(); \ + return value; \ +} \ + \ +__BUILDIO(bwl, bw, type) \ + \ +static inline void out##bwl##_p(unsigned type value, int port) \ +{ \ + out##bwl(value, port); \ + slow_down_io(); \ +} \ + \ +static inline unsigned type in##bwl##_p(int port) \ +{ \ + unsigned type value = in##bwl(port); \ + slow_down_io(); \ + return value; \ +} \ + \ +static inline void outs##bwl(int port, const void *addr, unsigned long count) \ +{ \ + asm volatile("rep; outs" #bwl \ + : "+S"(addr), "+c"(count) : "d"(port)); \ +} \ + \ +static inline void ins##bwl(int port, void *addr, unsigned long count) \ +{ \ + asm volatile("rep; ins" #bwl \ + : "+D"(addr), "+c"(count) : "d"(port)); \ } -BUILDIO(b,b,char) -BUILDIO(w,w,short) -BUILDIO(l,,int) +BUILDIO(b, b, char) +BUILDIO(w, w, short) +BUILDIO(l, , int) #endif diff --git a/include/asm-x86/io_64.h b/include/asm-x86/io_64.h index db0be2011a3..0930bedf9e4 100644 --- a/include/asm-x86/io_64.h +++ b/include/asm-x86/io_64.h @@ -58,60 +58,75 @@ static inline void slow_down_io(void) /* * Talk about misusing macros.. */ -#define __OUT1(s,x) \ +#define __OUT1(s, x) \ static inline void out##s(unsigned x value, unsigned short port) { -#define __OUT2(s,s1,s2) \ -__asm__ __volatile__ ("out" #s " %" s1 "0,%" s2 "1" +#define __OUT2(s, s1, s2) \ +asm volatile ("out" #s " %" s1 "0,%" s2 "1" #ifndef REALLY_SLOW_IO #define REALLY_SLOW_IO #define UNSET_REALLY_SLOW_IO #endif -#define __OUT(s,s1,x) \ -__OUT1(s,x) __OUT2(s,s1,"w") : : "a" (value), "Nd" (port)); } \ -__OUT1(s##_p, x) __OUT2(s, s1, "w") : : "a" (value), "Nd" (port)); \ - slow_down_io(); } +#define __OUT(s, s1, x) \ + __OUT1(s, x) __OUT2(s, s1, "w") : : "a" (value), "Nd" (port)); \ + } \ + __OUT1(s##_p, x) __OUT2(s, s1, "w") : : "a" (value), "Nd" (port)); \ + slow_down_io(); \ +} -#define __IN1(s) \ -static inline RETURN_TYPE in##s(unsigned short port) { RETURN_TYPE _v; +#define __IN1(s) \ +static inline RETURN_TYPE in##s(unsigned short port) \ +{ \ + RETURN_TYPE _v; -#define __IN2(s,s1,s2) \ -__asm__ __volatile__ ("in" #s " %" s2 "1,%" s1 "0" +#define __IN2(s, s1, s2) \ + asm volatile ("in" #s " %" s2 "1,%" s1 "0" -#define __IN(s,s1,i...) \ -__IN1(s) __IN2(s, s1, "w") : "=a" (_v) : "Nd" (port), ##i); return _v; } \ -__IN1(s##_p) __IN2(s, s1, "w") : "=a" (_v) : "Nd" (port), ##i); \ - slow_down_io(); return _v; } +#define __IN(s, s1, i...) \ + __IN1(s) __IN2(s, s1, "w") : "=a" (_v) : "Nd" (port), ##i); \ + return _v; \ + } \ + __IN1(s##_p) __IN2(s, s1, "w") : "=a" (_v) : "Nd" (port), ##i); \ + slow_down_io(); \ + return _v; } #ifdef UNSET_REALLY_SLOW_IO #undef REALLY_SLOW_IO #endif -#define __INS(s) \ -static inline void ins##s(unsigned short port, void * addr, unsigned long count) \ -{ __asm__ __volatile__ ("rep ; ins" #s \ -: "=D" (addr), "=c" (count) : "d" (port),"0" (addr),"1" (count)); } +#define __INS(s) \ +static inline void ins##s(unsigned short port, void *addr, \ + unsigned long count) \ +{ \ + asm volatile ("rep ; ins" #s \ + : "=D" (addr), "=c" (count) \ + : "d" (port), "0" (addr), "1" (count)); \ +} -#define __OUTS(s) \ -static inline void outs##s(unsigned short port, const void * addr, unsigned long count) \ -{ __asm__ __volatile__ ("rep ; outs" #s \ -: "=S" (addr), "=c" (count) : "d" (port),"0" (addr),"1" (count)); } +#define __OUTS(s) \ +static inline void outs##s(unsigned short port, const void *addr, \ + unsigned long count) \ +{ \ + asm volatile ("rep ; outs" #s \ + : "=S" (addr), "=c" (count) \ + : "d" (port), "0" (addr), "1" (count)); \ +} #define RETURN_TYPE unsigned char -__IN(b,"") +__IN(b, "") #undef RETURN_TYPE #define RETURN_TYPE unsigned short -__IN(w,"") +__IN(w, "") #undef RETURN_TYPE #define RETURN_TYPE unsigned int -__IN(l,"") +__IN(l, "") #undef RETURN_TYPE -__OUT(b,"b",char) -__OUT(w,"w",short) -__OUT(l,,int) +__OUT(b, "b", char) +__OUT(w, "w", short) +__OUT(l, , int) __INS(b) __INS(w) @@ -132,12 +147,12 @@ __OUTS(l) * Change virtual addresses to physical addresses and vv. * These are pretty trivial */ -static inline unsigned long virt_to_phys(volatile void * address) +static inline unsigned long virt_to_phys(volatile void *address) { return __pa(address); } -static inline void * phys_to_virt(unsigned long address) +static inline void *phys_to_virt(unsigned long address) { return __va(address); } @@ -200,18 +215,22 @@ static inline __u8 __readb(const volatile void __iomem *addr) { return *(__force volatile __u8 *)addr; } + static inline __u16 __readw(const volatile void __iomem *addr) { return *(__force volatile __u16 *)addr; } + static __always_inline __u32 __readl(const volatile void __iomem *addr) { return *(__force volatile __u32 *)addr; } + static inline __u64 __readq(const volatile void __iomem *addr) { return *(__force volatile __u64 *)addr; } + #define readb(x) __readb(x) #define readw(x) __readw(x) #define readl(x) __readl(x) @@ -231,37 +250,44 @@ static inline void __writel(__u32 b, volatile void __iomem *addr) { *(__force volatile __u32 *)addr = b; } + static inline void __writeq(__u64 b, volatile void __iomem *addr) { *(__force volatile __u64 *)addr = b; } + static inline void __writeb(__u8 b, volatile void __iomem *addr) { *(__force volatile __u8 *)addr = b; } + static inline void __writew(__u16 b, volatile void __iomem *addr) { *(__force volatile __u16 *)addr = b; } -#define writeq(val,addr) __writeq((val),(addr)) -#define writel(val,addr) __writel((val),(addr)) -#define writew(val,addr) __writew((val),(addr)) -#define writeb(val,addr) __writeb((val),(addr)) + +#define writeq(val, addr) __writeq((val), (addr)) +#define writel(val, addr) __writel((val), (addr)) +#define writew(val, addr) __writew((val), (addr)) +#define writeb(val, addr) __writeb((val), (addr)) #define __raw_writeb writeb #define __raw_writew writew #define __raw_writel writel #define __raw_writeq writeq -void __memcpy_fromio(void*,unsigned long,unsigned); -void __memcpy_toio(unsigned long,const void*,unsigned); +void __memcpy_fromio(void *, unsigned long, unsigned); +void __memcpy_toio(unsigned long, const void *, unsigned); -static inline void memcpy_fromio(void *to, const volatile void __iomem *from, unsigned len) +static inline void memcpy_fromio(void *to, const volatile void __iomem *from, + unsigned len) { - __memcpy_fromio(to,(unsigned long)from,len); + __memcpy_fromio(to, (unsigned long)from, len); } -static inline void memcpy_toio(volatile void __iomem *to, const void *from, unsigned len) + +static inline void memcpy_toio(volatile void __iomem *to, const void *from, + unsigned len) { - __memcpy_toio((unsigned long)to,from,len); + __memcpy_toio((unsigned long)to, from, len); } void memset_io(volatile void __iomem *a, int b, size_t c); @@ -276,18 +302,12 @@ void memset_io(volatile void __iomem *a, int b, size_t c); */ #define __ISA_IO_base ((char __iomem *)(PAGE_OFFSET)) -#define flush_write_buffers() +#define flush_write_buffers() extern int iommu_bio_merge; #define BIO_VMERGE_BOUNDARY iommu_bio_merge /* - * Convert a physical pointer to a virtual kernel pointer for /dev/mem - * access - */ -#define xlate_dev_mem_ptr(p) __va(p) - -/* * Convert a virtual cached pointer to an uncached pointer */ #define xlate_dev_kmem_ptr(p) p diff --git a/include/asm-x86/io_apic.h b/include/asm-x86/io_apic.h index 0f5b3fef0b0..d593e14f034 100644 --- a/include/asm-x86/io_apic.h +++ b/include/asm-x86/io_apic.h @@ -1,7 +1,7 @@ #ifndef __ASM_IO_APIC_H #define __ASM_IO_APIC_H -#include <asm/types.h> +#include <linux/types.h> #include <asm/mpspec.h> #include <asm/apicdef.h> @@ -110,6 +110,15 @@ extern int nr_ioapic_registers[MAX_IO_APICS]; * MP-BIOS irq configuration table structures: */ +#define MP_MAX_IOAPIC_PIN 127 + +struct mp_ioapic_routing { + int apic_id; + int gsi_base; + int gsi_end; + DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1); +}; + /* I/O APIC entries */ extern struct mpc_config_ioapic mp_ioapics[MAX_IO_APICS]; @@ -146,7 +155,6 @@ extern int io_apic_get_version(int ioapic); extern int io_apic_get_redir_entries(int ioapic); extern int io_apic_set_pci_routing(int ioapic, int pin, int irq, int edge_level, int active_high_low); -extern int timer_uses_ioapic_pin_0; #endif /* CONFIG_ACPI */ extern int (*ioapic_renumber_irq)(int ioapic, int irq); diff --git a/include/asm-x86/ioctls.h b/include/asm-x86/ioctls.h index 93c894dc515..c0c338bd406 100644 --- a/include/asm-x86/ioctls.h +++ b/include/asm-x86/ioctls.h @@ -47,12 +47,13 @@ #define TIOCSBRK 0x5427 /* BSD compatibility */ #define TIOCCBRK 0x5428 /* BSD compatibility */ #define TIOCGSID 0x5429 /* Return the session ID of FD */ -#define TCGETS2 _IOR('T',0x2A, struct termios2) -#define TCSETS2 _IOW('T',0x2B, struct termios2) -#define TCSETSW2 _IOW('T',0x2C, struct termios2) -#define TCSETSF2 _IOW('T',0x2D, struct termios2) -#define TIOCGPTN _IOR('T',0x30, unsigned int) /* Get Pty Number (of pty-mux device) */ -#define TIOCSPTLCK _IOW('T',0x31, int) /* Lock/unlock Pty */ +#define TCGETS2 _IOR('T', 0x2A, struct termios2) +#define TCSETS2 _IOW('T', 0x2B, struct termios2) +#define TCSETSW2 _IOW('T', 0x2C, struct termios2) +#define TCSETSF2 _IOW('T', 0x2D, struct termios2) +#define TIOCGPTN _IOR('T', 0x30, unsigned int) + /* Get Pty Number (of pty-mux device) */ +#define TIOCSPTLCK _IOW('T', 0x31, int) /* Lock/unlock Pty */ #define FIONCLEX 0x5450 #define FIOCLEX 0x5451 diff --git a/include/asm-x86/ipcbuf.h b/include/asm-x86/ipcbuf.h index 2adf8b39a40..ee678fd5159 100644 --- a/include/asm-x86/ipcbuf.h +++ b/include/asm-x86/ipcbuf.h @@ -11,8 +11,7 @@ * - 2 miscellaneous 32-bit values */ -struct ipc64_perm -{ +struct ipc64_perm { __kernel_key_t key; __kernel_uid32_t uid; __kernel_gid32_t gid; diff --git a/include/asm-x86/ipi.h b/include/asm-x86/ipi.h index 6d011bd6067..ecc80f341f3 100644 --- a/include/asm-x86/ipi.h +++ b/include/asm-x86/ipi.h @@ -27,7 +27,8 @@ * We use 'broadcast', CPU->CPU IPIs and self-IPIs too. */ -static inline unsigned int __prepare_ICR (unsigned int shortcut, int vector, unsigned int dest) +static inline unsigned int __prepare_ICR(unsigned int shortcut, int vector, + unsigned int dest) { unsigned int icr = shortcut | dest; @@ -42,12 +43,13 @@ static inline unsigned int __prepare_ICR (unsigned int shortcut, int vector, uns return icr; } -static inline int __prepare_ICR2 (unsigned int mask) +static inline int __prepare_ICR2(unsigned int mask) { return SET_APIC_DEST_FIELD(mask); } -static inline void __send_IPI_shortcut(unsigned int shortcut, int vector, unsigned int dest) +static inline void __send_IPI_shortcut(unsigned int shortcut, int vector, + unsigned int dest) { /* * Subtle. In the case of the 'never do double writes' workaround @@ -78,7 +80,8 @@ static inline void __send_IPI_shortcut(unsigned int shortcut, int vector, unsign * This is used to send an IPI with no shorthand notation (the destination is * specified in bits 56 to 63 of the ICR). */ -static inline void __send_IPI_dest_field(unsigned int mask, int vector, unsigned int dest) +static inline void __send_IPI_dest_field(unsigned int mask, int vector, + unsigned int dest) { unsigned long cfg; diff --git a/include/asm-x86/irq_32.h b/include/asm-x86/irq_32.h index aca9c96e8e6..0b79f318524 100644 --- a/include/asm-x86/irq_32.h +++ b/include/asm-x86/irq_32.h @@ -15,7 +15,7 @@ #include "irq_vectors.h" #include <asm/thread_info.h> -static __inline__ int irq_canonicalize(int irq) +static inline int irq_canonicalize(int irq) { return ((irq == 2) ? 9 : irq); } diff --git a/include/asm-x86/irq_64.h b/include/asm-x86/irq_64.h index 5006c6e7565..083d35a62c9 100644 --- a/include/asm-x86/irq_64.h +++ b/include/asm-x86/irq_64.h @@ -31,10 +31,10 @@ #define FIRST_SYSTEM_VECTOR 0xef /* duplicated in hw_irq.h */ -#define NR_IRQS (NR_VECTORS + (32 *NR_CPUS)) +#define NR_IRQS (NR_VECTORS + (32 * NR_CPUS)) #define NR_IRQ_VECTORS NR_IRQS -static __inline__ int irq_canonicalize(int irq) +static inline int irq_canonicalize(int irq) { return ((irq == 2) ? 9 : irq); } diff --git a/include/asm-x86/irqflags.h b/include/asm-x86/irqflags.h index 0e2292483b3..c242527f970 100644 --- a/include/asm-x86/irqflags.h +++ b/include/asm-x86/irqflags.h @@ -12,25 +12,21 @@ static inline unsigned long native_save_fl(void) { unsigned long flags; - __asm__ __volatile__( - "# __raw_save_flags\n\t" - "pushf ; pop %0" - : "=g" (flags) - : /* no input */ - : "memory" - ); + asm volatile("# __raw_save_flags\n\t" + "pushf ; pop %0" + : "=g" (flags) + : /* no input */ + : "memory"); return flags; } static inline void native_restore_fl(unsigned long flags) { - __asm__ __volatile__( - "push %0 ; popf" - : /* no output */ - :"g" (flags) - :"memory", "cc" - ); + asm volatile("push %0 ; popf" + : /* no output */ + :"g" (flags) + :"memory", "cc"); } static inline void native_irq_disable(void) @@ -70,26 +66,6 @@ static inline void raw_local_irq_restore(unsigned long flags) native_restore_fl(flags); } -#ifdef CONFIG_X86_VSMP - -/* - * Interrupt control for the VSMP architecture: - */ - -static inline void raw_local_irq_disable(void) -{ - unsigned long flags = __raw_local_save_flags(); - raw_local_irq_restore((flags & ~X86_EFLAGS_IF) | X86_EFLAGS_AC); -} - -static inline void raw_local_irq_enable(void) -{ - unsigned long flags = __raw_local_save_flags(); - raw_local_irq_restore((flags | X86_EFLAGS_IF) & (~X86_EFLAGS_AC)); -} - -#else - static inline void raw_local_irq_disable(void) { native_irq_disable(); @@ -100,8 +76,6 @@ static inline void raw_local_irq_enable(void) native_irq_enable(); } -#endif - /* * Used in the idle loop; sti takes one instruction cycle * to complete: @@ -153,23 +127,16 @@ static inline unsigned long __raw_local_irq_save(void) #endif /* CONFIG_PARAVIRT */ #ifndef __ASSEMBLY__ -#define raw_local_save_flags(flags) \ - do { (flags) = __raw_local_save_flags(); } while (0) +#define raw_local_save_flags(flags) \ + do { (flags) = __raw_local_save_flags(); } while (0) -#define raw_local_irq_save(flags) \ - do { (flags) = __raw_local_irq_save(); } while (0) +#define raw_local_irq_save(flags) \ + do { (flags) = __raw_local_irq_save(); } while (0) -#ifdef CONFIG_X86_VSMP -static inline int raw_irqs_disabled_flags(unsigned long flags) -{ - return !(flags & X86_EFLAGS_IF) || (flags & X86_EFLAGS_AC); -} -#else static inline int raw_irqs_disabled_flags(unsigned long flags) { return !(flags & X86_EFLAGS_IF); } -#endif static inline int raw_irqs_disabled(void) { diff --git a/include/asm-x86/kdebug.h b/include/asm-x86/kdebug.h index 99dcbafa151..96651bb59ba 100644 --- a/include/asm-x86/kdebug.h +++ b/include/asm-x86/kdebug.h @@ -20,15 +20,16 @@ enum die_val { DIE_CALL, DIE_NMI_IPI, DIE_PAGE_FAULT, + DIE_NMIUNKNOWN, }; extern void printk_address(unsigned long address, int reliable); -extern void die(const char *,struct pt_regs *,long); +extern void die(const char *, struct pt_regs *,long); extern int __must_check __die(const char *, struct pt_regs *, long); extern void show_registers(struct pt_regs *regs); extern void __show_registers(struct pt_regs *, int all); extern void show_trace(struct task_struct *t, struct pt_regs *regs, - unsigned long *sp, unsigned long bp); + unsigned long *sp, unsigned long bp); extern void __show_regs(struct pt_regs *regs); extern void show_regs(struct pt_regs *regs); extern unsigned long oops_begin(void); diff --git a/include/asm-x86/kexec.h b/include/asm-x86/kexec.h index c90d3c77afc..8f855a15f64 100644 --- a/include/asm-x86/kexec.h +++ b/include/asm-x86/kexec.h @@ -94,10 +94,9 @@ static inline void crash_fixup_ss_esp(struct pt_regs *newregs, { #ifdef CONFIG_X86_32 newregs->sp = (unsigned long)&(oldregs->sp); - __asm__ __volatile__( - "xorl %%eax, %%eax\n\t" - "movw %%ss, %%ax\n\t" - :"=a"(newregs->ss)); + asm volatile("xorl %%eax, %%eax\n\t" + "movw %%ss, %%ax\n\t" + :"=a"(newregs->ss)); #endif } @@ -114,39 +113,39 @@ static inline void crash_setup_regs(struct pt_regs *newregs, crash_fixup_ss_esp(newregs, oldregs); } else { #ifdef CONFIG_X86_32 - __asm__ __volatile__("movl %%ebx,%0" : "=m"(newregs->bx)); - __asm__ __volatile__("movl %%ecx,%0" : "=m"(newregs->cx)); - __asm__ __volatile__("movl %%edx,%0" : "=m"(newregs->dx)); - __asm__ __volatile__("movl %%esi,%0" : "=m"(newregs->si)); - __asm__ __volatile__("movl %%edi,%0" : "=m"(newregs->di)); - __asm__ __volatile__("movl %%ebp,%0" : "=m"(newregs->bp)); - __asm__ __volatile__("movl %%eax,%0" : "=m"(newregs->ax)); - __asm__ __volatile__("movl %%esp,%0" : "=m"(newregs->sp)); - __asm__ __volatile__("movl %%ss, %%eax;" :"=a"(newregs->ss)); - __asm__ __volatile__("movl %%cs, %%eax;" :"=a"(newregs->cs)); - __asm__ __volatile__("movl %%ds, %%eax;" :"=a"(newregs->ds)); - __asm__ __volatile__("movl %%es, %%eax;" :"=a"(newregs->es)); - __asm__ __volatile__("pushfl; popl %0" :"=m"(newregs->flags)); + asm volatile("movl %%ebx,%0" : "=m"(newregs->bx)); + asm volatile("movl %%ecx,%0" : "=m"(newregs->cx)); + asm volatile("movl %%edx,%0" : "=m"(newregs->dx)); + asm volatile("movl %%esi,%0" : "=m"(newregs->si)); + asm volatile("movl %%edi,%0" : "=m"(newregs->di)); + asm volatile("movl %%ebp,%0" : "=m"(newregs->bp)); + asm volatile("movl %%eax,%0" : "=m"(newregs->ax)); + asm volatile("movl %%esp,%0" : "=m"(newregs->sp)); + asm volatile("movl %%ss, %%eax;" :"=a"(newregs->ss)); + asm volatile("movl %%cs, %%eax;" :"=a"(newregs->cs)); + asm volatile("movl %%ds, %%eax;" :"=a"(newregs->ds)); + asm volatile("movl %%es, %%eax;" :"=a"(newregs->es)); + asm volatile("pushfl; popl %0" :"=m"(newregs->flags)); #else - __asm__ __volatile__("movq %%rbx,%0" : "=m"(newregs->bx)); - __asm__ __volatile__("movq %%rcx,%0" : "=m"(newregs->cx)); - __asm__ __volatile__("movq %%rdx,%0" : "=m"(newregs->dx)); - __asm__ __volatile__("movq %%rsi,%0" : "=m"(newregs->si)); - __asm__ __volatile__("movq %%rdi,%0" : "=m"(newregs->di)); - __asm__ __volatile__("movq %%rbp,%0" : "=m"(newregs->bp)); - __asm__ __volatile__("movq %%rax,%0" : "=m"(newregs->ax)); - __asm__ __volatile__("movq %%rsp,%0" : "=m"(newregs->sp)); - __asm__ __volatile__("movq %%r8,%0" : "=m"(newregs->r8)); - __asm__ __volatile__("movq %%r9,%0" : "=m"(newregs->r9)); - __asm__ __volatile__("movq %%r10,%0" : "=m"(newregs->r10)); - __asm__ __volatile__("movq %%r11,%0" : "=m"(newregs->r11)); - __asm__ __volatile__("movq %%r12,%0" : "=m"(newregs->r12)); - __asm__ __volatile__("movq %%r13,%0" : "=m"(newregs->r13)); - __asm__ __volatile__("movq %%r14,%0" : "=m"(newregs->r14)); - __asm__ __volatile__("movq %%r15,%0" : "=m"(newregs->r15)); - __asm__ __volatile__("movl %%ss, %%eax;" :"=a"(newregs->ss)); - __asm__ __volatile__("movl %%cs, %%eax;" :"=a"(newregs->cs)); - __asm__ __volatile__("pushfq; popq %0" :"=m"(newregs->flags)); + asm volatile("movq %%rbx,%0" : "=m"(newregs->bx)); + asm volatile("movq %%rcx,%0" : "=m"(newregs->cx)); + asm volatile("movq %%rdx,%0" : "=m"(newregs->dx)); + asm volatile("movq %%rsi,%0" : "=m"(newregs->si)); + asm volatile("movq %%rdi,%0" : "=m"(newregs->di)); + asm volatile("movq %%rbp,%0" : "=m"(newregs->bp)); + asm volatile("movq %%rax,%0" : "=m"(newregs->ax)); + asm volatile("movq %%rsp,%0" : "=m"(newregs->sp)); + asm volatile("movq %%r8,%0" : "=m"(newregs->r8)); + asm volatile("movq %%r9,%0" : "=m"(newregs->r9)); + asm volatile("movq %%r10,%0" : "=m"(newregs->r10)); + asm volatile("movq %%r11,%0" : "=m"(newregs->r11)); + asm volatile("movq %%r12,%0" : "=m"(newregs->r12)); + asm volatile("movq %%r13,%0" : "=m"(newregs->r13)); + asm volatile("movq %%r14,%0" : "=m"(newregs->r14)); + asm volatile("movq %%r15,%0" : "=m"(newregs->r15)); + asm volatile("movl %%ss, %%eax;" :"=a"(newregs->ss)); + asm volatile("movl %%cs, %%eax;" :"=a"(newregs->cs)); + asm volatile("pushfq; popq %0" :"=m"(newregs->flags)); #endif newregs->ip = (unsigned long)current_text_addr(); } diff --git a/include/asm-x86/kgdb.h b/include/asm-x86/kgdb.h new file mode 100644 index 00000000000..484c47554f3 --- /dev/null +++ b/include/asm-x86/kgdb.h @@ -0,0 +1,81 @@ +#ifndef _ASM_KGDB_H_ +#define _ASM_KGDB_H_ + +/* + * Copyright (C) 2001-2004 Amit S. Kale + * Copyright (C) 2008 Wind River Systems, Inc. + */ + +/* + * BUFMAX defines the maximum number of characters in inbound/outbound + * buffers at least NUMREGBYTES*2 are needed for register packets + * Longer buffer is needed to list all threads + */ +#define BUFMAX 1024 + +/* + * Note that this register image is in a different order than + * the register image that Linux produces at interrupt time. + * + * Linux's register image is defined by struct pt_regs in ptrace.h. + * Just why GDB uses a different order is a historical mystery. + */ +#ifdef CONFIG_X86_32 +enum regnames { + GDB_AX, /* 0 */ + GDB_CX, /* 1 */ + GDB_DX, /* 2 */ + GDB_BX, /* 3 */ + GDB_SP, /* 4 */ + GDB_BP, /* 5 */ + GDB_SI, /* 6 */ + GDB_DI, /* 7 */ + GDB_PC, /* 8 also known as eip */ + GDB_PS, /* 9 also known as eflags */ + GDB_CS, /* 10 */ + GDB_SS, /* 11 */ + GDB_DS, /* 12 */ + GDB_ES, /* 13 */ + GDB_FS, /* 14 */ + GDB_GS, /* 15 */ +}; +#else /* ! CONFIG_X86_32 */ +enum regnames { + GDB_AX, /* 0 */ + GDB_DX, /* 1 */ + GDB_CX, /* 2 */ + GDB_BX, /* 3 */ + GDB_SI, /* 4 */ + GDB_DI, /* 5 */ + GDB_BP, /* 6 */ + GDB_SP, /* 7 */ + GDB_R8, /* 8 */ + GDB_R9, /* 9 */ + GDB_R10, /* 10 */ + GDB_R11, /* 11 */ + GDB_R12, /* 12 */ + GDB_R13, /* 13 */ + GDB_R14, /* 14 */ + GDB_R15, /* 15 */ + GDB_PC, /* 16 */ + GDB_PS, /* 17 */ +}; +#endif /* CONFIG_X86_32 */ + +/* + * Number of bytes of registers: + */ +#ifdef CONFIG_X86_32 +# define NUMREGBYTES 64 +#else +# define NUMREGBYTES ((GDB_PS+1)*8) +#endif + +static inline void arch_kgdb_breakpoint(void) +{ + asm(" int $3"); +} +#define BREAK_INSTR_SIZE 1 +#define CACHE_FLUSH_IS_SAFE 1 + +#endif /* _ASM_KGDB_H_ */ diff --git a/include/asm-x86/kprobes.h b/include/asm-x86/kprobes.h index 61ad7b5d142..54980b0b389 100644 --- a/include/asm-x86/kprobes.h +++ b/include/asm-x86/kprobes.h @@ -35,12 +35,12 @@ typedef u8 kprobe_opcode_t; #define RELATIVEJUMP_INSTRUCTION 0xe9 #define MAX_INSN_SIZE 16 #define MAX_STACK_SIZE 64 -#define MIN_STACK_SIZE(ADDR) (((MAX_STACK_SIZE) < \ - (((unsigned long)current_thread_info()) + THREAD_SIZE \ - - (unsigned long)(ADDR))) \ - ? (MAX_STACK_SIZE) \ - : (((unsigned long)current_thread_info()) + THREAD_SIZE \ - - (unsigned long)(ADDR))) +#define MIN_STACK_SIZE(ADDR) \ + (((MAX_STACK_SIZE) < (((unsigned long)current_thread_info()) + \ + THREAD_SIZE - (unsigned long)(ADDR))) \ + ? (MAX_STACK_SIZE) \ + : (((unsigned long)current_thread_info()) + \ + THREAD_SIZE - (unsigned long)(ADDR))) #define flush_insn_slot(p) do { } while (0) diff --git a/include/asm-x86/kvm.h b/include/asm-x86/kvm.h index 7a71120426a..80eefef2cc7 100644 --- a/include/asm-x86/kvm.h +++ b/include/asm-x86/kvm.h @@ -188,4 +188,45 @@ struct kvm_cpuid2 { struct kvm_cpuid_entry2 entries[0]; }; +/* for KVM_GET_PIT and KVM_SET_PIT */ +struct kvm_pit_channel_state { + __u32 count; /* can be 65536 */ + __u16 latched_count; + __u8 count_latched; + __u8 status_latched; + __u8 status; + __u8 read_state; + __u8 write_state; + __u8 write_latch; + __u8 rw_mode; + __u8 mode; + __u8 bcd; + __u8 gate; + __s64 count_load_time; +}; + +struct kvm_pit_state { + struct kvm_pit_channel_state channels[3]; +}; + +#define KVM_TRC_INJ_VIRQ (KVM_TRC_HANDLER + 0x02) +#define KVM_TRC_REDELIVER_EVT (KVM_TRC_HANDLER + 0x03) +#define KVM_TRC_PEND_INTR (KVM_TRC_HANDLER + 0x04) +#define KVM_TRC_IO_READ (KVM_TRC_HANDLER + 0x05) +#define KVM_TRC_IO_WRITE (KVM_TRC_HANDLER + 0x06) +#define KVM_TRC_CR_READ (KVM_TRC_HANDLER + 0x07) +#define KVM_TRC_CR_WRITE (KVM_TRC_HANDLER + 0x08) +#define KVM_TRC_DR_READ (KVM_TRC_HANDLER + 0x09) +#define KVM_TRC_DR_WRITE (KVM_TRC_HANDLER + 0x0A) +#define KVM_TRC_MSR_READ (KVM_TRC_HANDLER + 0x0B) +#define KVM_TRC_MSR_WRITE (KVM_TRC_HANDLER + 0x0C) +#define KVM_TRC_CPUID (KVM_TRC_HANDLER + 0x0D) +#define KVM_TRC_INTR (KVM_TRC_HANDLER + 0x0E) +#define KVM_TRC_NMI (KVM_TRC_HANDLER + 0x0F) +#define KVM_TRC_VMMCALL (KVM_TRC_HANDLER + 0x10) +#define KVM_TRC_HLT (KVM_TRC_HANDLER + 0x11) +#define KVM_TRC_CLTS (KVM_TRC_HANDLER + 0x12) +#define KVM_TRC_LMSW (KVM_TRC_HANDLER + 0x13) +#define KVM_TRC_APIC_ACCESS (KVM_TRC_HANDLER + 0x14) + #endif diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h index 4702b04b979..9d963cd6533 100644 --- a/include/asm-x86/kvm_host.h +++ b/include/asm-x86/kvm_host.h @@ -20,17 +20,25 @@ #include <asm/desc.h> +#define KVM_MAX_VCPUS 16 +#define KVM_MEMORY_SLOTS 32 +/* memory slots that does not exposed to userspace */ +#define KVM_PRIVATE_MEM_SLOTS 4 + +#define KVM_PIO_PAGE_OFFSET 1 + #define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) #define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) -#define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) +#define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS | \ + 0xFFFFFF0000000000ULL) -#define KVM_GUEST_CR0_MASK \ +#define KVM_GUEST_CR0_MASK \ (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE \ | X86_CR0_NW | X86_CR0_CD) -#define KVM_VM_CR0_ALWAYS_ON \ +#define KVM_VM_CR0_ALWAYS_ON \ (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE | X86_CR0_TS \ | X86_CR0_MP) -#define KVM_GUEST_CR4_MASK \ +#define KVM_GUEST_CR4_MASK \ (X86_CR4_VME | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE | X86_CR4_VMXE) #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) @@ -38,6 +46,13 @@ #define INVALID_PAGE (~(hpa_t)0) #define UNMAPPED_GVA (~(gpa_t)0) +/* shadow tables are PAE even on non-PAE hosts */ +#define KVM_HPAGE_SHIFT 21 +#define KVM_HPAGE_SIZE (1UL << KVM_HPAGE_SHIFT) +#define KVM_HPAGE_MASK (~(KVM_HPAGE_SIZE - 1)) + +#define KVM_PAGES_PER_HPAGE (KVM_HPAGE_SIZE / PAGE_SIZE) + #define DE_VECTOR 0 #define UD_VECTOR 6 #define NM_VECTOR 7 @@ -47,6 +62,7 @@ #define SS_VECTOR 12 #define GP_VECTOR 13 #define PF_VECTOR 14 +#define MC_VECTOR 18 #define SELECTOR_TI_MASK (1 << 2) #define SELECTOR_RPL_MASK 0x03 @@ -57,7 +73,8 @@ #define KVM_PERMILLE_MMU_PAGES 20 #define KVM_MIN_ALLOC_MMU_PAGES 64 -#define KVM_NUM_MMU_PAGES 1024 +#define KVM_MMU_HASH_SHIFT 10 +#define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT) #define KVM_MIN_FREE_MMU_PAGES 5 #define KVM_REFILL_PAGES 25 #define KVM_MAX_CPUID_ENTRIES 40 @@ -105,6 +122,12 @@ enum { #define KVM_NR_MEM_OBJS 40 +struct kvm_guest_debug { + int enabled; + unsigned long bp[4]; + int singlestep; +}; + /* * We don't want allocation failures within the mmu code, so we preallocate * enough memory for a single page fault in a cache. @@ -133,12 +156,13 @@ struct kvm_pte_chain { union kvm_mmu_page_role { unsigned word; struct { - unsigned glevels : 4; - unsigned level : 4; - unsigned quadrant : 2; - unsigned pad_for_nice_hex_output : 6; - unsigned metaphysical : 1; - unsigned access : 3; + unsigned glevels:4; + unsigned level:4; + unsigned quadrant:2; + unsigned pad_for_nice_hex_output:6; + unsigned metaphysical:1; + unsigned access:3; + unsigned invalid:1; }; }; @@ -203,11 +227,6 @@ struct kvm_vcpu_arch { u64 shadow_efer; u64 apic_base; struct kvm_lapic *apic; /* kernel irqchip context */ -#define VCPU_MP_STATE_RUNNABLE 0 -#define VCPU_MP_STATE_UNINITIALIZED 1 -#define VCPU_MP_STATE_INIT_RECEIVED 2 -#define VCPU_MP_STATE_SIPI_RECEIVED 3 -#define VCPU_MP_STATE_HALTED 4 int mp_state; int sipi_vector; u64 ia32_misc_enable_msr; @@ -225,8 +244,9 @@ struct kvm_vcpu_arch { u64 *last_pte_updated; struct { - gfn_t gfn; /* presumed gfn during guest pte update */ - struct page *page; /* page corresponding to that gfn */ + gfn_t gfn; /* presumed gfn during guest pte update */ + pfn_t pfn; /* pfn corresponding to that gfn */ + int largepage; } update_pte; struct i387_fxsave_struct host_fx_image; @@ -260,6 +280,11 @@ struct kvm_vcpu_arch { /* emulate context */ struct x86_emulate_ctxt emulate_ctxt; + + gpa_t time; + struct kvm_vcpu_time_info hv_clock; + unsigned int time_offset; + struct page *time_page; }; struct kvm_mem_alias { @@ -282,10 +307,13 @@ struct kvm_arch{ struct list_head active_mmu_pages; struct kvm_pic *vpic; struct kvm_ioapic *vioapic; + struct kvm_pit *vpit; int round_robin_prev_vcpu; unsigned int tss_addr; struct page *apic_access_page; + + gpa_t wall_clock; }; struct kvm_vm_stat { @@ -297,6 +325,7 @@ struct kvm_vm_stat { u32 mmu_recycled; u32 mmu_cache_miss; u32 remote_tlb_flush; + u32 lpages; }; struct kvm_vcpu_stat { @@ -319,6 +348,7 @@ struct kvm_vcpu_stat { u32 fpu_reload; u32 insn_emulation; u32 insn_emulation_fail; + u32 hypercalls; }; struct descriptor_table { @@ -354,6 +384,7 @@ struct kvm_x86_ops { u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); void (*get_segment)(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); + int (*get_cpl)(struct kvm_vcpu *vcpu); void (*set_segment)(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); @@ -409,6 +440,15 @@ void kvm_mmu_zap_all(struct kvm *kvm); unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm); void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); + +int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, + const void *val, int bytes); +int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes, + gpa_t addr, unsigned long *ret); + +extern bool tdp_enabled; + enum emulation_result { EMULATE_DONE, /* no further processing */ EMULATE_DO_MMIO, /* kvm_run filled with mmio request */ @@ -428,6 +468,7 @@ void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr); void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long value, unsigned long *rflags); +void kvm_enable_efer_bits(u64); int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *data); int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); @@ -447,12 +488,14 @@ int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value); -void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr0); -unsigned long get_cr8(struct kvm_vcpu *vcpu); -void lmsw(struct kvm_vcpu *vcpu, unsigned long msw); +int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason); + +void kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); +void kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3); +void kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); +void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); +unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); +void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw); void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); @@ -490,6 +533,8 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu); int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); +void kvm_enable_tdp(void); + int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); int complete_pio(struct kvm_vcpu *vcpu); @@ -599,6 +644,7 @@ static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) #define ASM_VMX_VMWRITE_RSP_RDX ".byte 0x0f, 0x79, 0xd4" #define ASM_VMX_VMXOFF ".byte 0x0f, 0x01, 0xc4" #define ASM_VMX_VMXON_RAX ".byte 0xf3, 0x0f, 0xc7, 0x30" +#define ASM_VMX_INVVPID ".byte 0x66, 0x0f, 0x38, 0x81, 0x08" #define MSR_IA32_TIME_STAMP_COUNTER 0x010 @@ -606,6 +652,33 @@ static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) #define TSS_BASE_SIZE 0x68 #define TSS_IOPB_SIZE (65536 / 8) #define TSS_REDIRECTION_SIZE (256 / 8) -#define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) +#define RMODE_TSS_SIZE \ + (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) + +enum { + TASK_SWITCH_CALL = 0, + TASK_SWITCH_IRET = 1, + TASK_SWITCH_JMP = 2, + TASK_SWITCH_GATE = 3, +}; + +#define KVMTRACE_5D(evt, vcpu, d1, d2, d3, d4, d5, name) \ + trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \ + vcpu, 5, d1, d2, d3, d4, d5) +#define KVMTRACE_4D(evt, vcpu, d1, d2, d3, d4, name) \ + trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \ + vcpu, 4, d1, d2, d3, d4, 0) +#define KVMTRACE_3D(evt, vcpu, d1, d2, d3, name) \ + trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \ + vcpu, 3, d1, d2, d3, 0, 0) +#define KVMTRACE_2D(evt, vcpu, d1, d2, name) \ + trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \ + vcpu, 2, d1, d2, 0, 0, 0) +#define KVMTRACE_1D(evt, vcpu, d1, name) \ + trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \ + vcpu, 1, d1, 0, 0, 0, 0) +#define KVMTRACE_0D(evt, vcpu, name) \ + trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \ + vcpu, 0, 0, 0, 0, 0, 0) #endif diff --git a/include/asm-x86/kvm_para.h b/include/asm-x86/kvm_para.h index c6f3fd8d8c5..50984594207 100644 --- a/include/asm-x86/kvm_para.h +++ b/include/asm-x86/kvm_para.h @@ -10,10 +10,65 @@ * paravirtualization, the appropriate feature bit should be checked. */ #define KVM_CPUID_FEATURES 0x40000001 +#define KVM_FEATURE_CLOCKSOURCE 0 +#define KVM_FEATURE_NOP_IO_DELAY 1 +#define KVM_FEATURE_MMU_OP 2 + +#define MSR_KVM_WALL_CLOCK 0x11 +#define MSR_KVM_SYSTEM_TIME 0x12 + +#define KVM_MAX_MMU_OP_BATCH 32 + +/* Operations for KVM_HC_MMU_OP */ +#define KVM_MMU_OP_WRITE_PTE 1 +#define KVM_MMU_OP_FLUSH_TLB 2 +#define KVM_MMU_OP_RELEASE_PT 3 + +/* Payload for KVM_HC_MMU_OP */ +struct kvm_mmu_op_header { + __u32 op; + __u32 pad; +}; + +struct kvm_mmu_op_write_pte { + struct kvm_mmu_op_header header; + __u64 pte_phys; + __u64 pte_val; +}; + +struct kvm_mmu_op_flush_tlb { + struct kvm_mmu_op_header header; +}; + +struct kvm_mmu_op_release_pt { + struct kvm_mmu_op_header header; + __u64 pt_phys; +}; #ifdef __KERNEL__ #include <asm/processor.h> +/* xen binary-compatible interface. See xen headers for details */ +struct kvm_vcpu_time_info { + uint32_t version; + uint32_t pad0; + uint64_t tsc_timestamp; + uint64_t system_time; + uint32_t tsc_to_system_mul; + int8_t tsc_shift; + int8_t pad[3]; +} __attribute__((__packed__)); /* 32 bytes */ + +struct kvm_wall_clock { + uint32_t wc_version; + uint32_t wc_sec; + uint32_t wc_nsec; +} __attribute__((__packed__)); + + +extern void kvmclock_init(void); + + /* This instruction is vmcall. On non-VT architectures, it will generate a * trap that we will then rewrite to the appropriate instruction. */ diff --git a/include/asm-x86/kvm_x86_emulate.h b/include/asm-x86/kvm_x86_emulate.h index 7db91b9bdcd..d6337f941c9 100644 --- a/include/asm-x86/kvm_x86_emulate.h +++ b/include/asm-x86/kvm_x86_emulate.h @@ -68,10 +68,10 @@ struct x86_emulate_ops { * @val: [OUT] Value read from memory, zero-extended to 'u_long'. * @bytes: [IN ] Number of bytes to read from memory. */ - int (*read_emulated) (unsigned long addr, - void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu); + int (*read_emulated)(unsigned long addr, + void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu); /* * write_emulated: Read bytes from emulated/special memory area. @@ -80,10 +80,10 @@ struct x86_emulate_ops { * required). * @bytes: [IN ] Number of bytes to write to memory. */ - int (*write_emulated) (unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu); + int (*write_emulated)(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu); /* * cmpxchg_emulated: Emulate an atomic (LOCKed) CMPXCHG operation on an @@ -93,11 +93,11 @@ struct x86_emulate_ops { * @new: [IN ] Value to write to @addr. * @bytes: [IN ] Number of bytes to access using CMPXCHG. */ - int (*cmpxchg_emulated) (unsigned long addr, - const void *old, - const void *new, - unsigned int bytes, - struct kvm_vcpu *vcpu); + int (*cmpxchg_emulated)(unsigned long addr, + const void *old, + const void *new, + unsigned int bytes, + struct kvm_vcpu *vcpu); }; @@ -143,7 +143,7 @@ struct x86_emulate_ctxt { /* Register state before/after emulation. */ struct kvm_vcpu *vcpu; - /* Linear faulting address (if emulating a page-faulting instruction). */ + /* Linear faulting address (if emulating a page-faulting instruction) */ unsigned long eflags; /* Emulated execution mode, represented by an X86EMUL_MODE value. */ diff --git a/include/asm-x86/lguest.h b/include/asm-x86/lguest.h index 9b17571e9bc..be4a7247fa2 100644 --- a/include/asm-x86/lguest.h +++ b/include/asm-x86/lguest.h @@ -34,8 +34,7 @@ extern const char lgstart_iret[], lgend_iret[]; extern void lguest_iret(void); extern void lguest_init(void); -struct lguest_regs -{ +struct lguest_regs { /* Manually saved part. */ unsigned long eax, ebx, ecx, edx; unsigned long esi, edi, ebp; @@ -51,8 +50,7 @@ struct lguest_regs }; /* This is a guest-specific page (mapped ro) into the guest. */ -struct lguest_ro_state -{ +struct lguest_ro_state { /* Host information we need to restore when we switch back. */ u32 host_cr3; struct desc_ptr host_idt_desc; @@ -67,8 +65,7 @@ struct lguest_ro_state struct desc_struct guest_gdt[GDT_ENTRIES]; }; -struct lg_cpu_arch -{ +struct lg_cpu_arch { /* The GDT entries copied into lguest_ro_state when running. */ struct desc_struct gdt[GDT_ENTRIES]; @@ -85,7 +82,7 @@ static inline void lguest_set_ts(void) cr0 = read_cr0(); if (!(cr0 & 8)) - write_cr0(cr0|8); + write_cr0(cr0 | 8); } /* Full 4G segment descriptors, suitable for CS and DS. */ diff --git a/include/asm-x86/lguest_hcall.h b/include/asm-x86/lguest_hcall.h index f239e7069ca..a3241f28e34 100644 --- a/include/asm-x86/lguest_hcall.h +++ b/include/asm-x86/lguest_hcall.h @@ -46,7 +46,7 @@ hcall(unsigned long call, { /* "int" is the Intel instruction to trigger a trap. */ asm volatile("int $" __stringify(LGUEST_TRAP_ENTRY) - /* The call in %eax (aka "a") might be overwritten */ + /* The call in %eax (aka "a") might be overwritten */ : "=a"(call) /* The arguments are in %eax, %edx, %ebx & %ecx */ : "a"(call), "d"(arg1), "b"(arg2), "c"(arg3) @@ -62,8 +62,7 @@ hcall(unsigned long call, #define LGUEST_IRQS (NR_IRQS < 32 ? NR_IRQS: 32) #define LHCALL_RING_SIZE 64 -struct hcall_args -{ +struct hcall_args { /* These map directly onto eax, ebx, ecx, edx in struct lguest_regs */ unsigned long arg0, arg2, arg3, arg1; }; diff --git a/include/asm-x86/linkage.h b/include/asm-x86/linkage.h index c048353f4b8..64e444f8e85 100644 --- a/include/asm-x86/linkage.h +++ b/include/asm-x86/linkage.h @@ -1,6 +1,9 @@ #ifndef __ASM_LINKAGE_H #define __ASM_LINKAGE_H +#undef notrace +#define notrace __attribute__((no_instrument_function)) + #ifdef CONFIG_X86_64 #define __ALIGN .p2align 4,,15 #define __ALIGN_STR ".p2align 4,,15" diff --git a/include/asm-x86/local.h b/include/asm-x86/local.h index f852c62b331..330a72496ab 100644 --- a/include/asm-x86/local.h +++ b/include/asm-x86/local.h @@ -18,32 +18,28 @@ typedef struct { static inline void local_inc(local_t *l) { - __asm__ __volatile__( - _ASM_INC "%0" - :"+m" (l->a.counter)); + asm volatile(_ASM_INC "%0" + : "+m" (l->a.counter)); } static inline void local_dec(local_t *l) { - __asm__ __volatile__( - _ASM_DEC "%0" - :"+m" (l->a.counter)); + asm volatile(_ASM_DEC "%0" + : "+m" (l->a.counter)); } static inline void local_add(long i, local_t *l) { - __asm__ __volatile__( - _ASM_ADD "%1,%0" - :"+m" (l->a.counter) - :"ir" (i)); + asm volatile(_ASM_ADD "%1,%0" + : "+m" (l->a.counter) + : "ir" (i)); } static inline void local_sub(long i, local_t *l) { - __asm__ __volatile__( - _ASM_SUB "%1,%0" - :"+m" (l->a.counter) - :"ir" (i)); + asm volatile(_ASM_SUB "%1,%0" + : "+m" (l->a.counter) + : "ir" (i)); } /** @@ -59,10 +55,9 @@ static inline int local_sub_and_test(long i, local_t *l) { unsigned char c; - __asm__ __volatile__( - _ASM_SUB "%2,%0; sete %1" - :"+m" (l->a.counter), "=qm" (c) - :"ir" (i) : "memory"); + asm volatile(_ASM_SUB "%2,%0; sete %1" + : "+m" (l->a.counter), "=qm" (c) + : "ir" (i) : "memory"); return c; } @@ -78,10 +73,9 @@ static inline int local_dec_and_test(local_t *l) { unsigned char c; - __asm__ __volatile__( - _ASM_DEC "%0; sete %1" - :"+m" (l->a.counter), "=qm" (c) - : : "memory"); + asm volatile(_ASM_DEC "%0; sete %1" + : "+m" (l->a.counter), "=qm" (c) + : : "memory"); return c != 0; } @@ -97,10 +91,9 @@ static inline int local_inc_and_test(local_t *l) { unsigned char c; - __asm__ __volatile__( - _ASM_INC "%0; sete %1" - :"+m" (l->a.counter), "=qm" (c) - : : "memory"); + asm volatile(_ASM_INC "%0; sete %1" + : "+m" (l->a.counter), "=qm" (c) + : : "memory"); return c != 0; } @@ -117,10 +110,9 @@ static inline int local_add_negative(long i, local_t *l) { unsigned char c; - __asm__ __volatile__( - _ASM_ADD "%2,%0; sets %1" - :"+m" (l->a.counter), "=qm" (c) - :"ir" (i) : "memory"); + asm volatile(_ASM_ADD "%2,%0; sets %1" + : "+m" (l->a.counter), "=qm" (c) + : "ir" (i) : "memory"); return c; } @@ -141,10 +133,9 @@ static inline long local_add_return(long i, local_t *l) #endif /* Modern 486+ processor */ __i = i; - __asm__ __volatile__( - _ASM_XADD "%0, %1;" - :"+r" (i), "+m" (l->a.counter) - : : "memory"); + asm volatile(_ASM_XADD "%0, %1;" + : "+r" (i), "+m" (l->a.counter) + : : "memory"); return i + __i; #ifdef CONFIG_M386 @@ -182,11 +173,11 @@ static inline long local_sub_return(long i, local_t *l) #define local_add_unless(l, a, u) \ ({ \ long c, old; \ - c = local_read(l); \ + c = local_read((l)); \ for (;;) { \ if (unlikely(c == (u))) \ break; \ - old = local_cmpxchg((l), c, c + (a)); \ + old = local_cmpxchg((l), c, c + (a)); \ if (likely(old == c)) \ break; \ c = old; \ @@ -214,26 +205,30 @@ static inline long local_sub_return(long i, local_t *l) /* Need to disable preemption for the cpu local counters otherwise we could still access a variable of a previous CPU in a non atomic way. */ -#define cpu_local_wrap_v(l) \ - ({ local_t res__; \ - preempt_disable(); \ - res__ = (l); \ - preempt_enable(); \ - res__; }) +#define cpu_local_wrap_v(l) \ +({ \ + local_t res__; \ + preempt_disable(); \ + res__ = (l); \ + preempt_enable(); \ + res__; \ +}) #define cpu_local_wrap(l) \ - ({ preempt_disable(); \ - l; \ - preempt_enable(); }) \ - -#define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var(l))) -#define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i))) -#define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var(l))) -#define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var(l))) -#define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l))) -#define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l))) - -#define __cpu_local_inc(l) cpu_local_inc(l) -#define __cpu_local_dec(l) cpu_local_dec(l) +({ \ + preempt_disable(); \ + (l); \ + preempt_enable(); \ +}) \ + +#define cpu_local_read(l) cpu_local_wrap_v(local_read(&__get_cpu_var((l)))) +#define cpu_local_set(l, i) cpu_local_wrap(local_set(&__get_cpu_var((l)), (i))) +#define cpu_local_inc(l) cpu_local_wrap(local_inc(&__get_cpu_var((l)))) +#define cpu_local_dec(l) cpu_local_wrap(local_dec(&__get_cpu_var((l)))) +#define cpu_local_add(i, l) cpu_local_wrap(local_add((i), &__get_cpu_var((l)))) +#define cpu_local_sub(i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var((l)))) + +#define __cpu_local_inc(l) cpu_local_inc((l)) +#define __cpu_local_dec(l) cpu_local_dec((l)) #define __cpu_local_add(i, l) cpu_local_add((i), (l)) #define __cpu_local_sub(i, l) cpu_local_sub((i), (l)) diff --git a/include/asm-x86/mach-bigsmp/mach_apic.h b/include/asm-x86/mach-bigsmp/mach_apic.h index 6df235e8ea9..8327907c79b 100644 --- a/include/asm-x86/mach-bigsmp/mach_apic.h +++ b/include/asm-x86/mach-bigsmp/mach_apic.h @@ -1,10 +1,7 @@ #ifndef __ASM_MACH_APIC_H #define __ASM_MACH_APIC_H - -extern u8 bios_cpu_apicid[]; - -#define xapic_phys_to_log_apicid(cpu) (bios_cpu_apicid[cpu]) +#define xapic_phys_to_log_apicid(cpu) (per_cpu(x86_bios_cpu_apicid, cpu)) #define esr_disable (1) static inline int apic_id_registered(void) @@ -90,7 +87,7 @@ static inline int apicid_to_node(int logical_apicid) static inline int cpu_present_to_apicid(int mps_cpu) { if (mps_cpu < NR_CPUS) - return (int) bios_cpu_apicid[mps_cpu]; + return (int) per_cpu(x86_bios_cpu_apicid, mps_cpu); return BAD_APICID; } @@ -109,17 +106,6 @@ static inline int cpu_to_logical_apicid(int cpu) return cpu_physical_id(cpu); } -static inline int mpc_apic_id(struct mpc_config_processor *m, - struct mpc_config_translation *translation_record) -{ - printk("Processor #%d %u:%u APIC version %d\n", - m->mpc_apicid, - (m->mpc_cpufeature & CPU_FAMILY_MASK) >> 8, - (m->mpc_cpufeature & CPU_MODEL_MASK) >> 4, - m->mpc_apicver); - return m->mpc_apicid; -} - static inline physid_mask_t ioapic_phys_id_map(physid_mask_t phys_map) { /* For clustered we don't have a good way to do this yet - hack */ diff --git a/include/asm-x86/mach-default/mach_apic.h b/include/asm-x86/mach-default/mach_apic.h index e3c2c1012c1..21003b56ae9 100644 --- a/include/asm-x86/mach-default/mach_apic.h +++ b/include/asm-x86/mach-default/mach_apic.h @@ -1,6 +1,8 @@ #ifndef __ASM_MACH_APIC_H #define __ASM_MACH_APIC_H +#ifdef CONFIG_X86_LOCAL_APIC + #include <mach_apicdef.h> #include <asm/smp.h> @@ -14,24 +16,25 @@ static inline cpumask_t target_cpus(void) return cpumask_of_cpu(0); #endif } -#define TARGET_CPUS (target_cpus()) #define NO_BALANCE_IRQ (0) #define esr_disable (0) +#ifdef CONFIG_X86_64 +#include <asm/genapic.h> +#define INT_DELIVERY_MODE (genapic->int_delivery_mode) +#define INT_DEST_MODE (genapic->int_dest_mode) +#define TARGET_CPUS (genapic->target_cpus()) +#define apic_id_registered (genapic->apic_id_registered) +#define init_apic_ldr (genapic->init_apic_ldr) +#define cpu_mask_to_apicid (genapic->cpu_mask_to_apicid) +#define phys_pkg_id (genapic->phys_pkg_id) +#define vector_allocation_domain (genapic->vector_allocation_domain) +extern void setup_apic_routing(void); +#else #define INT_DELIVERY_MODE dest_LowestPrio #define INT_DEST_MODE 1 /* logical delivery broadcast to all procs */ - -static inline unsigned long check_apicid_used(physid_mask_t bitmap, int apicid) -{ - return physid_isset(apicid, bitmap); -} - -static inline unsigned long check_apicid_present(int bit) -{ - return physid_isset(bit, phys_cpu_present_map); -} - +#define TARGET_CPUS (target_cpus()) /* * Set up the logical destination ID. * @@ -49,23 +52,51 @@ static inline void init_apic_ldr(void) apic_write_around(APIC_LDR, val); } -static inline physid_mask_t ioapic_phys_id_map(physid_mask_t phys_map) +static inline int apic_id_registered(void) { - return phys_map; + return physid_isset(GET_APIC_ID(read_apic_id()), phys_cpu_present_map); +} + +static inline unsigned int cpu_mask_to_apicid(cpumask_t cpumask) +{ + return cpus_addr(cpumask)[0]; +} + +static inline u32 phys_pkg_id(u32 cpuid_apic, int index_msb) +{ + return cpuid_apic >> index_msb; } static inline void setup_apic_routing(void) { +#ifdef CONFIG_X86_IO_APIC printk("Enabling APIC mode: %s. Using %d I/O APICs\n", "Flat", nr_ioapics); +#endif } -static inline int multi_timer_check(int apic, int irq) +static inline int apicid_to_node(int logical_apicid) { return 0; } +#endif -static inline int apicid_to_node(int logical_apicid) +static inline unsigned long check_apicid_used(physid_mask_t bitmap, int apicid) +{ + return physid_isset(apicid, bitmap); +} + +static inline unsigned long check_apicid_present(int bit) +{ + return physid_isset(bit, phys_cpu_present_map); +} + +static inline physid_mask_t ioapic_phys_id_map(physid_mask_t phys_map) +{ + return phys_map; +} + +static inline int multi_timer_check(int apic, int irq) { return 0; } @@ -78,8 +109,8 @@ static inline int cpu_to_logical_apicid(int cpu) static inline int cpu_present_to_apicid(int mps_cpu) { - if (mps_cpu < get_physical_broadcast()) - return mps_cpu; + if (mps_cpu < NR_CPUS && cpu_present(mps_cpu)) + return (int)per_cpu(x86_bios_cpu_apicid, mps_cpu); else return BAD_APICID; } @@ -89,17 +120,6 @@ static inline physid_mask_t apicid_to_cpu_present(int phys_apicid) return physid_mask_of_physid(phys_apicid); } -static inline int mpc_apic_id(struct mpc_config_processor *m, - struct mpc_config_translation *translation_record) -{ - printk("Processor #%d %u:%u APIC version %d\n", - m->mpc_apicid, - (m->mpc_cpufeature & CPU_FAMILY_MASK) >> 8, - (m->mpc_cpufeature & CPU_MODEL_MASK) >> 4, - m->mpc_apicver); - return m->mpc_apicid; -} - static inline void setup_portio_remap(void) { } @@ -109,23 +129,9 @@ static inline int check_phys_apicid_present(int boot_cpu_physical_apicid) return physid_isset(boot_cpu_physical_apicid, phys_cpu_present_map); } -static inline int apic_id_registered(void) -{ - return physid_isset(GET_APIC_ID(apic_read(APIC_ID)), phys_cpu_present_map); -} - -static inline unsigned int cpu_mask_to_apicid(cpumask_t cpumask) -{ - return cpus_addr(cpumask)[0]; -} - static inline void enable_apic_mode(void) { } -static inline u32 phys_pkg_id(u32 cpuid_apic, int index_msb) -{ - return cpuid_apic >> index_msb; -} - +#endif /* CONFIG_X86_LOCAL_APIC */ #endif /* __ASM_MACH_APIC_H */ diff --git a/include/asm-x86/mach-default/mach_apicdef.h b/include/asm-x86/mach-default/mach_apicdef.h index ae984131909..e4b29ba37de 100644 --- a/include/asm-x86/mach-default/mach_apicdef.h +++ b/include/asm-x86/mach-default/mach_apicdef.h @@ -3,10 +3,14 @@ #include <asm/apic.h> +#ifdef CONFIG_X86_64 +#define APIC_ID_MASK (0xFFu<<24) +#define GET_APIC_ID(x) (((x)>>24)&0xFFu) +#define SET_APIC_ID(x) (((x)<<24)) +#else #define APIC_ID_MASK (0xF<<24) - static inline unsigned get_apic_id(unsigned long x) -{ +{ unsigned int ver = GET_APIC_VERSION(apic_read(APIC_LVR)); if (APIC_XAPIC(ver)) return (((x)>>24)&0xFF); @@ -15,5 +19,6 @@ static inline unsigned get_apic_id(unsigned long x) } #define GET_APIC_ID(x) get_apic_id(x) +#endif #endif diff --git a/include/asm-x86/mach-default/mach_ipi.h b/include/asm-x86/mach-default/mach_ipi.h index 0dba244c86d..be323364e68 100644 --- a/include/asm-x86/mach-default/mach_ipi.h +++ b/include/asm-x86/mach-default/mach_ipi.h @@ -9,10 +9,15 @@ void __send_IPI_shortcut(unsigned int shortcut, int vector); extern int no_broadcast; +#ifdef CONFIG_X86_64 +#include <asm/genapic.h> +#define send_IPI_mask (genapic->send_IPI_mask) +#else static inline void send_IPI_mask(cpumask_t mask, int vector) { send_IPI_mask_bitmask(mask, vector); } +#endif static inline void __local_send_IPI_allbutself(int vector) { @@ -33,6 +38,10 @@ static inline void __local_send_IPI_all(int vector) __send_IPI_shortcut(APIC_DEST_ALLINC, vector); } +#ifdef CONFIG_X86_64 +#define send_IPI_allbutself (genapic->send_IPI_allbutself) +#define send_IPI_all (genapic->send_IPI_all) +#else static inline void send_IPI_allbutself(int vector) { /* @@ -50,5 +59,6 @@ static inline void send_IPI_all(int vector) { __local_send_IPI_all(vector); } +#endif #endif /* __ASM_MACH_IPI_H */ diff --git a/include/asm-x86/mach-default/mach_mpparse.h b/include/asm-x86/mach-default/mach_mpparse.h index 1d383248258..d14108505bb 100644 --- a/include/asm-x86/mach-default/mach_mpparse.h +++ b/include/asm-x86/mach-default/mach_mpparse.h @@ -1,17 +1,6 @@ #ifndef __ASM_MACH_MPPARSE_H #define __ASM_MACH_MPPARSE_H -static inline void mpc_oem_bus_info(struct mpc_config_bus *m, char *name, - struct mpc_config_translation *translation) -{ -// Dprintk("Bus #%d is %s\n", m->mpc_busid, name); -} - -static inline void mpc_oem_pci_bus(struct mpc_config_bus *m, - struct mpc_config_translation *translation) -{ -} - static inline int mps_oem_check(struct mp_config_table *mpc, char *oem, char *productid) { diff --git a/include/asm-x86/mach-default/mach_reboot.h b/include/asm-x86/mach-default/mach_reboot.h deleted file mode 100644 index 6adee6a97de..00000000000 --- a/include/asm-x86/mach-default/mach_reboot.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * arch/i386/mach-generic/mach_reboot.h - * - * Machine specific reboot functions for generic. - * Split out from reboot.c by Osamu Tomita <tomita@cinet.co.jp> - */ -#ifndef _MACH_REBOOT_H -#define _MACH_REBOOT_H - -static inline void kb_wait(void) -{ - int i; - - for (i = 0; i < 0x10000; i++) - if ((inb_p(0x64) & 0x02) == 0) - break; -} - -static inline void mach_reboot(void) -{ - int i; - - /* old method, works on most machines */ - for (i = 0; i < 10; i++) { - kb_wait(); - udelay(50); - outb(0xfe, 0x64); /* pulse reset low */ - udelay(50); - } - - /* New method: sets the "System flag" which, when set, indicates - * successful completion of the keyboard controller self-test (Basic - * Assurance Test, BAT). This is needed for some machines with no - * keyboard plugged in. This read-modify-write sequence sets only the - * system flag - */ - for (i = 0; i < 10; i++) { - int cmd; - - outb(0x20, 0x64); /* read Controller Command Byte */ - udelay(50); - kb_wait(); - udelay(50); - cmd = inb(0x60); - udelay(50); - kb_wait(); - udelay(50); - outb(0x60, 0x64); /* write Controller Command Byte */ - udelay(50); - kb_wait(); - udelay(50); - outb(cmd | 0x14, 0x60); /* set "System flag" and "Keyboard Disabled" */ - udelay(50); - kb_wait(); - udelay(50); - outb(0xfe, 0x64); /* pulse reset low */ - udelay(50); - } -} - -#endif /* !_MACH_REBOOT_H */ diff --git a/include/asm-x86/mach-default/smpboot_hooks.h b/include/asm-x86/mach-default/smpboot_hooks.h index 7f45f631105..56d0e1fa025 100644 --- a/include/asm-x86/mach-default/smpboot_hooks.h +++ b/include/asm-x86/mach-default/smpboot_hooks.h @@ -33,7 +33,7 @@ static inline void smpboot_restore_warm_reset_vector(void) *((volatile long *) phys_to_virt(0x467)) = 0; } -static inline void smpboot_setup_io_apic(void) +static inline void __init smpboot_setup_io_apic(void) { /* * Here we can be sure that there is an IO-APIC in the system. Let's @@ -41,4 +41,11 @@ static inline void smpboot_setup_io_apic(void) */ if (!skip_ioapic_setup && nr_ioapics) setup_IO_APIC(); + else + nr_ioapics = 0; +} + +static inline void smpboot_clear_io_apic(void) +{ + nr_ioapics = 0; } diff --git a/include/asm-x86/mach-es7000/mach_apic.h b/include/asm-x86/mach-es7000/mach_apic.h index d23011fdf45..fbc8ad256f5 100644 --- a/include/asm-x86/mach-es7000/mach_apic.h +++ b/include/asm-x86/mach-es7000/mach_apic.h @@ -1,9 +1,7 @@ #ifndef __ASM_MACH_APIC_H #define __ASM_MACH_APIC_H -extern u8 bios_cpu_apicid[]; - -#define xapic_phys_to_log_apicid(cpu) (bios_cpu_apicid[cpu]) +#define xapic_phys_to_log_apicid(cpu) per_cpu(x86_bios_cpu_apicid, cpu) #define esr_disable (1) static inline int apic_id_registered(void) @@ -80,7 +78,7 @@ extern void enable_apic_mode(void); extern int apic_version [MAX_APICS]; static inline void setup_apic_routing(void) { - int apic = bios_cpu_apicid[smp_processor_id()]; + int apic = per_cpu(x86_bios_cpu_apicid, smp_processor_id()); printk("Enabling APIC mode: %s. Using %d I/O APICs, target cpus %lx\n", (apic_version[apic] == 0x14) ? "Physical Cluster" : "Logical Cluster", nr_ioapics, cpus_addr(TARGET_CPUS)[0]); @@ -102,7 +100,7 @@ static inline int cpu_present_to_apicid(int mps_cpu) if (!mps_cpu) return boot_cpu_physical_apicid; else if (mps_cpu < NR_CPUS) - return (int) bios_cpu_apicid[mps_cpu]; + return (int) per_cpu(x86_bios_cpu_apicid, mps_cpu); else return BAD_APICID; } @@ -129,16 +127,6 @@ static inline int cpu_to_logical_apicid(int cpu) #endif } -static inline int mpc_apic_id(struct mpc_config_processor *m, struct mpc_config_translation *unused) -{ - printk("Processor #%d %u:%u APIC version %d\n", - m->mpc_apicid, - (m->mpc_cpufeature & CPU_FAMILY_MASK) >> 8, - (m->mpc_cpufeature & CPU_MODEL_MASK) >> 4, - m->mpc_apicver); - return (m->mpc_apicid); -} - static inline physid_mask_t ioapic_phys_id_map(physid_mask_t phys_map) { /* For clustered we don't have a good way to do this yet - hack */ @@ -153,7 +141,7 @@ static inline void setup_portio_remap(void) extern unsigned int boot_cpu_physical_apicid; static inline int check_phys_apicid_present(int cpu_physical_apicid) { - boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID)); + boot_cpu_physical_apicid = GET_APIC_ID(read_apic_id()); return (1); } diff --git a/include/asm-x86/mach-es7000/mach_mpparse.h b/include/asm-x86/mach-es7000/mach_mpparse.h index 52ee75cd0fe..ef26d352362 100644 --- a/include/asm-x86/mach-es7000/mach_mpparse.h +++ b/include/asm-x86/mach-es7000/mach_mpparse.h @@ -3,17 +3,6 @@ #include <linux/acpi.h> -static inline void mpc_oem_bus_info(struct mpc_config_bus *m, char *name, - struct mpc_config_translation *translation) -{ - Dprintk("Bus #%d is %s\n", m->mpc_busid, name); -} - -static inline void mpc_oem_pci_bus(struct mpc_config_bus *m, - struct mpc_config_translation *translation) -{ -} - extern int parse_unisys_oem (char *oemptr); extern int find_unisys_acpi_oem_table(unsigned long *oem_addr); extern void setup_unisys(void); diff --git a/include/asm-x86/mach-generic/mach_apic.h b/include/asm-x86/mach-generic/mach_apic.h index a236e702152..6eff343e123 100644 --- a/include/asm-x86/mach-generic/mach_apic.h +++ b/include/asm-x86/mach-generic/mach_apic.h @@ -19,7 +19,6 @@ #define cpu_to_logical_apicid (genapic->cpu_to_logical_apicid) #define cpu_present_to_apicid (genapic->cpu_present_to_apicid) #define apicid_to_cpu_present (genapic->apicid_to_cpu_present) -#define mpc_apic_id (genapic->mpc_apic_id) #define setup_portio_remap (genapic->setup_portio_remap) #define check_apicid_present (genapic->check_apicid_present) #define check_phys_apicid_present (genapic->check_phys_apicid_present) diff --git a/include/asm-x86/mach-generic/mach_mpparse.h b/include/asm-x86/mach-generic/mach_mpparse.h index dbd9fce54f4..0d0b5ba2e9d 100644 --- a/include/asm-x86/mach-generic/mach_mpparse.h +++ b/include/asm-x86/mach-generic/mach_mpparse.h @@ -1,11 +1,6 @@ #ifndef _MACH_MPPARSE_H #define _MACH_MPPARSE_H 1 -#include <asm/genapic.h> - -#define mpc_oem_bus_info (genapic->mpc_oem_bus_info) -#define mpc_oem_pci_bus (genapic->mpc_oem_pci_bus) - int mps_oem_check(struct mp_config_table *mpc, char *oem, char *productid); int acpi_madt_oem_check(char *oem_id, char *oem_table_id); diff --git a/include/asm-x86/mach-numaq/mach_apic.h b/include/asm-x86/mach-numaq/mach_apic.h index 3b637fac890..75a56e5afbe 100644 --- a/include/asm-x86/mach-numaq/mach_apic.h +++ b/include/asm-x86/mach-numaq/mach_apic.h @@ -95,6 +95,16 @@ static inline physid_mask_t apicid_to_cpu_present(int logical_apicid) return physid_mask_of_physid(cpu + 4*node); } +struct mpc_config_translation { + unsigned char mpc_type; + unsigned char trans_len; + unsigned char trans_type; + unsigned char trans_quad; + unsigned char trans_global; + unsigned char trans_local; + unsigned short trans_reserved; +}; + static inline int mpc_apic_id(struct mpc_config_processor *m, struct mpc_config_translation *translation_record) { diff --git a/include/asm-x86/mach-numaq/mach_mpparse.h b/include/asm-x86/mach-numaq/mach_mpparse.h index 51bbac8fc0c..459b1240118 100644 --- a/include/asm-x86/mach-numaq/mach_mpparse.h +++ b/include/asm-x86/mach-numaq/mach_mpparse.h @@ -1,25 +1,10 @@ #ifndef __ASM_MACH_MPPARSE_H #define __ASM_MACH_MPPARSE_H -static inline void mpc_oem_bus_info(struct mpc_config_bus *m, char *name, - struct mpc_config_translation *translation) -{ - int quad = translation->trans_quad; - int local = translation->trans_local; - - mp_bus_id_to_node[m->mpc_busid] = quad; - mp_bus_id_to_local[m->mpc_busid] = local; - printk("Bus #%d is %s (node %d)\n", m->mpc_busid, name, quad); -} - -static inline void mpc_oem_pci_bus(struct mpc_config_bus *m, - struct mpc_config_translation *translation) -{ - int quad = translation->trans_quad; - int local = translation->trans_local; - - quad_local_to_mp_bus_id[quad][local] = m->mpc_busid; -} +extern void mpc_oem_bus_info(struct mpc_config_bus *m, char *name, + struct mpc_config_translation *translation); +extern void mpc_oem_pci_bus(struct mpc_config_bus *m, + struct mpc_config_translation *translation); /* Hook from generic ACPI tables.c */ static inline void acpi_madt_oem_check(char *oem_id, char *oem_table_id) diff --git a/include/asm-x86/mach-summit/mach_apic.h b/include/asm-x86/mach-summit/mach_apic.h index 062c97f6100..1f76c2e7023 100644 --- a/include/asm-x86/mach-summit/mach_apic.h +++ b/include/asm-x86/mach-summit/mach_apic.h @@ -40,7 +40,6 @@ static inline unsigned long check_apicid_present(int bit) #define apicid_cluster(apicid) ((apicid) & XAPIC_DEST_CLUSTER_MASK) -extern u8 bios_cpu_apicid[]; extern u8 cpu_2_logical_apicid[]; static inline void init_apic_ldr(void) @@ -110,7 +109,7 @@ static inline int cpu_to_logical_apicid(int cpu) static inline int cpu_present_to_apicid(int mps_cpu) { if (mps_cpu < NR_CPUS) - return (int)bios_cpu_apicid[mps_cpu]; + return (int)per_cpu(x86_bios_cpu_apicid, mps_cpu); else return BAD_APICID; } @@ -126,17 +125,6 @@ static inline physid_mask_t apicid_to_cpu_present(int apicid) return physid_mask_of_physid(0); } -static inline int mpc_apic_id(struct mpc_config_processor *m, - struct mpc_config_translation *translation_record) -{ - printk("Processor #%d %u:%u APIC version %d\n", - m->mpc_apicid, - (m->mpc_cpufeature & CPU_FAMILY_MASK) >> 8, - (m->mpc_cpufeature & CPU_MODEL_MASK) >> 4, - m->mpc_apicver); - return m->mpc_apicid; -} - static inline void setup_portio_remap(void) { } diff --git a/include/asm-x86/mach-summit/mach_mpparse.h b/include/asm-x86/mach-summit/mach_mpparse.h index c2520539d93..fdf59170133 100644 --- a/include/asm-x86/mach-summit/mach_mpparse.h +++ b/include/asm-x86/mach-summit/mach_mpparse.h @@ -12,17 +12,6 @@ extern void setup_summit(void); #define setup_summit() {} #endif -static inline void mpc_oem_bus_info(struct mpc_config_bus *m, char *name, - struct mpc_config_translation *translation) -{ - Dprintk("Bus #%d is %s\n", m->mpc_busid, name); -} - -static inline void mpc_oem_pci_bus(struct mpc_config_bus *m, - struct mpc_config_translation *translation) -{ -} - static inline int mps_oem_check(struct mp_config_table *mpc, char *oem, char *productid) { diff --git a/include/asm-x86/mach-visws/mach_apic.h b/include/asm-x86/mach-visws/mach_apic.h index efac6f0d139..a9ef33a8a99 100644 --- a/include/asm-x86/mach-visws/mach_apic.h +++ b/include/asm-x86/mach-visws/mach_apic.h @@ -23,7 +23,7 @@ static inline int apic_id_registered(void) { - return physid_isset(GET_APIC_ID(apic_read(APIC_ID)), phys_cpu_present_map); + return physid_isset(GET_APIC_ID(read_apic_id()), phys_cpu_present_map); } /* diff --git a/include/asm-x86/mach-visws/smpboot_hooks.h b/include/asm-x86/mach-visws/smpboot_hooks.h index d926471fa35..c9b83e395a2 100644 --- a/include/asm-x86/mach-visws/smpboot_hooks.h +++ b/include/asm-x86/mach-visws/smpboot_hooks.h @@ -22,3 +22,7 @@ static inline void smpboot_restore_warm_reset_vector(void) static inline void smpboot_setup_io_apic(void) { } + +static inline void smpboot_clear_io_apic(void) +{ +} diff --git a/include/asm-x86/mach_apic.h b/include/asm-x86/mach_apic.h deleted file mode 100644 index 7b7115a0c1c..00000000000 --- a/include/asm-x86/mach_apic.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef __ASM_MACH_APIC_H -#define __ASM_MACH_APIC_H - -/* - * Copyright 2004 James Cleverdon, IBM. - * Subject to the GNU Public License, v.2 - * - * Generic APIC sub-arch defines. - * - * Hacked for x86-64 by James Cleverdon from i386 architecture code by - * Martin Bligh, Andi Kleen, James Bottomley, John Stultz, and - * James Cleverdon. - */ - -#include <asm/genapic.h> - -#define INT_DELIVERY_MODE (genapic->int_delivery_mode) -#define INT_DEST_MODE (genapic->int_dest_mode) -#define TARGET_CPUS (genapic->target_cpus()) -#define vector_allocation_domain (genapic->vector_allocation_domain) -#define apic_id_registered (genapic->apic_id_registered) -#define init_apic_ldr (genapic->init_apic_ldr) -#define send_IPI_mask (genapic->send_IPI_mask) -#define send_IPI_allbutself (genapic->send_IPI_allbutself) -#define send_IPI_all (genapic->send_IPI_all) -#define cpu_mask_to_apicid (genapic->cpu_mask_to_apicid) -#define phys_pkg_id (genapic->phys_pkg_id) - -#endif /* __ASM_MACH_APIC_H */ diff --git a/include/asm-x86/mc146818rtc.h b/include/asm-x86/mc146818rtc.h index cdd9f965835..daf1ccde77a 100644 --- a/include/asm-x86/mc146818rtc.h +++ b/include/asm-x86/mc146818rtc.h @@ -42,7 +42,7 @@ extern volatile unsigned long cmos_lock; static inline void lock_cmos(unsigned char reg) { unsigned long new; - new = ((smp_processor_id()+1) << 8) | reg; + new = ((smp_processor_id() + 1) << 8) | reg; for (;;) { if (cmos_lock) { cpu_relax(); @@ -57,22 +57,26 @@ static inline void unlock_cmos(void) { cmos_lock = 0; } + static inline int do_i_have_lock_cmos(void) { - return (cmos_lock >> 8) == (smp_processor_id()+1); + return (cmos_lock >> 8) == (smp_processor_id() + 1); } + static inline unsigned char current_lock_cmos_reg(void) { return cmos_lock & 0xff; } -#define lock_cmos_prefix(reg) \ + +#define lock_cmos_prefix(reg) \ do { \ unsigned long cmos_flags; \ local_irq_save(cmos_flags); \ lock_cmos(reg) -#define lock_cmos_suffix(reg) \ - unlock_cmos(); \ - local_irq_restore(cmos_flags); \ + +#define lock_cmos_suffix(reg) \ + unlock_cmos(); \ + local_irq_restore(cmos_flags); \ } while (0) #else #define lock_cmos_prefix(reg) do {} while (0) diff --git a/include/asm-x86/mca_dma.h b/include/asm-x86/mca_dma.h index fbb1f3b7127..c3dca6edc6b 100644 --- a/include/asm-x86/mca_dma.h +++ b/include/asm-x86/mca_dma.h @@ -12,18 +12,18 @@ * count by 2 when using 16-bit dma; that is not handled by these functions. * * Ramen Noodles are yummy. - * - * 1998 Tymm Twillman <tymm@computer.org> + * + * 1998 Tymm Twillman <tymm@computer.org> */ /* - * Registers that are used by the DMA controller; FN is the function register + * Registers that are used by the DMA controller; FN is the function register * (tell the controller what to do) and EXE is the execution register (how * to do it) */ #define MCA_DMA_REG_FN 0x18 -#define MCA_DMA_REG_EXE 0x1A +#define MCA_DMA_REG_EXE 0x1A /* * Functions that the DMA controller can do @@ -43,9 +43,9 @@ /* * Modes (used by setting MCA_DMA_FN_MODE in the function register) - * + * * Note that the MODE_READ is read from memory (write to device), and - * MODE_WRITE is vice-versa. + * MODE_WRITE is vice-versa. */ #define MCA_DMA_MODE_XFER 0x04 /* read by default */ @@ -63,7 +63,7 @@ * IRQ context. */ -static __inline__ void mca_enable_dma(unsigned int dmanr) +static inline void mca_enable_dma(unsigned int dmanr) { outb(MCA_DMA_FN_RESET_MASK | dmanr, MCA_DMA_REG_FN); } @@ -76,7 +76,7 @@ static __inline__ void mca_enable_dma(unsigned int dmanr) * IRQ context. */ -static __inline__ void mca_disable_dma(unsigned int dmanr) +static inline void mca_disable_dma(unsigned int dmanr) { outb(MCA_DMA_FN_MASK | dmanr, MCA_DMA_REG_FN); } @@ -87,10 +87,10 @@ static __inline__ void mca_disable_dma(unsigned int dmanr) * @a: 24bit bus address * * Load the address register in the DMA controller. This has a 24bit - * limitation (16Mb). + * limitation (16Mb). */ -static __inline__ void mca_set_dma_addr(unsigned int dmanr, unsigned int a) +static inline void mca_set_dma_addr(unsigned int dmanr, unsigned int a) { outb(MCA_DMA_FN_SET_ADDR | dmanr, MCA_DMA_REG_FN); outb(a & 0xff, MCA_DMA_REG_EXE); @@ -106,14 +106,14 @@ static __inline__ void mca_set_dma_addr(unsigned int dmanr, unsigned int a) * limitation (16Mb). The return is a bus address. */ -static __inline__ unsigned int mca_get_dma_addr(unsigned int dmanr) +static inline unsigned int mca_get_dma_addr(unsigned int dmanr) { unsigned int addr; outb(MCA_DMA_FN_GET_ADDR | dmanr, MCA_DMA_REG_FN); addr = inb(MCA_DMA_REG_EXE); addr |= inb(MCA_DMA_REG_EXE) << 8; - addr |= inb(MCA_DMA_REG_EXE) << 16; + addr |= inb(MCA_DMA_REG_EXE) << 16; return addr; } @@ -127,7 +127,7 @@ static __inline__ unsigned int mca_get_dma_addr(unsigned int dmanr) * Setting a count of zero will not do what you expect. */ -static __inline__ void mca_set_dma_count(unsigned int dmanr, unsigned int count) +static inline void mca_set_dma_count(unsigned int dmanr, unsigned int count) { count--; /* transfers one more than count -- correct for this */ @@ -144,7 +144,7 @@ static __inline__ void mca_set_dma_count(unsigned int dmanr, unsigned int count) * on this DMA channel. */ -static __inline__ unsigned int mca_get_dma_residue(unsigned int dmanr) +static inline unsigned int mca_get_dma_residue(unsigned int dmanr) { unsigned short count; @@ -164,12 +164,12 @@ static __inline__ unsigned int mca_get_dma_residue(unsigned int dmanr) * with an I/O port target. */ -static __inline__ void mca_set_dma_io(unsigned int dmanr, unsigned int io_addr) +static inline void mca_set_dma_io(unsigned int dmanr, unsigned int io_addr) { /* * DMA from a port address -- set the io address */ - + outb(MCA_DMA_FN_SET_IO | dmanr, MCA_DMA_REG_FN); outb(io_addr & 0xff, MCA_DMA_REG_EXE); outb((io_addr >> 8) & 0xff, MCA_DMA_REG_EXE); @@ -192,7 +192,7 @@ static __inline__ void mca_set_dma_io(unsigned int dmanr, unsigned int io_addr) * %MCA_DMA_MODE_16 to do 16bit transfers. */ -static __inline__ void mca_set_dma_mode(unsigned int dmanr, unsigned int mode) +static inline void mca_set_dma_mode(unsigned int dmanr, unsigned int mode) { outb(MCA_DMA_FN_SET_MODE | dmanr, MCA_DMA_REG_FN); outb(mode, MCA_DMA_REG_EXE); diff --git a/include/asm-x86/mmu.h b/include/asm-x86/mmu.h index efa962c3889..00e88679e11 100644 --- a/include/asm-x86/mmu.h +++ b/include/asm-x86/mmu.h @@ -10,10 +10,10 @@ * * cpu_vm_mask is used to optimize ldt flushing. */ -typedef struct { +typedef struct { void *ldt; #ifdef CONFIG_X86_64 - rwlock_t ldtlock; + rwlock_t ldtlock; #endif int size; struct mutex lock; diff --git a/include/asm-x86/mmu_context_32.h b/include/asm-x86/mmu_context_32.h index 8198d1cca1f..9756ae0f1dd 100644 --- a/include/asm-x86/mmu_context_32.h +++ b/include/asm-x86/mmu_context_32.h @@ -62,7 +62,7 @@ static inline void switch_mm(struct mm_struct *prev, BUG_ON(per_cpu(cpu_tlbstate, cpu).active_mm != next); if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) { - /* We were in lazy tlb mode and leave_mm disabled + /* We were in lazy tlb mode and leave_mm disabled * tlb flush IPI delivery. We must reload %cr3. */ load_cr3(next->pgd); @@ -75,10 +75,10 @@ static inline void switch_mm(struct mm_struct *prev, #define deactivate_mm(tsk, mm) \ asm("movl %0,%%gs": :"r" (0)); -#define activate_mm(prev, next) \ - do { \ - paravirt_activate_mm(prev, next); \ - switch_mm((prev),(next),NULL); \ - } while(0); +#define activate_mm(prev, next) \ +do { \ + paravirt_activate_mm((prev), (next)); \ + switch_mm((prev), (next), NULL); \ +} while (0); #endif diff --git a/include/asm-x86/mmu_context_64.h b/include/asm-x86/mmu_context_64.h index ad6dc821ef9..ca44c71e7fb 100644 --- a/include/asm-x86/mmu_context_64.h +++ b/include/asm-x86/mmu_context_64.h @@ -20,12 +20,12 @@ void destroy_context(struct mm_struct *mm); static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) { #ifdef CONFIG_SMP - if (read_pda(mmu_state) == TLBSTATE_OK) + if (read_pda(mmu_state) == TLBSTATE_OK) write_pda(mmu_state, TLBSTATE_LAZY); #endif } -static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, +static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { unsigned cpu = smp_processor_id(); @@ -39,7 +39,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, cpu_set(cpu, next->cpu_vm_mask); load_cr3(next->pgd); - if (unlikely(next->context.ldt != prev->context.ldt)) + if (unlikely(next->context.ldt != prev->context.ldt)) load_LDT_nolock(&next->context); } #ifdef CONFIG_SMP @@ -48,7 +48,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, if (read_pda(active_mm) != next) BUG(); if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) { - /* We were in lazy tlb mode and leave_mm disabled + /* We were in lazy tlb mode and leave_mm disabled * tlb flush IPI delivery. We must reload CR3 * to make sure to use no freed page tables. */ @@ -59,13 +59,14 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, #endif } -#define deactivate_mm(tsk,mm) do { \ - load_gs_index(0); \ - asm volatile("movl %0,%%fs"::"r"(0)); \ -} while(0) +#define deactivate_mm(tsk, mm) \ +do { \ + load_gs_index(0); \ + asm volatile("movl %0,%%fs"::"r"(0)); \ +} while (0) -#define activate_mm(prev, next) \ - switch_mm((prev),(next),NULL) +#define activate_mm(prev, next) \ + switch_mm((prev), (next), NULL) #endif diff --git a/include/asm-x86/mmx.h b/include/asm-x86/mmx.h index 46b71da9986..940881218ff 100644 --- a/include/asm-x86/mmx.h +++ b/include/asm-x86/mmx.h @@ -6,7 +6,7 @@ */ #include <linux/types.h> - + extern void *_mmx_memcpy(void *to, const void *from, size_t size); extern void mmx_clear_page(void *page); extern void mmx_copy_page(void *to, void *from); diff --git a/include/asm-x86/mmzone_32.h b/include/asm-x86/mmzone_32.h index 274a59566c4..cb2cad0b65a 100644 --- a/include/asm-x86/mmzone_32.h +++ b/include/asm-x86/mmzone_32.h @@ -18,7 +18,7 @@ extern struct pglist_data *node_data[]; #include <asm/srat.h> #endif -extern int get_memcfg_numa_flat(void ); +extern int get_memcfg_numa_flat(void); /* * This allows any one NUMA architecture to be compiled * for, and still fall back to the flat function if it @@ -129,7 +129,7 @@ static inline int pfn_valid(int pfn) struct pglist_data __maybe_unused \ *__alloc_bootmem_node__pgdat = (pgdat); \ __alloc_bootmem_node(NODE_DATA(0), (x), PAGE_SIZE, \ - __pa(MAX_DMA_ADDRESS)) \ + __pa(MAX_DMA_ADDRESS)); \ }) #define alloc_bootmem_low_pages_node(pgdat, x) \ ({ \ diff --git a/include/asm-x86/mmzone_64.h b/include/asm-x86/mmzone_64.h index ebaf9663aa8..594bd0dc1d0 100644 --- a/include/asm-x86/mmzone_64.h +++ b/include/asm-x86/mmzone_64.h @@ -7,7 +7,7 @@ #ifdef CONFIG_NUMA -#define VIRTUAL_BUG_ON(x) +#define VIRTUAL_BUG_ON(x) #include <asm/smp.h> @@ -16,7 +16,7 @@ struct memnode { int shift; unsigned int mapsize; s16 *map; - s16 embedded_map[64-8]; + s16 embedded_map[64 - 8]; } ____cacheline_aligned; /* total size = 128 bytes */ extern struct memnode memnode; #define memnode_shift memnode.shift @@ -25,27 +25,27 @@ extern struct memnode memnode; extern struct pglist_data *node_data[]; -static inline __attribute__((pure)) int phys_to_nid(unsigned long addr) -{ - unsigned nid; +static inline __attribute__((pure)) int phys_to_nid(unsigned long addr) +{ + unsigned nid; VIRTUAL_BUG_ON(!memnodemap); VIRTUAL_BUG_ON((addr >> memnode_shift) >= memnodemapsize); - nid = memnodemap[addr >> memnode_shift]; - VIRTUAL_BUG_ON(nid >= MAX_NUMNODES || !node_data[nid]); - return nid; -} + nid = memnodemap[addr >> memnode_shift]; + VIRTUAL_BUG_ON(nid >= MAX_NUMNODES || !node_data[nid]); + return nid; +} #define NODE_DATA(nid) (node_data[nid]) #define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn) -#define node_end_pfn(nid) (NODE_DATA(nid)->node_start_pfn + \ +#define node_end_pfn(nid) (NODE_DATA(nid)->node_start_pfn + \ NODE_DATA(nid)->node_spanned_pages) extern int early_pfn_to_nid(unsigned long pfn); #ifdef CONFIG_NUMA_EMU -#define FAKE_NODE_MIN_SIZE (64*1024*1024) -#define FAKE_NODE_MIN_HASH_MASK (~(FAKE_NODE_MIN_SIZE - 1uL)) +#define FAKE_NODE_MIN_SIZE (64 * 1024 * 1024) +#define FAKE_NODE_MIN_HASH_MASK (~(FAKE_NODE_MIN_SIZE - 1UL)) #endif #endif diff --git a/include/asm-x86/mpspec.h b/include/asm-x86/mpspec.h index 781ad74ab9e..57a991b9c05 100644 --- a/include/asm-x86/mpspec.h +++ b/include/asm-x86/mpspec.h @@ -1,16 +1,13 @@ #ifndef _AM_X86_MPSPEC_H #define _AM_X86_MPSPEC_H +#include <linux/init.h> + #include <asm/mpspec_def.h> #ifdef CONFIG_X86_32 #include <mach_mpspec.h> -extern int mp_bus_id_to_type[MAX_MP_BUSSES]; -extern int mp_bus_id_to_node[MAX_MP_BUSSES]; -extern int mp_bus_id_to_local[MAX_MP_BUSSES]; -extern int quad_local_to_mp_bus_id[NR_CPUS/4][4]; - extern unsigned int def_to_bigsmp; extern int apic_version[MAX_APICS]; extern u8 apicid_2_node[]; @@ -24,27 +21,30 @@ extern int pic_mode; /* Each PCI slot may be a combo card with its own bus. 4 IRQ pins per slot. */ #define MAX_IRQ_SOURCES (MAX_MP_BUSSES * 4) -extern DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES); +extern void early_find_smp_config(void); +extern void early_get_smp_config(void); #endif +#if defined(CONFIG_MCA) || defined(CONFIG_EISA) +extern int mp_bus_id_to_type[MAX_MP_BUSSES]; +#endif + +extern DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES); + extern int mp_bus_id_to_pci_bus[MAX_MP_BUSSES]; extern unsigned int boot_cpu_physical_apicid; extern int smp_found_config; -extern int nr_ioapics; -extern int mp_irq_entries; -extern struct mpc_config_intsrc mp_irqs[MAX_IRQ_SOURCES]; extern int mpc_default_type; extern unsigned long mp_lapic_addr; extern void find_smp_config(void); extern void get_smp_config(void); +void __cpuinit generic_processor_info(int apicid, int version); #ifdef CONFIG_ACPI -extern void mp_register_lapic(u8 id, u8 enabled); -extern void mp_register_lapic_address(u64 address); -extern void mp_register_ioapic(u8 id, u32 address, u32 gsi_base); +extern void mp_register_ioapic(int id, u32 address, u32 gsi_base); extern void mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, u32 gsi); extern void mp_config_acpi_legacy_irqs(void); @@ -53,8 +53,7 @@ extern int mp_register_gsi(u32 gsi, int edge_level, int active_high_low); #define PHYSID_ARRAY_SIZE BITS_TO_LONGS(MAX_APICS) -struct physid_mask -{ +struct physid_mask { unsigned long mask[PHYSID_ARRAY_SIZE]; }; @@ -63,34 +62,34 @@ typedef struct physid_mask physid_mask_t; #define physid_set(physid, map) set_bit(physid, (map).mask) #define physid_clear(physid, map) clear_bit(physid, (map).mask) #define physid_isset(physid, map) test_bit(physid, (map).mask) -#define physid_test_and_set(physid, map) \ +#define physid_test_and_set(physid, map) \ test_and_set_bit(physid, (map).mask) -#define physids_and(dst, src1, src2) \ +#define physids_and(dst, src1, src2) \ bitmap_and((dst).mask, (src1).mask, (src2).mask, MAX_APICS) -#define physids_or(dst, src1, src2) \ +#define physids_or(dst, src1, src2) \ bitmap_or((dst).mask, (src1).mask, (src2).mask, MAX_APICS) -#define physids_clear(map) \ +#define physids_clear(map) \ bitmap_zero((map).mask, MAX_APICS) -#define physids_complement(dst, src) \ +#define physids_complement(dst, src) \ bitmap_complement((dst).mask, (src).mask, MAX_APICS) -#define physids_empty(map) \ +#define physids_empty(map) \ bitmap_empty((map).mask, MAX_APICS) -#define physids_equal(map1, map2) \ +#define physids_equal(map1, map2) \ bitmap_equal((map1).mask, (map2).mask, MAX_APICS) -#define physids_weight(map) \ +#define physids_weight(map) \ bitmap_weight((map).mask, MAX_APICS) -#define physids_shift_right(d, s, n) \ +#define physids_shift_right(d, s, n) \ bitmap_shift_right((d).mask, (s).mask, n, MAX_APICS) -#define physids_shift_left(d, s, n) \ +#define physids_shift_left(d, s, n) \ bitmap_shift_left((d).mask, (s).mask, n, MAX_APICS) #define physids_coerce(map) ((map).mask[0]) diff --git a/include/asm-x86/mpspec_def.h b/include/asm-x86/mpspec_def.h index 3504617fe64..dc6ef85e362 100644 --- a/include/asm-x86/mpspec_def.h +++ b/include/asm-x86/mpspec_def.h @@ -11,7 +11,7 @@ * information is. */ -#define SMP_MAGIC_IDENT (('_'<<24)|('P'<<16)|('M'<<8)|'_') +#define SMP_MAGIC_IDENT (('_'<<24) | ('P'<<16) | ('M'<<8) | '_') #ifdef CONFIG_X86_32 # define MAX_MPC_ENTRY 1024 @@ -23,8 +23,7 @@ # define MAX_APICS 255 #endif -struct intel_mp_floating -{ +struct intel_mp_floating { char mpf_signature[4]; /* "_MP_" */ unsigned int mpf_physptr; /* Configuration table address */ unsigned char mpf_length; /* Our length (paragraphs) */ @@ -39,14 +38,13 @@ struct intel_mp_floating #define MPC_SIGNATURE "PCMP" -struct mp_config_table -{ +struct mp_config_table { char mpc_signature[4]; unsigned short mpc_length; /* Size of table */ - char mpc_spec; /* 0x01 */ - char mpc_checksum; - char mpc_oem[8]; - char mpc_productid[12]; + char mpc_spec; /* 0x01 */ + char mpc_checksum; + char mpc_oem[8]; + char mpc_productid[12]; unsigned int mpc_oemptr; /* 0 if not present */ unsigned short mpc_oemsize; /* 0 if not present */ unsigned short mpc_oemcount; @@ -71,8 +69,7 @@ struct mp_config_table #define CPU_MODEL_MASK 0x00F0 #define CPU_FAMILY_MASK 0x0F00 -struct mpc_config_processor -{ +struct mpc_config_processor { unsigned char mpc_type; unsigned char mpc_apicid; /* Local APIC number */ unsigned char mpc_apicver; /* Its versions */ @@ -82,8 +79,7 @@ struct mpc_config_processor unsigned int mpc_reserved[2]; }; -struct mpc_config_bus -{ +struct mpc_config_bus { unsigned char mpc_type; unsigned char mpc_busid; unsigned char mpc_bustype[6]; @@ -111,8 +107,7 @@ struct mpc_config_bus #define MPC_APIC_USABLE 0x01 -struct mpc_config_ioapic -{ +struct mpc_config_ioapic { unsigned char mpc_type; unsigned char mpc_apicid; unsigned char mpc_apicver; @@ -120,8 +115,7 @@ struct mpc_config_ioapic unsigned int mpc_apicaddr; }; -struct mpc_config_intsrc -{ +struct mpc_config_intsrc { unsigned char mpc_type; unsigned char mpc_irqtype; unsigned short mpc_irqflag; @@ -144,8 +138,7 @@ enum mp_irq_source_types { #define MP_APIC_ALL 0xFF -struct mpc_config_lintsrc -{ +struct mpc_config_lintsrc { unsigned char mpc_type; unsigned char mpc_irqtype; unsigned short mpc_irqflag; @@ -157,8 +150,7 @@ struct mpc_config_lintsrc #define MPC_OEM_SIGNATURE "_OEM" -struct mp_config_oemtable -{ +struct mp_config_oemtable { char oem_signature[4]; unsigned short oem_length; /* Size of table */ char oem_rev; /* 0x01 */ @@ -166,17 +158,6 @@ struct mp_config_oemtable char mpc_oem[8]; }; -struct mpc_config_translation -{ - unsigned char mpc_type; - unsigned char trans_len; - unsigned char trans_type; - unsigned char trans_quad; - unsigned char trans_global; - unsigned char trans_local; - unsigned short trans_reserved; -}; - /* * Default configurations * @@ -196,4 +177,3 @@ enum mp_bustype { MP_BUS_MCA, }; #endif - diff --git a/include/asm-x86/msidef.h b/include/asm-x86/msidef.h index 5b8acddb70f..296f29ce426 100644 --- a/include/asm-x86/msidef.h +++ b/include/asm-x86/msidef.h @@ -11,7 +11,8 @@ #define MSI_DATA_VECTOR_SHIFT 0 #define MSI_DATA_VECTOR_MASK 0x000000ff -#define MSI_DATA_VECTOR(v) (((v) << MSI_DATA_VECTOR_SHIFT) & MSI_DATA_VECTOR_MASK) +#define MSI_DATA_VECTOR(v) (((v) << MSI_DATA_VECTOR_SHIFT) & \ + MSI_DATA_VECTOR_MASK) #define MSI_DATA_DELIVERY_MODE_SHIFT 8 #define MSI_DATA_DELIVERY_FIXED (0 << MSI_DATA_DELIVERY_MODE_SHIFT) @@ -37,11 +38,14 @@ #define MSI_ADDR_DEST_MODE_LOGICAL (1 << MSI_ADDR_DEST_MODE_SHIFT) #define MSI_ADDR_REDIRECTION_SHIFT 3 -#define MSI_ADDR_REDIRECTION_CPU (0 << MSI_ADDR_REDIRECTION_SHIFT) /* dedicated cpu */ -#define MSI_ADDR_REDIRECTION_LOWPRI (1 << MSI_ADDR_REDIRECTION_SHIFT) /* lowest priority */ +#define MSI_ADDR_REDIRECTION_CPU (0 << MSI_ADDR_REDIRECTION_SHIFT) + /* dedicated cpu */ +#define MSI_ADDR_REDIRECTION_LOWPRI (1 << MSI_ADDR_REDIRECTION_SHIFT) + /* lowest priority */ #define MSI_ADDR_DEST_ID_SHIFT 12 #define MSI_ADDR_DEST_ID_MASK 0x00ffff0 -#define MSI_ADDR_DEST_ID(dest) (((dest) << MSI_ADDR_DEST_ID_SHIFT) & MSI_ADDR_DEST_ID_MASK) +#define MSI_ADDR_DEST_ID(dest) (((dest) << MSI_ADDR_DEST_ID_SHIFT) & \ + MSI_ADDR_DEST_ID_MASK) #endif /* ASM_MSIDEF_H */ diff --git a/include/asm-x86/msr-index.h b/include/asm-x86/msr-index.h index fae118a2527..09413ad39d3 100644 --- a/include/asm-x86/msr-index.h +++ b/include/asm-x86/msr-index.h @@ -57,6 +57,8 @@ #define MSR_MTRRfix4K_F8000 0x0000026f #define MSR_MTRRdefType 0x000002ff +#define MSR_IA32_CR_PAT 0x00000277 + #define MSR_IA32_DEBUGCTLMSR 0x000001d9 #define MSR_IA32_LASTBRANCHFROMIP 0x000001db #define MSR_IA32_LASTBRANCHTOIP 0x000001dc @@ -83,6 +85,7 @@ /* AMD64 MSRs. Not complete. See the architecture manual for a more complete list. */ +#define MSR_AMD64_NB_CFG 0xc001001f #define MSR_AMD64_IBSFETCHCTL 0xc0011030 #define MSR_AMD64_IBSFETCHLINAD 0xc0011031 #define MSR_AMD64_IBSFETCHPHYSAD 0xc0011032 @@ -109,6 +112,7 @@ #define MSR_K8_SYSCFG 0xc0010010 #define MSR_K8_HWCR 0xc0010015 #define MSR_K8_ENABLE_C1E 0xc0010055 +#define MSR_K8_TSEG_ADDR 0xc0010112 #define K8_MTRRFIXRANGE_DRAM_ENABLE 0x00040000 /* MtrrFixDramEn bit */ #define K8_MTRRFIXRANGE_DRAM_MODIFY 0x00080000 /* MtrrFixDramModEn bit */ #define K8_MTRR_RDMEM_WRMEM_MASK 0x18181818 /* Mask: RdMem|WrMem */ diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index 3ca29ebebbb..3707650a169 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -16,8 +16,8 @@ static inline unsigned long long native_read_tscp(unsigned int *aux) { unsigned long low, high; - asm volatile (".byte 0x0f,0x01,0xf9" - : "=a" (low), "=d" (high), "=c" (*aux)); + asm volatile(".byte 0x0f,0x01,0xf9" + : "=a" (low), "=d" (high), "=c" (*aux)); return low | ((u64)high >> 32); } @@ -29,7 +29,7 @@ static inline unsigned long long native_read_tscp(unsigned int *aux) */ #ifdef CONFIG_X86_64 #define DECLARE_ARGS(val, low, high) unsigned low, high -#define EAX_EDX_VAL(val, low, high) (low | ((u64)(high) << 32)) +#define EAX_EDX_VAL(val, low, high) ((low) | ((u64)(high) << 32)) #define EAX_EDX_ARGS(val, low, high) "a" (low), "d" (high) #define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) #else @@ -57,7 +57,7 @@ static inline unsigned long long native_read_msr_safe(unsigned int msr, ".section .fixup,\"ax\"\n\t" "3: mov %3,%0 ; jmp 1b\n\t" ".previous\n\t" - _ASM_EXTABLE(2b,3b) + _ASM_EXTABLE(2b, 3b) : "=r" (*err), EAX_EDX_RET(val, low, high) : "c" (msr), "i" (-EFAULT)); return EAX_EDX_VAL(val, low, high); @@ -78,10 +78,10 @@ static inline int native_write_msr_safe(unsigned int msr, ".section .fixup,\"ax\"\n\t" "3: mov %4,%0 ; jmp 1b\n\t" ".previous\n\t" - _ASM_EXTABLE(2b,3b) + _ASM_EXTABLE(2b, 3b) : "=a" (err) : "c" (msr), "0" (low), "d" (high), - "i" (-EFAULT)); + "i" (-EFAULT)); return err; } @@ -116,23 +116,23 @@ static inline unsigned long long native_read_pmc(int counter) * pointer indirection), this allows gcc to optimize better */ -#define rdmsr(msr,val1,val2) \ - do { \ - u64 __val = native_read_msr(msr); \ - (val1) = (u32)__val; \ - (val2) = (u32)(__val >> 32); \ - } while(0) +#define rdmsr(msr, val1, val2) \ +do { \ + u64 __val = native_read_msr((msr)); \ + (val1) = (u32)__val; \ + (val2) = (u32)(__val >> 32); \ +} while (0) static inline void wrmsr(unsigned msr, unsigned low, unsigned high) { native_write_msr(msr, low, high); } -#define rdmsrl(msr,val) \ - ((val) = native_read_msr(msr)) +#define rdmsrl(msr, val) \ + ((val) = native_read_msr((msr))) #define wrmsrl(msr, val) \ - native_write_msr(msr, (u32)((u64)(val)), (u32)((u64)(val) >> 32)) + native_write_msr((msr), (u32)((u64)(val)), (u32)((u64)(val) >> 32)) /* wrmsr with exception handling */ static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high) @@ -141,14 +141,22 @@ static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high) } /* rdmsr with exception handling */ -#define rdmsr_safe(msr,p1,p2) \ - ({ \ - int __err; \ - u64 __val = native_read_msr_safe(msr, &__err); \ - (*p1) = (u32)__val; \ - (*p2) = (u32)(__val >> 32); \ - __err; \ - }) +#define rdmsr_safe(msr, p1, p2) \ +({ \ + int __err; \ + u64 __val = native_read_msr_safe((msr), &__err); \ + (*p1) = (u32)__val; \ + (*p2) = (u32)(__val >> 32); \ + __err; \ +}) + +static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) +{ + int err; + + *p = native_read_msr_safe(msr, &err); + return err; +} #define rdtscl(low) \ ((low) = (u32)native_read_tsc()) @@ -156,35 +164,37 @@ static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high) #define rdtscll(val) \ ((val) = native_read_tsc()) -#define rdpmc(counter,low,high) \ - do { \ - u64 _l = native_read_pmc(counter); \ - (low) = (u32)_l; \ - (high) = (u32)(_l >> 32); \ - } while(0) +#define rdpmc(counter, low, high) \ +do { \ + u64 _l = native_read_pmc((counter)); \ + (low) = (u32)_l; \ + (high) = (u32)(_l >> 32); \ +} while (0) -#define rdtscp(low, high, aux) \ - do { \ - unsigned long long _val = native_read_tscp(&(aux)); \ - (low) = (u32)_val; \ - (high) = (u32)(_val >> 32); \ - } while (0) +#define rdtscp(low, high, aux) \ +do { \ + unsigned long long _val = native_read_tscp(&(aux)); \ + (low) = (u32)_val; \ + (high) = (u32)(_val >> 32); \ +} while (0) #define rdtscpll(val, aux) (val) = native_read_tscp(&(aux)) #endif /* !CONFIG_PARAVIRT */ -#define checking_wrmsrl(msr,val) wrmsr_safe(msr,(u32)(val),(u32)((val)>>32)) +#define checking_wrmsrl(msr, val) wrmsr_safe((msr), (u32)(val), \ + (u32)((val) >> 32)) -#define write_tsc(val1,val2) wrmsr(0x10, val1, val2) +#define write_tsc(val1, val2) wrmsr(0x10, (val1), (val2)) -#define write_rdtscp_aux(val) wrmsr(0xc0000103, val, 0) +#define write_rdtscp_aux(val) wrmsr(0xc0000103, (val), 0) #ifdef CONFIG_SMP void rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); void wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); + int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); #else /* CONFIG_SMP */ static inline void rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) @@ -195,7 +205,8 @@ static inline void wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) { wrmsr(msr_no, l, h); } -static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) +static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, + u32 *l, u32 *h) { return rdmsr_safe(msr_no, l, h); } diff --git a/include/asm-x86/mtrr.h b/include/asm-x86/mtrr.h index 319d065800b..a69a01a5172 100644 --- a/include/asm-x86/mtrr.h +++ b/include/asm-x86/mtrr.h @@ -28,8 +28,7 @@ #define MTRR_IOCTL_BASE 'M' -struct mtrr_sentry -{ +struct mtrr_sentry { unsigned long base; /* Base address */ unsigned int size; /* Size of region */ unsigned int type; /* Type of region */ @@ -41,8 +40,7 @@ struct mtrr_sentry will break. */ #ifdef __i386__ -struct mtrr_gentry -{ +struct mtrr_gentry { unsigned int regnum; /* Register number */ unsigned long base; /* Base address */ unsigned int size; /* Size of region */ @@ -51,8 +49,7 @@ struct mtrr_gentry #else /* __i386__ */ -struct mtrr_gentry -{ +struct mtrr_gentry { unsigned long base; /* Base address */ unsigned int size; /* Size of region */ unsigned int regnum; /* Register number */ @@ -86,38 +83,45 @@ struct mtrr_gentry /* The following functions are for use by other drivers */ # ifdef CONFIG_MTRR +extern u8 mtrr_type_lookup(u64 addr, u64 end); extern void mtrr_save_fixed_ranges(void *); extern void mtrr_save_state(void); -extern int mtrr_add (unsigned long base, unsigned long size, - unsigned int type, bool increment); -extern int mtrr_add_page (unsigned long base, unsigned long size, - unsigned int type, bool increment); -extern int mtrr_del (int reg, unsigned long base, unsigned long size); -extern int mtrr_del_page (int reg, unsigned long base, unsigned long size); +extern int mtrr_add(unsigned long base, unsigned long size, + unsigned int type, bool increment); +extern int mtrr_add_page(unsigned long base, unsigned long size, + unsigned int type, bool increment); +extern int mtrr_del(int reg, unsigned long base, unsigned long size); +extern int mtrr_del_page(int reg, unsigned long base, unsigned long size); extern void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi); extern void mtrr_ap_init(void); extern void mtrr_bp_init(void); extern int mtrr_trim_uncached_memory(unsigned long end_pfn); +extern int amd_special_default_mtrr(void); # else +static inline u8 mtrr_type_lookup(u64 addr, u64 end) +{ + /* + * Return no-MTRRs: + */ + return 0xff; +} #define mtrr_save_fixed_ranges(arg) do {} while (0) #define mtrr_save_state() do {} while (0) -static __inline__ int mtrr_add (unsigned long base, unsigned long size, - unsigned int type, bool increment) +static inline int mtrr_add(unsigned long base, unsigned long size, + unsigned int type, bool increment) { return -ENODEV; } -static __inline__ int mtrr_add_page (unsigned long base, unsigned long size, +static inline int mtrr_add_page(unsigned long base, unsigned long size, unsigned int type, bool increment) { return -ENODEV; } -static __inline__ int mtrr_del (int reg, unsigned long base, - unsigned long size) +static inline int mtrr_del(int reg, unsigned long base, unsigned long size) { return -ENODEV; } -static __inline__ int mtrr_del_page (int reg, unsigned long base, - unsigned long size) +static inline int mtrr_del_page(int reg, unsigned long base, unsigned long size) { return -ENODEV; } @@ -125,7 +129,9 @@ static inline int mtrr_trim_uncached_memory(unsigned long end_pfn) { return 0; } -static __inline__ void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi) {;} +static inline void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi) +{ +} #define mtrr_ap_init() do {} while (0) #define mtrr_bp_init() do {} while (0) @@ -134,15 +140,13 @@ static __inline__ void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi) {;} #ifdef CONFIG_COMPAT #include <linux/compat.h> -struct mtrr_sentry32 -{ +struct mtrr_sentry32 { compat_ulong_t base; /* Base address */ compat_uint_t size; /* Size of region */ compat_uint_t type; /* Type of region */ }; -struct mtrr_gentry32 -{ +struct mtrr_gentry32 { compat_ulong_t regnum; /* Register number */ compat_uint_t base; /* Base address */ compat_uint_t size; /* Size of region */ @@ -151,16 +155,17 @@ struct mtrr_gentry32 #define MTRR_IOCTL_BASE 'M' -#define MTRRIOC32_ADD_ENTRY _IOW(MTRR_IOCTL_BASE, 0, struct mtrr_sentry32) -#define MTRRIOC32_SET_ENTRY _IOW(MTRR_IOCTL_BASE, 1, struct mtrr_sentry32) -#define MTRRIOC32_DEL_ENTRY _IOW(MTRR_IOCTL_BASE, 2, struct mtrr_sentry32) -#define MTRRIOC32_GET_ENTRY _IOWR(MTRR_IOCTL_BASE, 3, struct mtrr_gentry32) -#define MTRRIOC32_KILL_ENTRY _IOW(MTRR_IOCTL_BASE, 4, struct mtrr_sentry32) -#define MTRRIOC32_ADD_PAGE_ENTRY _IOW(MTRR_IOCTL_BASE, 5, struct mtrr_sentry32) -#define MTRRIOC32_SET_PAGE_ENTRY _IOW(MTRR_IOCTL_BASE, 6, struct mtrr_sentry32) -#define MTRRIOC32_DEL_PAGE_ENTRY _IOW(MTRR_IOCTL_BASE, 7, struct mtrr_sentry32) -#define MTRRIOC32_GET_PAGE_ENTRY _IOWR(MTRR_IOCTL_BASE, 8, struct mtrr_gentry32) -#define MTRRIOC32_KILL_PAGE_ENTRY _IOW(MTRR_IOCTL_BASE, 9, struct mtrr_sentry32) +#define MTRRIOC32_ADD_ENTRY _IOW(MTRR_IOCTL_BASE, 0, struct mtrr_sentry32) +#define MTRRIOC32_SET_ENTRY _IOW(MTRR_IOCTL_BASE, 1, struct mtrr_sentry32) +#define MTRRIOC32_DEL_ENTRY _IOW(MTRR_IOCTL_BASE, 2, struct mtrr_sentry32) +#define MTRRIOC32_GET_ENTRY _IOWR(MTRR_IOCTL_BASE, 3, struct mtrr_gentry32) +#define MTRRIOC32_KILL_ENTRY _IOW(MTRR_IOCTL_BASE, 4, struct mtrr_sentry32) +#define MTRRIOC32_ADD_PAGE_ENTRY _IOW(MTRR_IOCTL_BASE, 5, struct mtrr_sentry32) +#define MTRRIOC32_SET_PAGE_ENTRY _IOW(MTRR_IOCTL_BASE, 6, struct mtrr_sentry32) +#define MTRRIOC32_DEL_PAGE_ENTRY _IOW(MTRR_IOCTL_BASE, 7, struct mtrr_sentry32) +#define MTRRIOC32_GET_PAGE_ENTRY _IOWR(MTRR_IOCTL_BASE, 8, struct mtrr_gentry32) +#define MTRRIOC32_KILL_PAGE_ENTRY \ + _IOW(MTRR_IOCTL_BASE, 9, struct mtrr_sentry32) #endif /* CONFIG_COMPAT */ #endif /* __KERNEL__ */ diff --git a/include/asm-x86/mutex_32.h b/include/asm-x86/mutex_32.h index bbeefb96ddf..73e928ef5f0 100644 --- a/include/asm-x86/mutex_32.h +++ b/include/asm-x86/mutex_32.h @@ -9,7 +9,7 @@ #ifndef _ASM_MUTEX_H #define _ASM_MUTEX_H -#include "asm/alternative.h" +#include <asm/alternative.h> /** * __mutex_fastpath_lock - try to take the lock by moving the count @@ -21,22 +21,20 @@ * wasn't 1 originally. This function MUST leave the value lower than 1 * even when the "1" assertion wasn't true. */ -#define __mutex_fastpath_lock(count, fail_fn) \ -do { \ - unsigned int dummy; \ - \ - typecheck(atomic_t *, count); \ +#define __mutex_fastpath_lock(count, fail_fn) \ +do { \ + unsigned int dummy; \ + \ + typecheck(atomic_t *, count); \ typecheck_fn(void (*)(atomic_t *), fail_fn); \ - \ - __asm__ __volatile__( \ - LOCK_PREFIX " decl (%%eax) \n" \ - " jns 1f \n" \ - " call "#fail_fn" \n" \ - "1: \n" \ - \ - :"=a" (dummy) \ - : "a" (count) \ - : "memory", "ecx", "edx"); \ + \ + asm volatile(LOCK_PREFIX " decl (%%eax)\n" \ + " jns 1f \n" \ + " call " #fail_fn "\n" \ + "1:\n" \ + : "=a" (dummy) \ + : "a" (count) \ + : "memory", "ecx", "edx"); \ } while (0) @@ -50,8 +48,8 @@ do { \ * wasn't 1 originally. This function returns 0 if the fastpath succeeds, * or anything the slow path function returns */ -static inline int -__mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *)) +static inline int __mutex_fastpath_lock_retval(atomic_t *count, + int (*fail_fn)(atomic_t *)) { if (unlikely(atomic_dec_return(count) < 0)) return fail_fn(count); @@ -72,22 +70,20 @@ __mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *)) * __mutex_slowpath_needs_to_unlock() macro needs to return 1, it needs * to return 0 otherwise. */ -#define __mutex_fastpath_unlock(count, fail_fn) \ -do { \ - unsigned int dummy; \ - \ - typecheck(atomic_t *, count); \ +#define __mutex_fastpath_unlock(count, fail_fn) \ +do { \ + unsigned int dummy; \ + \ + typecheck(atomic_t *, count); \ typecheck_fn(void (*)(atomic_t *), fail_fn); \ - \ - __asm__ __volatile__( \ - LOCK_PREFIX " incl (%%eax) \n" \ - " jg 1f \n" \ - " call "#fail_fn" \n" \ - "1: \n" \ - \ - :"=a" (dummy) \ - : "a" (count) \ - : "memory", "ecx", "edx"); \ + \ + asm volatile(LOCK_PREFIX " incl (%%eax)\n" \ + " jg 1f\n" \ + " call " #fail_fn "\n" \ + "1:\n" \ + : "=a" (dummy) \ + : "a" (count) \ + : "memory", "ecx", "edx"); \ } while (0) #define __mutex_slowpath_needs_to_unlock() 1 @@ -104,8 +100,8 @@ do { \ * Additionally, if the value was < 0 originally, this function must not leave * it to 0 on failure. */ -static inline int -__mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) +static inline int __mutex_fastpath_trylock(atomic_t *count, + int (*fail_fn)(atomic_t *)) { /* * We have two variants here. The cmpxchg based one is the best one diff --git a/include/asm-x86/mutex_64.h b/include/asm-x86/mutex_64.h index 6c2949a3c67..f3fae9becb3 100644 --- a/include/asm-x86/mutex_64.h +++ b/include/asm-x86/mutex_64.h @@ -16,23 +16,21 @@ * * Atomically decrements @v and calls <fail_fn> if the result is negative. */ -#define __mutex_fastpath_lock(v, fail_fn) \ -do { \ - unsigned long dummy; \ - \ - typecheck(atomic_t *, v); \ - typecheck_fn(void (*)(atomic_t *), fail_fn); \ - \ - __asm__ __volatile__( \ - LOCK_PREFIX " decl (%%rdi) \n" \ - " jns 1f \n" \ - " call "#fail_fn" \n" \ - "1:" \ - \ - :"=D" (dummy) \ - : "D" (v) \ - : "rax", "rsi", "rdx", "rcx", \ - "r8", "r9", "r10", "r11", "memory"); \ +#define __mutex_fastpath_lock(v, fail_fn) \ +do { \ + unsigned long dummy; \ + \ + typecheck(atomic_t *, v); \ + typecheck_fn(void (*)(atomic_t *), fail_fn); \ + \ + asm volatile(LOCK_PREFIX " decl (%%rdi)\n" \ + " jns 1f \n" \ + " call " #fail_fn "\n" \ + "1:" \ + : "=D" (dummy) \ + : "D" (v) \ + : "rax", "rsi", "rdx", "rcx", \ + "r8", "r9", "r10", "r11", "memory"); \ } while (0) /** @@ -45,9 +43,8 @@ do { \ * it wasn't 1 originally. This function returns 0 if the fastpath succeeds, * or anything the slow path function returns */ -static inline int -__mutex_fastpath_lock_retval(atomic_t *count, - int (*fail_fn)(atomic_t *)) +static inline int __mutex_fastpath_lock_retval(atomic_t *count, + int (*fail_fn)(atomic_t *)) { if (unlikely(atomic_dec_return(count) < 0)) return fail_fn(count); @@ -62,23 +59,21 @@ __mutex_fastpath_lock_retval(atomic_t *count, * * Atomically increments @v and calls <fail_fn> if the result is nonpositive. */ -#define __mutex_fastpath_unlock(v, fail_fn) \ -do { \ - unsigned long dummy; \ - \ - typecheck(atomic_t *, v); \ - typecheck_fn(void (*)(atomic_t *), fail_fn); \ - \ - __asm__ __volatile__( \ - LOCK_PREFIX " incl (%%rdi) \n" \ - " jg 1f \n" \ - " call "#fail_fn" \n" \ - "1: " \ - \ - :"=D" (dummy) \ - : "D" (v) \ - : "rax", "rsi", "rdx", "rcx", \ - "r8", "r9", "r10", "r11", "memory"); \ +#define __mutex_fastpath_unlock(v, fail_fn) \ +do { \ + unsigned long dummy; \ + \ + typecheck(atomic_t *, v); \ + typecheck_fn(void (*)(atomic_t *), fail_fn); \ + \ + asm volatile(LOCK_PREFIX " incl (%%rdi)\n" \ + " jg 1f\n" \ + " call " #fail_fn "\n" \ + "1:" \ + : "=D" (dummy) \ + : "D" (v) \ + : "rax", "rsi", "rdx", "rcx", \ + "r8", "r9", "r10", "r11", "memory"); \ } while (0) #define __mutex_slowpath_needs_to_unlock() 1 @@ -93,8 +88,8 @@ do { \ * if it wasn't 1 originally. [the fallback function is never used on * x86_64, because all x86_64 CPUs have a CMPXCHG instruction.] */ -static inline int -__mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) +static inline int __mutex_fastpath_trylock(atomic_t *count, + int (*fail_fn)(atomic_t *)) { if (likely(atomic_cmpxchg(count, 1, 0) == 1)) return 1; diff --git a/include/asm-x86/nmi.h b/include/asm-x86/nmi.h index 53ccac14cea..1e363021e72 100644 --- a/include/asm-x86/nmi.h +++ b/include/asm-x86/nmi.h @@ -1,5 +1,93 @@ -#ifdef CONFIG_X86_32 -# include "nmi_32.h" +#ifndef _ASM_X86_NMI_H_ +#define _ASM_X86_NMI_H_ + +#include <linux/pm.h> +#include <asm/irq.h> +#include <asm/io.h> + +#ifdef ARCH_HAS_NMI_WATCHDOG + +/** + * do_nmi_callback + * + * Check to see if a callback exists and execute it. Return 1 + * if the handler exists and was handled successfully. + */ +int do_nmi_callback(struct pt_regs *regs, int cpu); + +#ifdef CONFIG_PM + +/** Replace the PM callback routine for NMI. */ +struct pm_dev *set_nmi_pm_callback(pm_callback callback); + +/** Unset the PM callback routine back to the default. */ +void unset_nmi_pm_callback(struct pm_dev *dev); + #else -# include "nmi_64.h" + +static inline struct pm_dev *set_nmi_pm_callback(pm_callback callback) +{ + return 0; +} + +static inline void unset_nmi_pm_callback(struct pm_dev *dev) +{ +} + +#endif /* CONFIG_PM */ + +#ifdef CONFIG_X86_64 +extern void default_do_nmi(struct pt_regs *); +extern void die_nmi(char *str, struct pt_regs *regs, int do_panic); +extern void nmi_watchdog_default(void); +#else +#define nmi_watchdog_default() do {} while (0) +#endif + +extern int check_nmi_watchdog(void); +extern int nmi_watchdog_enabled; +extern int unknown_nmi_panic; +extern int avail_to_resrv_perfctr_nmi_bit(unsigned int); +extern int avail_to_resrv_perfctr_nmi(unsigned int); +extern int reserve_perfctr_nmi(unsigned int); +extern void release_perfctr_nmi(unsigned int); +extern int reserve_evntsel_nmi(unsigned int); +extern void release_evntsel_nmi(unsigned int); + +extern void setup_apic_nmi_watchdog(void *); +extern void stop_apic_nmi_watchdog(void *); +extern void disable_timer_nmi_watchdog(void); +extern void enable_timer_nmi_watchdog(void); +extern int nmi_watchdog_tick(struct pt_regs *regs, unsigned reason); + +extern atomic_t nmi_active; +extern unsigned int nmi_watchdog; +#define NMI_DISABLED -1 +#define NMI_NONE 0 +#define NMI_IO_APIC 1 +#define NMI_LOCAL_APIC 2 +#define NMI_INVALID 3 +#define NMI_DEFAULT NMI_DISABLED + +struct ctl_table; +struct file; +extern int proc_nmi_enabled(struct ctl_table *, int , struct file *, + void __user *, size_t *, loff_t *); +extern int unknown_nmi_panic; + +void __trigger_all_cpu_backtrace(void); +#define trigger_all_cpu_backtrace() __trigger_all_cpu_backtrace() + +#endif + +void lapic_watchdog_stop(void); +int lapic_watchdog_init(unsigned nmi_hz); +int lapic_wd_event(unsigned nmi_hz); +unsigned lapic_adjust_nmi_hz(unsigned hz); +int lapic_watchdog_ok(void); +void disable_lapic_nmi_watchdog(void); +void enable_lapic_nmi_watchdog(void); +void stop_nmi(void); +void restart_nmi(void); + #endif diff --git a/include/asm-x86/nmi_32.h b/include/asm-x86/nmi_32.h deleted file mode 100644 index 7206c7e8a38..00000000000 --- a/include/asm-x86/nmi_32.h +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef ASM_NMI_H -#define ASM_NMI_H - -#include <linux/pm.h> -#include <asm/irq.h> - -#ifdef ARCH_HAS_NMI_WATCHDOG - -/** - * do_nmi_callback - * - * Check to see if a callback exists and execute it. Return 1 - * if the handler exists and was handled successfully. - */ -int do_nmi_callback(struct pt_regs *regs, int cpu); - -extern int nmi_watchdog_enabled; -extern int avail_to_resrv_perfctr_nmi_bit(unsigned int); -extern int avail_to_resrv_perfctr_nmi(unsigned int); -extern int reserve_perfctr_nmi(unsigned int); -extern void release_perfctr_nmi(unsigned int); -extern int reserve_evntsel_nmi(unsigned int); -extern void release_evntsel_nmi(unsigned int); - -extern void setup_apic_nmi_watchdog (void *); -extern void stop_apic_nmi_watchdog (void *); -extern void disable_timer_nmi_watchdog(void); -extern void enable_timer_nmi_watchdog(void); -extern int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); - -extern atomic_t nmi_active; -extern unsigned int nmi_watchdog; -#define NMI_DISABLED -1 -#define NMI_NONE 0 -#define NMI_IO_APIC 1 -#define NMI_LOCAL_APIC 2 -#define NMI_INVALID 3 -#define NMI_DEFAULT NMI_DISABLED - -struct ctl_table; -struct file; -extern int proc_nmi_enabled(struct ctl_table *, int , struct file *, - void __user *, size_t *, loff_t *); -extern int unknown_nmi_panic; - -void __trigger_all_cpu_backtrace(void); -#define trigger_all_cpu_backtrace() __trigger_all_cpu_backtrace() - -#endif - -void lapic_watchdog_stop(void); -int lapic_watchdog_init(unsigned nmi_hz); -int lapic_wd_event(unsigned nmi_hz); -unsigned lapic_adjust_nmi_hz(unsigned hz); -int lapic_watchdog_ok(void); -void disable_lapic_nmi_watchdog(void); -void enable_lapic_nmi_watchdog(void); -void stop_nmi(void); -void restart_nmi(void); - -#endif /* ASM_NMI_H */ diff --git a/include/asm-x86/nmi_64.h b/include/asm-x86/nmi_64.h deleted file mode 100644 index 2eeb74e5f3f..00000000000 --- a/include/asm-x86/nmi_64.h +++ /dev/null @@ -1,90 +0,0 @@ -#ifndef ASM_NMI_H -#define ASM_NMI_H - -#include <linux/pm.h> -#include <asm/io.h> - -/** - * do_nmi_callback - * - * Check to see if a callback exists and execute it. Return 1 - * if the handler exists and was handled successfully. - */ -int do_nmi_callback(struct pt_regs *regs, int cpu); - -#ifdef CONFIG_PM - -/** Replace the PM callback routine for NMI. */ -struct pm_dev * set_nmi_pm_callback(pm_callback callback); - -/** Unset the PM callback routine back to the default. */ -void unset_nmi_pm_callback(struct pm_dev * dev); - -#else - -static inline struct pm_dev * set_nmi_pm_callback(pm_callback callback) -{ - return 0; -} - -static inline void unset_nmi_pm_callback(struct pm_dev * dev) -{ -} - -#endif /* CONFIG_PM */ - -extern void default_do_nmi(struct pt_regs *); -extern void die_nmi(char *str, struct pt_regs *regs, int do_panic); - -#define get_nmi_reason() inb(0x61) - -extern int unknown_nmi_panic; -extern int nmi_watchdog_enabled; - -extern int check_nmi_watchdog(void); -extern int avail_to_resrv_perfctr_nmi_bit(unsigned int); -extern int avail_to_resrv_perfctr_nmi(unsigned int); -extern int reserve_perfctr_nmi(unsigned int); -extern void release_perfctr_nmi(unsigned int); -extern int reserve_evntsel_nmi(unsigned int); -extern void release_evntsel_nmi(unsigned int); - -extern void setup_apic_nmi_watchdog (void *); -extern void stop_apic_nmi_watchdog (void *); -extern void disable_timer_nmi_watchdog(void); -extern void enable_timer_nmi_watchdog(void); -extern int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); - -extern void nmi_watchdog_default(void); - -extern atomic_t nmi_active; -extern unsigned int nmi_watchdog; -#define NMI_DISABLED -1 -#define NMI_NONE 0 -#define NMI_IO_APIC 1 -#define NMI_LOCAL_APIC 2 -#define NMI_INVALID 3 -#define NMI_DEFAULT NMI_DISABLED - -struct ctl_table; -struct file; -extern int proc_nmi_enabled(struct ctl_table *, int , struct file *, - void __user *, size_t *, loff_t *); - -extern int unknown_nmi_panic; - -void __trigger_all_cpu_backtrace(void); -#define trigger_all_cpu_backtrace() __trigger_all_cpu_backtrace() - - -void lapic_watchdog_stop(void); -int lapic_watchdog_init(unsigned nmi_hz); -int lapic_wd_event(unsigned nmi_hz); -unsigned lapic_adjust_nmi_hz(unsigned hz); -int lapic_watchdog_ok(void); -void disable_lapic_nmi_watchdog(void); -void enable_lapic_nmi_watchdog(void); -void stop_nmi(void); -void restart_nmi(void); - -#endif /* ASM_NMI_H */ diff --git a/include/asm-x86/nops.h b/include/asm-x86/nops.h index b3930ae539b..ad0bedd10b8 100644 --- a/include/asm-x86/nops.h +++ b/include/asm-x86/nops.h @@ -5,6 +5,8 @@ /* generic versions from gas 1: nop + the following instructions are NOT nops in 64-bit mode, + for 64-bit mode use K8 or P6 nops instead 2: movl %esi,%esi 3: leal 0x00(%esi),%esi 4: leal 0x00(,%esi,1),%esi diff --git a/include/asm-x86/numa_64.h b/include/asm-x86/numa_64.h index 15fe07cde58..22e87c9f6a8 100644 --- a/include/asm-x86/numa_64.h +++ b/include/asm-x86/numa_64.h @@ -1,14 +1,16 @@ -#ifndef _ASM_X8664_NUMA_H +#ifndef _ASM_X8664_NUMA_H #define _ASM_X8664_NUMA_H 1 #include <linux/nodemask.h> #include <asm/apicdef.h> struct bootnode { - u64 start,end; + u64 start; + u64 end; }; -extern int compute_hash_shift(struct bootnode *nodes, int numnodes); +extern int compute_hash_shift(struct bootnode *nodes, int numblks, + int *nodeids); #define ZONE_ALIGN (1UL << (MAX_ORDER+PAGE_SHIFT)) diff --git a/include/asm-x86/numaq.h b/include/asm-x86/numaq.h index 38f710dc37f..94b86c31239 100644 --- a/include/asm-x86/numaq.h +++ b/include/asm-x86/numaq.h @@ -3,7 +3,7 @@ * * Copyright (C) 2002, IBM Corp. * - * All rights reserved. + * All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -33,7 +33,8 @@ extern int get_memcfg_numaq(void); /* * SYS_CFG_DATA_PRIV_ADDR, struct eachquadmem, and struct sys_cfg_data are the */ -#define SYS_CFG_DATA_PRIV_ADDR 0x0009d000 /* place for scd in private quad space */ +#define SYS_CFG_DATA_PRIV_ADDR 0x0009d000 /* place for scd in private + quad space */ /* * Communication area for each processor on lynxer-processor tests. @@ -139,7 +140,7 @@ struct sys_cfg_data { unsigned int low_shrd_mem_base; /* 0 or 512MB or 1GB */ unsigned int low_shrd_mem_quad_offset; /* 0,128M,256M,512M,1G */ /* may not be totally populated */ - unsigned int split_mem_enbl; /* 0 for no low shared memory */ + unsigned int split_mem_enbl; /* 0 for no low shared memory */ unsigned int mmio_sz; /* Size of total system memory mapped I/O */ /* (in MB). */ unsigned int quad_spin_lock; /* Spare location used for quad */ @@ -152,7 +153,7 @@ struct sys_cfg_data { /* * memory configuration area for each quad */ - struct eachquadmem eq[MAX_NUMNODES]; /* indexed by quad id */ + struct eachquadmem eq[MAX_NUMNODES]; /* indexed by quad id */ }; static inline unsigned long *get_zholes_size(int nid) diff --git a/include/asm-x86/olpc.h b/include/asm-x86/olpc.h new file mode 100644 index 00000000000..97d47133486 --- /dev/null +++ b/include/asm-x86/olpc.h @@ -0,0 +1,132 @@ +/* OLPC machine specific definitions */ + +#ifndef ASM_OLPC_H_ +#define ASM_OLPC_H_ + +#include <asm/geode.h> + +struct olpc_platform_t { + int flags; + uint32_t boardrev; + int ecver; +}; + +#define OLPC_F_PRESENT 0x01 +#define OLPC_F_DCON 0x02 +#define OLPC_F_VSA 0x04 + +#ifdef CONFIG_OLPC + +extern struct olpc_platform_t olpc_platform_info; + +/* + * OLPC board IDs contain the major build number within the mask 0x0ff0, + * and the minor build number withing 0x000f. Pre-builds have a minor + * number less than 8, and normal builds start at 8. For example, 0x0B10 + * is a PreB1, and 0x0C18 is a C1. + */ + +static inline uint32_t olpc_board(uint8_t id) +{ + return (id << 4) | 0x8; +} + +static inline uint32_t olpc_board_pre(uint8_t id) +{ + return id << 4; +} + +static inline int machine_is_olpc(void) +{ + return (olpc_platform_info.flags & OLPC_F_PRESENT) ? 1 : 0; +} + +/* + * The DCON is OLPC's Display Controller. It has a number of unique + * features that we might want to take advantage of.. + */ +static inline int olpc_has_dcon(void) +{ + return (olpc_platform_info.flags & OLPC_F_DCON) ? 1 : 0; +} + +/* + * The VSA is software from AMD that typical Geode bioses will include. + * It is used to emulate the PCI bus, VGA, etc. OLPC's Open Firmware does + * not include the VSA; instead, PCI is emulated by the kernel. + * + * The VSA is described further in arch/x86/pci/olpc.c. + */ +static inline int olpc_has_vsa(void) +{ + return (olpc_platform_info.flags & OLPC_F_VSA) ? 1 : 0; +} + +/* + * The "Mass Production" version of OLPC's XO is identified as being model + * C2. During the prototype phase, the following models (in chronological + * order) were created: A1, B1, B2, B3, B4, C1. The A1 through B2 models + * were based on Geode GX CPUs, and models after that were based upon + * Geode LX CPUs. There were also some hand-assembled models floating + * around, referred to as PreB1, PreB2, etc. + */ +static inline int olpc_board_at_least(uint32_t rev) +{ + return olpc_platform_info.boardrev >= rev; +} + +#else + +static inline int machine_is_olpc(void) +{ + return 0; +} + +static inline int olpc_has_dcon(void) +{ + return 0; +} + +static inline int olpc_has_vsa(void) +{ + return 0; +} + +#endif + +/* EC related functions */ + +extern int olpc_ec_cmd(unsigned char cmd, unsigned char *inbuf, size_t inlen, + unsigned char *outbuf, size_t outlen); + +extern int olpc_ec_mask_set(uint8_t bits); +extern int olpc_ec_mask_unset(uint8_t bits); + +/* EC commands */ + +#define EC_FIRMWARE_REV 0x08 + +/* SCI source values */ + +#define EC_SCI_SRC_EMPTY 0x00 +#define EC_SCI_SRC_GAME 0x01 +#define EC_SCI_SRC_BATTERY 0x02 +#define EC_SCI_SRC_BATSOC 0x04 +#define EC_SCI_SRC_BATERR 0x08 +#define EC_SCI_SRC_EBOOK 0x10 +#define EC_SCI_SRC_WLAN 0x20 +#define EC_SCI_SRC_ACPWR 0x40 +#define EC_SCI_SRC_ALL 0x7F + +/* GPIO assignments */ + +#define OLPC_GPIO_MIC_AC geode_gpio(1) +#define OLPC_GPIO_DCON_IRQ geode_gpio(7) +#define OLPC_GPIO_THRM_ALRM geode_gpio(10) +#define OLPC_GPIO_SMB_CLK geode_gpio(14) +#define OLPC_GPIO_SMB_DATA geode_gpio(15) +#define OLPC_GPIO_WORKAUX geode_gpio(24) +#define OLPC_GPIO_LID geode_gpio(26) +#define OLPC_GPIO_ECSCI geode_gpio(27) + +#endif diff --git a/include/asm-x86/page.h b/include/asm-x86/page.h index a05b2896492..b381f4a5a0b 100644 --- a/include/asm-x86/page.h +++ b/include/asm-x86/page.h @@ -33,10 +33,8 @@ #ifdef CONFIG_X86_64 #include <asm/page_64.h> -#define max_pfn_mapped end_pfn_map #else #include <asm/page_32.h> -#define max_pfn_mapped max_low_pfn #endif /* CONFIG_X86_64 */ #define PAGE_OFFSET ((unsigned long)__PAGE_OFFSET) @@ -49,6 +47,9 @@ #ifndef __ASSEMBLY__ extern int page_is_ram(unsigned long pagenr); +extern int devmem_is_allowed(unsigned long pagenr); + +extern unsigned long max_pfn_mapped; struct page; diff --git a/include/asm-x86/page_32.h b/include/asm-x86/page_32.h index 5f7257fd589..424e82f8ae2 100644 --- a/include/asm-x86/page_32.h +++ b/include/asm-x86/page_32.h @@ -47,7 +47,10 @@ typedef unsigned long pgdval_t; typedef unsigned long pgprotval_t; typedef unsigned long phys_addr_t; -typedef union { pteval_t pte, pte_low; } pte_t; +typedef union { + pteval_t pte; + pteval_t pte_low; +} pte_t; #endif /* __ASSEMBLY__ */ #endif /* CONFIG_X86_PAE */ @@ -61,7 +64,7 @@ typedef struct page *pgtable_t; #endif #ifndef __ASSEMBLY__ -#define __phys_addr(x) ((x)-PAGE_OFFSET) +#define __phys_addr(x) ((x) - PAGE_OFFSET) #define __phys_reloc_hide(x) RELOC_HIDE((x), 0) #ifdef CONFIG_FLATMEM @@ -78,7 +81,7 @@ extern unsigned int __VMALLOC_RESERVE; extern int sysctl_legacy_va_layout; #define VMALLOC_RESERVE ((unsigned long)__VMALLOC_RESERVE) -#define MAXMEM (-__PAGE_OFFSET-__VMALLOC_RESERVE) +#define MAXMEM (-__PAGE_OFFSET - __VMALLOC_RESERVE) #ifdef CONFIG_X86_USE_3DNOW #include <asm/mmx.h> diff --git a/include/asm-x86/page_64.h b/include/asm-x86/page_64.h index 143546073b9..6ea72859c49 100644 --- a/include/asm-x86/page_64.h +++ b/include/asm-x86/page_64.h @@ -5,7 +5,7 @@ #define THREAD_ORDER 1 #define THREAD_SIZE (PAGE_SIZE << THREAD_ORDER) -#define CURRENT_MASK (~(THREAD_SIZE-1)) +#define CURRENT_MASK (~(THREAD_SIZE - 1)) #define EXCEPTION_STACK_ORDER 0 #define EXCEPTION_STKSZ (PAGE_SIZE << EXCEPTION_STACK_ORDER) @@ -48,10 +48,10 @@ #define __VIRTUAL_MASK_SHIFT 48 /* - * Kernel image size is limited to 128 MB (see level2_kernel_pgt in + * Kernel image size is limited to 512 MB (see level2_kernel_pgt in * arch/x86/kernel/head_64.S), and it is mapped here: */ -#define KERNEL_IMAGE_SIZE (128*1024*1024) +#define KERNEL_IMAGE_SIZE (512 * 1024 * 1024) #define KERNEL_IMAGE_START _AC(0xffffffff80000000, UL) #ifndef __ASSEMBLY__ @@ -59,7 +59,6 @@ void clear_page(void *page); void copy_page(void *to, void *from); extern unsigned long end_pfn; -extern unsigned long end_pfn_map; extern unsigned long phys_base; extern unsigned long __phys_addr(unsigned long); @@ -81,6 +80,9 @@ typedef struct { pteval_t pte; } pte_t; #define vmemmap ((struct page *)VMEMMAP_START) +extern unsigned long init_memory_mapping(unsigned long start, + unsigned long end); + #endif /* !__ASSEMBLY__ */ #ifdef CONFIG_FLATMEM diff --git a/include/asm-x86/param.h b/include/asm-x86/param.h index c996ec4da0c..6f0d0422f4c 100644 --- a/include/asm-x86/param.h +++ b/include/asm-x86/param.h @@ -3,8 +3,8 @@ #ifdef __KERNEL__ # define HZ CONFIG_HZ /* Internal kernel timer frequency */ -# define USER_HZ 100 /* .. some user interfaces are in "ticks" */ -# define CLOCKS_PER_SEC (USER_HZ) /* like times() */ +# define USER_HZ 100 /* some user interfaces are */ +# define CLOCKS_PER_SEC (USER_HZ) /* in "ticks" like times() */ #endif #ifndef HZ diff --git a/include/asm-x86/paravirt.h b/include/asm-x86/paravirt.h index d6236eb4646..0f13b945e24 100644 --- a/include/asm-x86/paravirt.h +++ b/include/asm-x86/paravirt.h @@ -220,18 +220,21 @@ struct pv_mmu_ops { unsigned long va); /* Hooks for allocating/releasing pagetable pages */ - void (*alloc_pt)(struct mm_struct *mm, u32 pfn); - void (*alloc_pd)(struct mm_struct *mm, u32 pfn); - void (*alloc_pd_clone)(u32 pfn, u32 clonepfn, u32 start, u32 count); - void (*release_pt)(u32 pfn); - void (*release_pd)(u32 pfn); + void (*alloc_pte)(struct mm_struct *mm, u32 pfn); + void (*alloc_pmd)(struct mm_struct *mm, u32 pfn); + void (*alloc_pmd_clone)(u32 pfn, u32 clonepfn, u32 start, u32 count); + void (*alloc_pud)(struct mm_struct *mm, u32 pfn); + void (*release_pte)(u32 pfn); + void (*release_pmd)(u32 pfn); + void (*release_pud)(u32 pfn); /* Pagetable manipulation functions */ void (*set_pte)(pte_t *ptep, pte_t pteval); void (*set_pte_at)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval); void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval); - void (*pte_update)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); + void (*pte_update)(struct mm_struct *mm, unsigned long addr, + pte_t *ptep); void (*pte_update_defer)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); @@ -246,7 +249,8 @@ struct pv_mmu_ops { void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); - void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); + void (*pte_clear)(struct mm_struct *mm, unsigned long addr, + pte_t *ptep); void (*pmd_clear)(pmd_t *pmdp); #endif /* CONFIG_X86_PAE */ @@ -274,8 +278,7 @@ struct pv_mmu_ops { /* This contains all the paravirt structures: we get a convenient * number for each function using the offset which we use to indicate * what to patch. */ -struct paravirt_patch_template -{ +struct paravirt_patch_template { struct pv_init_ops pv_init_ops; struct pv_time_ops pv_time_ops; struct pv_cpu_ops pv_cpu_ops; @@ -660,43 +663,56 @@ static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) } /* These should all do BUG_ON(_err), but our headers are too tangled. */ -#define rdmsr(msr,val1,val2) do { \ +#define rdmsr(msr, val1, val2) \ +do { \ int _err; \ u64 _l = paravirt_read_msr(msr, &_err); \ val1 = (u32)_l; \ val2 = _l >> 32; \ -} while(0) +} while (0) -#define wrmsr(msr,val1,val2) do { \ +#define wrmsr(msr, val1, val2) \ +do { \ paravirt_write_msr(msr, val1, val2); \ -} while(0) +} while (0) -#define rdmsrl(msr,val) do { \ +#define rdmsrl(msr, val) \ +do { \ int _err; \ val = paravirt_read_msr(msr, &_err); \ -} while(0) +} while (0) -#define wrmsrl(msr,val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32) -#define wrmsr_safe(msr,a,b) paravirt_write_msr(msr, a, b) +#define wrmsrl(msr, val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32) +#define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b) /* rdmsr with exception handling */ -#define rdmsr_safe(msr,a,b) ({ \ +#define rdmsr_safe(msr, a, b) \ +({ \ int _err; \ u64 _l = paravirt_read_msr(msr, &_err); \ (*a) = (u32)_l; \ (*b) = _l >> 32; \ - _err; }) + _err; \ +}) +static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) +{ + int err; + + *p = paravirt_read_msr(msr, &err); + return err; +} static inline u64 paravirt_read_tsc(void) { return PVOP_CALL0(u64, pv_cpu_ops.read_tsc); } -#define rdtscl(low) do { \ +#define rdtscl(low) \ +do { \ u64 _l = paravirt_read_tsc(); \ low = (int)_l; \ -} while(0) +} while (0) #define rdtscll(val) (val = paravirt_read_tsc()) @@ -711,11 +727,12 @@ static inline unsigned long long paravirt_read_pmc(int counter) return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter); } -#define rdpmc(counter,low,high) do { \ +#define rdpmc(counter, low, high) \ +do { \ u64 _l = paravirt_read_pmc(counter); \ low = (u32)_l; \ high = _l >> 32; \ -} while(0) +} while (0) static inline unsigned long long paravirt_rdtscp(unsigned int *aux) { @@ -794,7 +811,8 @@ static inline void set_iopl_mask(unsigned mask) } /* The paravirtualized I/O functions */ -static inline void slow_down_io(void) { +static inline void slow_down_io(void) +{ pv_cpu_ops.io_delay(); #ifdef REALLY_SLOW_IO pv_cpu_ops.io_delay(); @@ -894,28 +912,37 @@ static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, &cpumask, mm, va); } -static inline void paravirt_alloc_pt(struct mm_struct *mm, unsigned pfn) +static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned pfn) { - PVOP_VCALL2(pv_mmu_ops.alloc_pt, mm, pfn); + PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn); } -static inline void paravirt_release_pt(unsigned pfn) +static inline void paravirt_release_pte(unsigned pfn) { - PVOP_VCALL1(pv_mmu_ops.release_pt, pfn); + PVOP_VCALL1(pv_mmu_ops.release_pte, pfn); } -static inline void paravirt_alloc_pd(struct mm_struct *mm, unsigned pfn) +static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned pfn) +{ + PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn); +} + +static inline void paravirt_alloc_pmd_clone(unsigned pfn, unsigned clonepfn, + unsigned start, unsigned count) +{ + PVOP_VCALL4(pv_mmu_ops.alloc_pmd_clone, pfn, clonepfn, start, count); +} +static inline void paravirt_release_pmd(unsigned pfn) { - PVOP_VCALL2(pv_mmu_ops.alloc_pd, mm, pfn); + PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn); } -static inline void paravirt_alloc_pd_clone(unsigned pfn, unsigned clonepfn, - unsigned start, unsigned count) +static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned pfn) { - PVOP_VCALL4(pv_mmu_ops.alloc_pd_clone, pfn, clonepfn, start, count); + PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn); } -static inline void paravirt_release_pd(unsigned pfn) +static inline void paravirt_release_pud(unsigned pfn) { - PVOP_VCALL1(pv_mmu_ops.release_pd, pfn); + PVOP_VCALL1(pv_mmu_ops.release_pud, pfn); } #ifdef CONFIG_HIGHPTE diff --git a/include/asm-x86/parport.h b/include/asm-x86/parport.h index 019cbca24a3..3c4ffeb467e 100644 --- a/include/asm-x86/parport.h +++ b/include/asm-x86/parport.h @@ -1,10 +1,10 @@ #ifndef _ASM_X86_PARPORT_H #define _ASM_X86_PARPORT_H -static int __devinit parport_pc_find_isa_ports (int autoirq, int autodma); -static int __devinit parport_pc_find_nonpci_ports (int autoirq, int autodma) +static int __devinit parport_pc_find_isa_ports(int autoirq, int autodma); +static int __devinit parport_pc_find_nonpci_ports(int autoirq, int autodma) { - return parport_pc_find_isa_ports (autoirq, autodma); + return parport_pc_find_isa_ports(autoirq, autodma); } #endif /* _ASM_X86_PARPORT_H */ diff --git a/include/asm-x86/pat.h b/include/asm-x86/pat.h new file mode 100644 index 00000000000..8b822b5a178 --- /dev/null +++ b/include/asm-x86/pat.h @@ -0,0 +1,16 @@ + +#ifndef _ASM_PAT_H +#define _ASM_PAT_H 1 + +#include <linux/types.h> + +extern int pat_wc_enabled; + +extern void pat_init(void); + +extern int reserve_memtype(u64 start, u64 end, + unsigned long req_type, unsigned long *ret_type); +extern int free_memtype(u64 start, u64 end); + +#endif + diff --git a/include/asm-x86/pci-direct.h b/include/asm-x86/pci-direct.h index 6823fa4f1af..5b21485be57 100644 --- a/include/asm-x86/pci-direct.h +++ b/include/asm-x86/pci-direct.h @@ -4,7 +4,7 @@ #include <linux/types.h> /* Direct PCI access. This is used for PCI accesses in early boot before - the PCI subsystem works. */ + the PCI subsystem works. */ extern u32 read_pci_config(u8 bus, u8 slot, u8 func, u8 offset); extern u8 read_pci_config_byte(u8 bus, u8 slot, u8 func, u8 offset); diff --git a/include/asm-x86/pci.h b/include/asm-x86/pci.h index c61190cb9e1..30bbde0cb34 100644 --- a/include/asm-x86/pci.h +++ b/include/asm-x86/pci.h @@ -8,18 +8,19 @@ #include <asm/scatterlist.h> #include <asm/io.h> - #ifdef __KERNEL__ struct pci_sysdata { int domain; /* PCI domain */ int node; /* NUMA node */ #ifdef CONFIG_X86_64 - void* iommu; /* IOMMU private data */ + void *iommu; /* IOMMU private data */ #endif }; /* scan a bus after allocating a pci_sysdata for it */ +extern struct pci_bus *pci_scan_bus_on_node(int busno, struct pci_ops *ops, + int node); extern struct pci_bus *pci_scan_bus_with_sysdata(int busno); static inline int pci_domain_nr(struct pci_bus *bus) @@ -52,7 +53,7 @@ extern unsigned long pci_mem_start; #define PCIBIOS_MIN_CARDBUS_IO 0x4000 void pcibios_config_init(void); -struct pci_bus * pcibios_scan_root(int bus); +struct pci_bus *pcibios_scan_root(int bus); void pcibios_set_master(struct pci_dev *dev); void pcibios_penalize_isa_irq(int irq, int active); @@ -62,7 +63,8 @@ int pcibios_set_irq_routing(struct pci_dev *dev, int pin, int irq); #define HAVE_PCI_MMAP extern int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, - enum pci_mmap_state mmap_state, int write_combine); + enum pci_mmap_state mmap_state, + int write_combine); #ifdef CONFIG_PCI diff --git a/include/asm-x86/pci_64.h b/include/asm-x86/pci_64.h index 37469031453..f330234ffa5 100644 --- a/include/asm-x86/pci_64.h +++ b/include/asm-x86/pci_64.h @@ -1,12 +1,10 @@ #ifndef __x8664_PCI_H #define __x8664_PCI_H - #ifdef __KERNEL__ - #ifdef CONFIG_CALGARY_IOMMU -static inline void* pci_iommu(struct pci_bus *bus) +static inline void *pci_iommu(struct pci_bus *bus) { struct pci_sysdata *sd = bus->sysdata; return sd->iommu; @@ -19,12 +17,12 @@ static inline void set_pci_iommu(struct pci_bus *bus, void *val) } #endif /* CONFIG_CALGARY_IOMMU */ +extern int (*pci_config_read)(int seg, int bus, int dev, int fn, + int reg, int len, u32 *value); +extern int (*pci_config_write)(int seg, int bus, int dev, int fn, + int reg, int len, u32 value); -extern int (*pci_config_read)(int seg, int bus, int dev, int fn, int reg, int len, u32 *value); -extern int (*pci_config_write)(int seg, int bus, int dev, int fn, int reg, int len, u32 value); - - - +extern void dma32_reserve_bootmem(void); extern void pci_iommu_alloc(void); /* The PCI address space does equal the physical memory @@ -65,5 +63,4 @@ extern void pci_iommu_alloc(void); #endif /* __KERNEL__ */ - #endif /* __x8664_PCI_H */ diff --git a/include/asm-x86/pda.h b/include/asm-x86/pda.h index c0305bff0f1..101fb9e1195 100644 --- a/include/asm-x86/pda.h +++ b/include/asm-x86/pda.h @@ -22,7 +22,6 @@ struct x8664_pda { offset 40!!! */ #endif char *irqstackptr; - unsigned int nodenumber; /* number of current node */ unsigned int __softirq_pending; unsigned int __nmi_count; /* number of NMI on this CPUs */ short mmu_state; @@ -58,34 +57,36 @@ extern struct x8664_pda _proxy_pda; #define pda_offset(field) offsetof(struct x8664_pda, field) -#define pda_to_op(op, field, val) do { \ - typedef typeof(_proxy_pda.field) T__; \ - if (0) { T__ tmp__; tmp__ = (val); } /* type checking */ \ - switch (sizeof(_proxy_pda.field)) { \ - case 2: \ - asm(op "w %1,%%gs:%c2" : \ - "+m" (_proxy_pda.field) : \ - "ri" ((T__)val), \ - "i"(pda_offset(field))); \ - break; \ - case 4: \ - asm(op "l %1,%%gs:%c2" : \ - "+m" (_proxy_pda.field) : \ - "ri" ((T__)val), \ - "i" (pda_offset(field))); \ - break; \ - case 8: \ - asm(op "q %1,%%gs:%c2": \ - "+m" (_proxy_pda.field) : \ - "ri" ((T__)val), \ - "i"(pda_offset(field))); \ - break; \ - default: \ - __bad_pda_field(); \ - } \ - } while (0) +#define pda_to_op(op, field, val) \ +do { \ + typedef typeof(_proxy_pda.field) T__; \ + if (0) { T__ tmp__; tmp__ = (val); } /* type checking */ \ + switch (sizeof(_proxy_pda.field)) { \ + case 2: \ + asm(op "w %1,%%gs:%c2" : \ + "+m" (_proxy_pda.field) : \ + "ri" ((T__)val), \ + "i"(pda_offset(field))); \ + break; \ + case 4: \ + asm(op "l %1,%%gs:%c2" : \ + "+m" (_proxy_pda.field) : \ + "ri" ((T__)val), \ + "i" (pda_offset(field))); \ + break; \ + case 8: \ + asm(op "q %1,%%gs:%c2": \ + "+m" (_proxy_pda.field) : \ + "ri" ((T__)val), \ + "i"(pda_offset(field))); \ + break; \ + default: \ + __bad_pda_field(); \ + } \ +} while (0) -#define pda_from_op(op,field) ({ \ +#define pda_from_op(op, field) \ +({ \ typeof(_proxy_pda.field) ret__; \ switch (sizeof(_proxy_pda.field)) { \ case 2: \ @@ -93,23 +94,24 @@ extern struct x8664_pda _proxy_pda; "=r" (ret__) : \ "i" (pda_offset(field)), \ "m" (_proxy_pda.field)); \ - break; \ + break; \ case 4: \ asm(op "l %%gs:%c1,%0": \ "=r" (ret__): \ "i" (pda_offset(field)), \ "m" (_proxy_pda.field)); \ - break; \ + break; \ case 8: \ asm(op "q %%gs:%c1,%0": \ "=r" (ret__) : \ "i" (pda_offset(field)), \ "m" (_proxy_pda.field)); \ - break; \ + break; \ default: \ __bad_pda_field(); \ - } \ - ret__; }) + } \ + ret__; \ +}) #define read_pda(field) pda_from_op("mov", field) #define write_pda(field, val) pda_to_op("mov", field, val) @@ -118,12 +120,13 @@ extern struct x8664_pda _proxy_pda; #define or_pda(field, val) pda_to_op("or", field, val) /* This is not atomic against other CPUs -- CPU preemption needs to be off */ -#define test_and_clear_bit_pda(bit, field) ({ \ - int old__; \ - asm volatile("btr %2,%%gs:%c3\n\tsbbl %0,%0" \ - : "=r" (old__), "+m" (_proxy_pda.field) \ - : "dIr" (bit), "i" (pda_offset(field)) : "memory"); \ - old__; \ +#define test_and_clear_bit_pda(bit, field) \ +({ \ + int old__; \ + asm volatile("btr %2,%%gs:%c3\n\tsbbl %0,%0" \ + : "=r" (old__), "+m" (_proxy_pda.field) \ + : "dIr" (bit), "i" (pda_offset(field)) : "memory");\ + old__; \ }) #endif diff --git a/include/asm-x86/percpu.h b/include/asm-x86/percpu.h index 0dec00f27eb..736fc3bb8e1 100644 --- a/include/asm-x86/percpu.h +++ b/include/asm-x86/percpu.h @@ -85,58 +85,62 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off); * don't give an lvalue though). */ extern void __bad_percpu_size(void); -#define percpu_to_op(op,var,val) \ - do { \ - typedef typeof(var) T__; \ - if (0) { T__ tmp__; tmp__ = (val); } \ - switch (sizeof(var)) { \ - case 1: \ - asm(op "b %1,"__percpu_seg"%0" \ - : "+m" (var) \ - :"ri" ((T__)val)); \ - break; \ - case 2: \ - asm(op "w %1,"__percpu_seg"%0" \ - : "+m" (var) \ - :"ri" ((T__)val)); \ - break; \ - case 4: \ - asm(op "l %1,"__percpu_seg"%0" \ - : "+m" (var) \ - :"ri" ((T__)val)); \ - break; \ - default: __bad_percpu_size(); \ - } \ - } while (0) - -#define percpu_from_op(op,var) \ - ({ \ - typeof(var) ret__; \ - switch (sizeof(var)) { \ - case 1: \ - asm(op "b "__percpu_seg"%1,%0" \ - : "=r" (ret__) \ - : "m" (var)); \ - break; \ - case 2: \ - asm(op "w "__percpu_seg"%1,%0" \ - : "=r" (ret__) \ - : "m" (var)); \ - break; \ - case 4: \ - asm(op "l "__percpu_seg"%1,%0" \ - : "=r" (ret__) \ - : "m" (var)); \ - break; \ - default: __bad_percpu_size(); \ - } \ - ret__; }) +#define percpu_to_op(op, var, val) \ +do { \ + typedef typeof(var) T__; \ + if (0) { \ + T__ tmp__; \ + tmp__ = (val); \ + } \ + switch (sizeof(var)) { \ + case 1: \ + asm(op "b %1,"__percpu_seg"%0" \ + : "+m" (var) \ + : "ri" ((T__)val)); \ + break; \ + case 2: \ + asm(op "w %1,"__percpu_seg"%0" \ + : "+m" (var) \ + : "ri" ((T__)val)); \ + break; \ + case 4: \ + asm(op "l %1,"__percpu_seg"%0" \ + : "+m" (var) \ + : "ri" ((T__)val)); \ + break; \ + default: __bad_percpu_size(); \ + } \ +} while (0) + +#define percpu_from_op(op, var) \ +({ \ + typeof(var) ret__; \ + switch (sizeof(var)) { \ + case 1: \ + asm(op "b "__percpu_seg"%1,%0" \ + : "=r" (ret__) \ + : "m" (var)); \ + break; \ + case 2: \ + asm(op "w "__percpu_seg"%1,%0" \ + : "=r" (ret__) \ + : "m" (var)); \ + break; \ + case 4: \ + asm(op "l "__percpu_seg"%1,%0" \ + : "=r" (ret__) \ + : "m" (var)); \ + break; \ + default: __bad_percpu_size(); \ + } \ + ret__; \ +}) #define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var) -#define x86_write_percpu(var,val) percpu_to_op("mov", per_cpu__##var, val) -#define x86_add_percpu(var,val) percpu_to_op("add", per_cpu__##var, val) -#define x86_sub_percpu(var,val) percpu_to_op("sub", per_cpu__##var, val) -#define x86_or_percpu(var,val) percpu_to_op("or", per_cpu__##var, val) +#define x86_write_percpu(var, val) percpu_to_op("mov", per_cpu__##var, val) +#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val) +#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val) +#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val) #endif /* !__ASSEMBLY__ */ #endif /* !CONFIG_X86_64 */ #endif /* _ASM_X86_PERCPU_H_ */ diff --git a/include/asm-x86/pgalloc.h b/include/asm-x86/pgalloc.h index 5886eed0588..91e4641f3f3 100644 --- a/include/asm-x86/pgalloc.h +++ b/include/asm-x86/pgalloc.h @@ -1,5 +1,110 @@ -#ifdef CONFIG_X86_32 -# include "pgalloc_32.h" +#ifndef _ASM_X86_PGALLOC_H +#define _ASM_X86_PGALLOC_H + +#include <linux/threads.h> +#include <linux/mm.h> /* for struct page */ +#include <linux/pagemap.h> + +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> #else -# include "pgalloc_64.h" +static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) {} +static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) {} +static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn, + unsigned long start, unsigned long count) {} +static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) {} +static inline void paravirt_release_pte(unsigned long pfn) {} +static inline void paravirt_release_pmd(unsigned long pfn) {} +static inline void paravirt_release_pud(unsigned long pfn) {} #endif + +/* + * Allocate and free page tables. + */ +extern pgd_t *pgd_alloc(struct mm_struct *); +extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); + +extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long); +extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long); + +/* Should really implement gc for free page table pages. This could be + done with a reference count in struct page. */ + +static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) +{ + BUG_ON((unsigned long)pte & (PAGE_SIZE-1)); + free_page((unsigned long)pte); +} + +static inline void pte_free(struct mm_struct *mm, struct page *pte) +{ + __free_page(pte); +} + +extern void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte); + +static inline void pmd_populate_kernel(struct mm_struct *mm, + pmd_t *pmd, pte_t *pte) +{ + paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); + set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); +} + +static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, + struct page *pte) +{ + unsigned long pfn = page_to_pfn(pte); + + paravirt_alloc_pte(mm, pfn); + set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE)); +} + +#define pmd_pgtable(pmd) pmd_page(pmd) + +#if PAGETABLE_LEVELS > 2 +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return (pmd_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); +} + +static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) +{ + BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); + free_page((unsigned long)pmd); +} + +extern void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); + +#ifdef CONFIG_X86_PAE +extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd); +#else /* !CONFIG_X86_PAE */ +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) +{ + paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); + set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd))); +} +#endif /* CONFIG_X86_PAE */ + +#if PAGETABLE_LEVELS > 3 +static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) +{ + paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); + set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud))); +} + +static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return (pud_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); +} + +static inline void pud_free(struct mm_struct *mm, pud_t *pud) +{ + BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); + free_page((unsigned long)pud); +} + +extern void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud); +#endif /* PAGETABLE_LEVELS > 3 */ +#endif /* PAGETABLE_LEVELS > 2 */ + +#endif /* _ASM_X86_PGALLOC_H */ diff --git a/include/asm-x86/pgalloc_32.h b/include/asm-x86/pgalloc_32.h deleted file mode 100644 index 6bea6e5b5ee..00000000000 --- a/include/asm-x86/pgalloc_32.h +++ /dev/null @@ -1,95 +0,0 @@ -#ifndef _I386_PGALLOC_H -#define _I386_PGALLOC_H - -#include <linux/threads.h> -#include <linux/mm.h> /* for struct page */ -#include <linux/pagemap.h> -#include <asm/tlb.h> -#include <asm-generic/tlb.h> - -#ifdef CONFIG_PARAVIRT -#include <asm/paravirt.h> -#else -#define paravirt_alloc_pt(mm, pfn) do { } while (0) -#define paravirt_alloc_pd(mm, pfn) do { } while (0) -#define paravirt_alloc_pd_clone(pfn, clonepfn, start, count) do { } while (0) -#define paravirt_release_pt(pfn) do { } while (0) -#define paravirt_release_pd(pfn) do { } while (0) -#endif - -static inline void pmd_populate_kernel(struct mm_struct *mm, - pmd_t *pmd, pte_t *pte) -{ - paravirt_alloc_pt(mm, __pa(pte) >> PAGE_SHIFT); - set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); -} - -static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte) -{ - unsigned long pfn = page_to_pfn(pte); - - paravirt_alloc_pt(mm, pfn); - set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE)); -} -#define pmd_pgtable(pmd) pmd_page(pmd) - -/* - * Allocate and free page tables. - */ -extern pgd_t *pgd_alloc(struct mm_struct *); -extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); - -extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long); -extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long); - -static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) -{ - free_page((unsigned long)pte); -} - -static inline void pte_free(struct mm_struct *mm, pgtable_t pte) -{ - pgtable_page_dtor(pte); - __free_page(pte); -} - - -extern void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte); - -#ifdef CONFIG_X86_PAE -/* - * In the PAE case we free the pmds as part of the pgd. - */ -static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) -{ - return (pmd_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); -} - -static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) -{ - BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); - free_page((unsigned long)pmd); -} - -extern void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); - -static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd) -{ - paravirt_alloc_pd(mm, __pa(pmd) >> PAGE_SHIFT); - - /* Note: almost everything apart from _PAGE_PRESENT is - reserved at the pmd (PDPT) level. */ - set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT)); - - /* - * According to Intel App note "TLBs, Paging-Structure Caches, - * and Their Invalidation", April 2007, document 317080-001, - * section 8.1: in PAE mode we explicitly have to flush the - * TLB via cr3 if the top-level pgd is changed... - */ - if (mm == current->active_mm) - write_cr3(read_cr3()); -} -#endif /* CONFIG_X86_PAE */ - -#endif /* _I386_PGALLOC_H */ diff --git a/include/asm-x86/pgalloc_64.h b/include/asm-x86/pgalloc_64.h deleted file mode 100644 index 8d6722320dc..00000000000 --- a/include/asm-x86/pgalloc_64.h +++ /dev/null @@ -1,133 +0,0 @@ -#ifndef _X86_64_PGALLOC_H -#define _X86_64_PGALLOC_H - -#include <asm/pda.h> -#include <linux/threads.h> -#include <linux/mm.h> - -#define pmd_populate_kernel(mm, pmd, pte) \ - set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte))) -#define pud_populate(mm, pud, pmd) \ - set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd))) -#define pgd_populate(mm, pgd, pud) \ - set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud))) - -#define pmd_pgtable(pmd) pmd_page(pmd) - -static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte) -{ - set_pmd(pmd, __pmd(_PAGE_TABLE | (page_to_pfn(pte) << PAGE_SHIFT))); -} - -static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) -{ - BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); - free_page((unsigned long)pmd); -} - -static inline pmd_t *pmd_alloc_one (struct mm_struct *mm, unsigned long addr) -{ - return (pmd_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); -} - -static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) -{ - return (pud_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); -} - -static inline void pud_free(struct mm_struct *mm, pud_t *pud) -{ - BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); - free_page((unsigned long)pud); -} - -static inline void pgd_list_add(pgd_t *pgd) -{ - struct page *page = virt_to_page(pgd); - unsigned long flags; - - spin_lock_irqsave(&pgd_lock, flags); - list_add(&page->lru, &pgd_list); - spin_unlock_irqrestore(&pgd_lock, flags); -} - -static inline void pgd_list_del(pgd_t *pgd) -{ - struct page *page = virt_to_page(pgd); - unsigned long flags; - - spin_lock_irqsave(&pgd_lock, flags); - list_del(&page->lru); - spin_unlock_irqrestore(&pgd_lock, flags); -} - -static inline pgd_t *pgd_alloc(struct mm_struct *mm) -{ - unsigned boundary; - pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT); - if (!pgd) - return NULL; - pgd_list_add(pgd); - /* - * Copy kernel pointers in from init. - * Could keep a freelist or slab cache of those because the kernel - * part never changes. - */ - boundary = pgd_index(__PAGE_OFFSET); - memset(pgd, 0, boundary * sizeof(pgd_t)); - memcpy(pgd + boundary, - init_level4_pgt + boundary, - (PTRS_PER_PGD - boundary) * sizeof(pgd_t)); - return pgd; -} - -static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) -{ - BUG_ON((unsigned long)pgd & (PAGE_SIZE-1)); - pgd_list_del(pgd); - free_page((unsigned long)pgd); -} - -static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) -{ - return (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); -} - -static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) -{ - struct page *page; - void *p; - - p = (void *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); - if (!p) - return NULL; - page = virt_to_page(p); - pgtable_page_ctor(page); - return page; -} - -/* Should really implement gc for free page table pages. This could be - done with a reference count in struct page. */ - -static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) -{ - BUG_ON((unsigned long)pte & (PAGE_SIZE-1)); - free_page((unsigned long)pte); -} - -static inline void pte_free(struct mm_struct *mm, pgtable_t pte) -{ - pgtable_page_dtor(pte); - __free_page(pte); -} - -#define __pte_free_tlb(tlb,pte) \ -do { \ - pgtable_page_dtor((pte)); \ - tlb_remove_page((tlb), (pte)); \ -} while (0) - -#define __pmd_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x)) -#define __pud_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x)) - -#endif /* _X86_64_PGALLOC_H */ diff --git a/include/asm-x86/pgtable-2level.h b/include/asm-x86/pgtable-2level.h index 701404fab30..46bc52c0eae 100644 --- a/include/asm-x86/pgtable-2level.h +++ b/include/asm-x86/pgtable-2level.h @@ -26,7 +26,8 @@ static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) native_set_pte(ptep, pte); } -static inline void native_set_pte_present(struct mm_struct *mm, unsigned long addr, +static inline void native_set_pte_present(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, pte_t pte) { native_set_pte(ptep, pte); @@ -37,7 +38,8 @@ static inline void native_pmd_clear(pmd_t *pmdp) native_set_pmd(pmdp, __pmd(0)); } -static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *xp) +static inline void native_pte_clear(struct mm_struct *mm, + unsigned long addr, pte_t *xp) { *xp = native_make_pte(0); } @@ -61,16 +63,18 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp) */ #define PTE_FILE_MAX_BITS 29 -#define pte_to_pgoff(pte) \ - ((((pte).pte_low >> 1) & 0x1f ) + (((pte).pte_low >> 8) << 5 )) +#define pte_to_pgoff(pte) \ + ((((pte).pte_low >> 1) & 0x1f) + (((pte).pte_low >> 8) << 5)) -#define pgoff_to_pte(off) \ - ((pte_t) { .pte_low = (((off) & 0x1f) << 1) + (((off) >> 5) << 8) + _PAGE_FILE }) +#define pgoff_to_pte(off) \ + ((pte_t) { .pte_low = (((off) & 0x1f) << 1) + \ + (((off) >> 5) << 8) + _PAGE_FILE }) /* Encode and de-code a swap entry */ #define __swp_type(x) (((x).val >> 1) & 0x1f) #define __swp_offset(x) ((x).val >> 8) -#define __swp_entry(type, offset) ((swp_entry_t) { ((type) << 1) | ((offset) << 8) }) +#define __swp_entry(type, offset) \ + ((swp_entry_t) { ((type) << 1) | ((offset) << 8) }) #define __pte_to_swp_entry(pte) ((swp_entry_t) { (pte).pte_low }) #define __swp_entry_to_pte(x) ((pte_t) { .pte = (x).val }) diff --git a/include/asm-x86/pgtable-3level.h b/include/asm-x86/pgtable-3level.h index 1d763eec740..8b4a9d44b7f 100644 --- a/include/asm-x86/pgtable-3level.h +++ b/include/asm-x86/pgtable-3level.h @@ -8,22 +8,26 @@ * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com> */ -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %p(%08lx%08lx).\n", __FILE__, __LINE__, &(e), (e).pte_high, (e).pte_low) -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %p(%016Lx).\n", __FILE__, __LINE__, &(e), pmd_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %p(%016Lx).\n", __FILE__, __LINE__, &(e), pgd_val(e)) - +#define pte_ERROR(e) \ + printk("%s:%d: bad pte %p(%08lx%08lx).\n", \ + __FILE__, __LINE__, &(e), (e).pte_high, (e).pte_low) +#define pmd_ERROR(e) \ + printk("%s:%d: bad pmd %p(%016Lx).\n", \ + __FILE__, __LINE__, &(e), pmd_val(e)) +#define pgd_ERROR(e) \ + printk("%s:%d: bad pgd %p(%016Lx).\n", \ + __FILE__, __LINE__, &(e), pgd_val(e)) static inline int pud_none(pud_t pud) { return pud_val(pud) == 0; } + static inline int pud_bad(pud_t pud) { return (pud_val(pud) & ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER)) != 0; } + static inline int pud_present(pud_t pud) { return pud_val(pud) & _PAGE_PRESENT; @@ -48,7 +52,8 @@ static inline void native_set_pte(pte_t *ptep, pte_t pte) * we are justified in merely clearing the PTE present bit, followed * by a set. The ordering here is important. */ -static inline void native_set_pte_present(struct mm_struct *mm, unsigned long addr, +static inline void native_set_pte_present(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, pte_t pte) { ptep->pte_low = 0; @@ -60,15 +65,17 @@ static inline void native_set_pte_present(struct mm_struct *mm, unsigned long ad static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) { - set_64bit((unsigned long long *)(ptep),native_pte_val(pte)); + set_64bit((unsigned long long *)(ptep), native_pte_val(pte)); } + static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) { - set_64bit((unsigned long long *)(pmdp),native_pmd_val(pmd)); + set_64bit((unsigned long long *)(pmdp), native_pmd_val(pmd)); } + static inline void native_set_pud(pud_t *pudp, pud_t pud) { - set_64bit((unsigned long long *)(pudp),native_pud_val(pud)); + set_64bit((unsigned long long *)(pudp), native_pud_val(pud)); } /* @@ -76,7 +83,8 @@ static inline void native_set_pud(pud_t *pudp, pud_t pud) * entry, so clear the bottom half first and enforce ordering with a compiler * barrier. */ -static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) { ptep->pte_low = 0; smp_wmb(); @@ -107,20 +115,19 @@ static inline void pud_clear(pud_t *pudp) * current pgd to avoid unnecessary TLB flushes. */ pgd = read_cr3(); - if (__pa(pudp) >= pgd && __pa(pudp) < (pgd + sizeof(pgd_t)*PTRS_PER_PGD)) + if (__pa(pudp) >= pgd && __pa(pudp) < + (pgd + sizeof(pgd_t)*PTRS_PER_PGD)) write_cr3(pgd); } -#define pud_page(pud) \ -((struct page *) __va(pud_val(pud) & PAGE_MASK)) +#define pud_page(pud) ((struct page *) __va(pud_val(pud) & PAGE_MASK)) -#define pud_page_vaddr(pud) \ -((unsigned long) __va(pud_val(pud) & PAGE_MASK)) +#define pud_page_vaddr(pud) ((unsigned long) __va(pud_val(pud) & PAGE_MASK)) /* Find an entry in the second-level page table.. */ -#define pmd_offset(pud, address) ((pmd_t *) pud_page(*(pud)) + \ - pmd_index(address)) +#define pmd_offset(pud, address) ((pmd_t *)pud_page(*(pud)) + \ + pmd_index(address)) #ifdef CONFIG_SMP static inline pte_t native_ptep_get_and_clear(pte_t *ptep) @@ -161,7 +168,8 @@ static inline unsigned long pte_pfn(pte_t pte) * put the 32 bits of offset into the high part. */ #define pte_to_pgoff(pte) ((pte).pte_high) -#define pgoff_to_pte(off) ((pte_t) { { .pte_low = _PAGE_FILE, .pte_high = (off) } }) +#define pgoff_to_pte(off) \ + ((pte_t) { { .pte_low = _PAGE_FILE, .pte_high = (off) } }) #define PTE_FILE_MAX_BITS 32 /* Encode and de-code a swap entry */ diff --git a/include/asm-x86/pgtable.h b/include/asm-x86/pgtable.h index 9cf472aeb9c..801b31f7145 100644 --- a/include/asm-x86/pgtable.h +++ b/include/asm-x86/pgtable.h @@ -1,16 +1,15 @@ #ifndef _ASM_X86_PGTABLE_H #define _ASM_X86_PGTABLE_H -#define USER_PTRS_PER_PGD ((TASK_SIZE-1)/PGDIR_SIZE+1) #define FIRST_USER_ADDRESS 0 -#define _PAGE_BIT_PRESENT 0 -#define _PAGE_BIT_RW 1 -#define _PAGE_BIT_USER 2 -#define _PAGE_BIT_PWT 3 -#define _PAGE_BIT_PCD 4 -#define _PAGE_BIT_ACCESSED 5 -#define _PAGE_BIT_DIRTY 6 +#define _PAGE_BIT_PRESENT 0 /* is present */ +#define _PAGE_BIT_RW 1 /* writeable */ +#define _PAGE_BIT_USER 2 /* userspace addressable */ +#define _PAGE_BIT_PWT 3 /* page write through */ +#define _PAGE_BIT_PCD 4 /* page cache disabled */ +#define _PAGE_BIT_ACCESSED 5 /* was accessed (raised by CPU) */ +#define _PAGE_BIT_DIRTY 6 /* was written to (raised by CPU) */ #define _PAGE_BIT_FILE 6 #define _PAGE_BIT_PSE 7 /* 4 MB (or 2MB) page */ #define _PAGE_BIT_PAT 7 /* on 4KB pages */ @@ -48,24 +47,39 @@ #endif /* If _PAGE_PRESENT is clear, we use these: */ -#define _PAGE_FILE _PAGE_DIRTY /* nonlinear file mapping, saved PTE; unset:swap */ +#define _PAGE_FILE _PAGE_DIRTY /* nonlinear file mapping, + * saved PTE; unset:swap */ #define _PAGE_PROTNONE _PAGE_PSE /* if the user mapped it with PROT_NONE; pte_present gives true */ -#define _PAGE_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | _PAGE_ACCESSED | _PAGE_DIRTY) -#define _KERNPG_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY) +#define _PAGE_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | \ + _PAGE_ACCESSED | _PAGE_DIRTY) +#define _KERNPG_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | \ + _PAGE_DIRTY) #define _PAGE_CHG_MASK (PTE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY) -#define PAGE_NONE __pgprot(_PAGE_PROTNONE | _PAGE_ACCESSED) -#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | _PAGE_ACCESSED | _PAGE_NX) +#define _PAGE_CACHE_MASK (_PAGE_PCD | _PAGE_PWT) +#define _PAGE_CACHE_WB (0) +#define _PAGE_CACHE_WC (_PAGE_PWT) +#define _PAGE_CACHE_UC_MINUS (_PAGE_PCD) +#define _PAGE_CACHE_UC (_PAGE_PCD | _PAGE_PWT) -#define PAGE_SHARED_EXEC __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | _PAGE_ACCESSED) -#define PAGE_COPY_NOEXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED | _PAGE_NX) -#define PAGE_COPY_EXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED) +#define PAGE_NONE __pgprot(_PAGE_PROTNONE | _PAGE_ACCESSED) +#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | \ + _PAGE_ACCESSED | _PAGE_NX) + +#define PAGE_SHARED_EXEC __pgprot(_PAGE_PRESENT | _PAGE_RW | \ + _PAGE_USER | _PAGE_ACCESSED) +#define PAGE_COPY_NOEXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | \ + _PAGE_ACCESSED | _PAGE_NX) +#define PAGE_COPY_EXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | \ + _PAGE_ACCESSED) #define PAGE_COPY PAGE_COPY_NOEXEC -#define PAGE_READONLY __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED | _PAGE_NX) -#define PAGE_READONLY_EXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED) +#define PAGE_READONLY __pgprot(_PAGE_PRESENT | _PAGE_USER | \ + _PAGE_ACCESSED | _PAGE_NX) +#define PAGE_READONLY_EXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | \ + _PAGE_ACCESSED) #ifdef CONFIG_X86_32 #define _PAGE_KERNEL_EXEC \ @@ -84,6 +98,7 @@ extern pteval_t __PAGE_KERNEL, __PAGE_KERNEL_EXEC; #define __PAGE_KERNEL_RO (__PAGE_KERNEL & ~_PAGE_RW) #define __PAGE_KERNEL_RX (__PAGE_KERNEL_EXEC & ~_PAGE_RW) #define __PAGE_KERNEL_EXEC_NOCACHE (__PAGE_KERNEL_EXEC | _PAGE_PCD | _PAGE_PWT) +#define __PAGE_KERNEL_WC (__PAGE_KERNEL | _PAGE_CACHE_WC) #define __PAGE_KERNEL_NOCACHE (__PAGE_KERNEL | _PAGE_PCD | _PAGE_PWT) #define __PAGE_KERNEL_UC_MINUS (__PAGE_KERNEL | _PAGE_PCD) #define __PAGE_KERNEL_VSYSCALL (__PAGE_KERNEL_RX | _PAGE_USER) @@ -101,6 +116,7 @@ extern pteval_t __PAGE_KERNEL, __PAGE_KERNEL_EXEC; #define PAGE_KERNEL_RO MAKE_GLOBAL(__PAGE_KERNEL_RO) #define PAGE_KERNEL_EXEC MAKE_GLOBAL(__PAGE_KERNEL_EXEC) #define PAGE_KERNEL_RX MAKE_GLOBAL(__PAGE_KERNEL_RX) +#define PAGE_KERNEL_WC MAKE_GLOBAL(__PAGE_KERNEL_WC) #define PAGE_KERNEL_NOCACHE MAKE_GLOBAL(__PAGE_KERNEL_NOCACHE) #define PAGE_KERNEL_UC_MINUS MAKE_GLOBAL(__PAGE_KERNEL_UC_MINUS) #define PAGE_KERNEL_EXEC_NOCACHE MAKE_GLOBAL(__PAGE_KERNEL_EXEC_NOCACHE) @@ -134,7 +150,7 @@ extern pteval_t __PAGE_KERNEL, __PAGE_KERNEL_EXEC; * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. */ -extern unsigned long empty_zero_page[PAGE_SIZE/sizeof(unsigned long)]; +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) extern spinlock_t pgd_lock; @@ -144,30 +160,111 @@ extern struct list_head pgd_list; * The following only work if pte_present() is true. * Undefined behaviour if not.. */ -static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } -static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } -static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_RW; } -static inline int pte_file(pte_t pte) { return pte_val(pte) & _PAGE_FILE; } -static inline int pte_huge(pte_t pte) { return pte_val(pte) & _PAGE_PSE; } -static inline int pte_global(pte_t pte) { return pte_val(pte) & _PAGE_GLOBAL; } -static inline int pte_exec(pte_t pte) { return !(pte_val(pte) & _PAGE_NX); } - -static inline int pmd_large(pmd_t pte) { - return (pmd_val(pte) & (_PAGE_PSE|_PAGE_PRESENT)) == - (_PAGE_PSE|_PAGE_PRESENT); +static inline int pte_dirty(pte_t pte) +{ + return pte_val(pte) & _PAGE_DIRTY; +} + +static inline int pte_young(pte_t pte) +{ + return pte_val(pte) & _PAGE_ACCESSED; } -static inline pte_t pte_mkclean(pte_t pte) { return __pte(pte_val(pte) & ~(pteval_t)_PAGE_DIRTY); } -static inline pte_t pte_mkold(pte_t pte) { return __pte(pte_val(pte) & ~(pteval_t)_PAGE_ACCESSED); } -static inline pte_t pte_wrprotect(pte_t pte) { return __pte(pte_val(pte) & ~(pteval_t)_PAGE_RW); } -static inline pte_t pte_mkexec(pte_t pte) { return __pte(pte_val(pte) & ~(pteval_t)_PAGE_NX); } -static inline pte_t pte_mkdirty(pte_t pte) { return __pte(pte_val(pte) | _PAGE_DIRTY); } -static inline pte_t pte_mkyoung(pte_t pte) { return __pte(pte_val(pte) | _PAGE_ACCESSED); } -static inline pte_t pte_mkwrite(pte_t pte) { return __pte(pte_val(pte) | _PAGE_RW); } -static inline pte_t pte_mkhuge(pte_t pte) { return __pte(pte_val(pte) | _PAGE_PSE); } -static inline pte_t pte_clrhuge(pte_t pte) { return __pte(pte_val(pte) & ~(pteval_t)_PAGE_PSE); } -static inline pte_t pte_mkglobal(pte_t pte) { return __pte(pte_val(pte) | _PAGE_GLOBAL); } -static inline pte_t pte_clrglobal(pte_t pte) { return __pte(pte_val(pte) & ~(pteval_t)_PAGE_GLOBAL); } +static inline int pte_write(pte_t pte) +{ + return pte_val(pte) & _PAGE_RW; +} + +static inline int pte_file(pte_t pte) +{ + return pte_val(pte) & _PAGE_FILE; +} + +static inline int pte_huge(pte_t pte) +{ + return pte_val(pte) & _PAGE_PSE; +} + +static inline int pte_global(pte_t pte) +{ + return pte_val(pte) & _PAGE_GLOBAL; +} + +static inline int pte_exec(pte_t pte) +{ + return !(pte_val(pte) & _PAGE_NX); +} + +static inline int pte_special(pte_t pte) +{ + return 0; +} + +static inline int pmd_large(pmd_t pte) +{ + return (pmd_val(pte) & (_PAGE_PSE | _PAGE_PRESENT)) == + (_PAGE_PSE | _PAGE_PRESENT); +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + return __pte(pte_val(pte) & ~(pteval_t)_PAGE_DIRTY); +} + +static inline pte_t pte_mkold(pte_t pte) +{ + return __pte(pte_val(pte) & ~(pteval_t)_PAGE_ACCESSED); +} + +static inline pte_t pte_wrprotect(pte_t pte) +{ + return __pte(pte_val(pte) & ~(pteval_t)_PAGE_RW); +} + +static inline pte_t pte_mkexec(pte_t pte) +{ + return __pte(pte_val(pte) & ~(pteval_t)_PAGE_NX); +} + +static inline pte_t pte_mkdirty(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_DIRTY); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_ACCESSED); +} + +static inline pte_t pte_mkwrite(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_RW); +} + +static inline pte_t pte_mkhuge(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_PSE); +} + +static inline pte_t pte_clrhuge(pte_t pte) +{ + return __pte(pte_val(pte) & ~(pteval_t)_PAGE_PSE); +} + +static inline pte_t pte_mkglobal(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_GLOBAL); +} + +static inline pte_t pte_clrglobal(pte_t pte) +{ + return __pte(pte_val(pte) & ~(pteval_t)_PAGE_GLOBAL); +} + +static inline pte_t pte_mkspecial(pte_t pte) +{ + return pte; +} extern pteval_t __supported_pte_mask; @@ -201,6 +298,15 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #define canon_pgprot(p) __pgprot(pgprot_val(p) & __supported_pte_mask) +#ifndef __ASSEMBLY__ +#define __HAVE_PHYS_MEM_ACCESS_PROT +struct file; +pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, + unsigned long size, pgprot_t vma_prot); +int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, + unsigned long size, pgprot_t *vma_prot); +#endif + #ifdef CONFIG_PARAVIRT #include <asm/paravirt.h> #else /* !CONFIG_PARAVIRT */ @@ -242,6 +348,9 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) # include "pgtable_64.h" #endif +#define KERNEL_PGD_BOUNDARY pgd_index(PAGE_OFFSET) +#define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY) + #ifndef __ASSEMBLY__ enum { @@ -301,40 +410,21 @@ static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr, * bit at the same time. */ #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS -#define ptep_set_access_flags(vma, address, ptep, entry, dirty) \ -({ \ - int __changed = !pte_same(*(ptep), entry); \ - if (__changed && dirty) { \ - *ptep = entry; \ - pte_update_defer((vma)->vm_mm, (address), (ptep)); \ - flush_tlb_page(vma, address); \ - } \ - __changed; \ -}) +extern int ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, + pte_t entry, int dirty); #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG -#define ptep_test_and_clear_young(vma, addr, ptep) ({ \ - int __ret = 0; \ - if (pte_young(*(ptep))) \ - __ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, \ - &(ptep)->pte); \ - if (__ret) \ - pte_update((vma)->vm_mm, addr, ptep); \ - __ret; \ -}) +extern int ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -#define ptep_clear_flush_young(vma, address, ptep) \ -({ \ - int __young; \ - __young = ptep_test_and_clear_young((vma), (address), (ptep)); \ - if (__young) \ - flush_tlb_page(vma, address); \ - __young; \ -}) +extern int ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep); #define __HAVE_ARCH_PTEP_GET_AND_CLEAR -static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) { pte_t pte = native_ptep_get_and_clear(ptep); pte_update(mm, addr, ptep); @@ -342,7 +432,9 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, } #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL -static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long addr, pte_t *ptep, int full) +static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + int full) { pte_t pte; if (full) { @@ -358,12 +450,29 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long } #define __HAVE_ARCH_PTEP_SET_WRPROTECT -static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +static inline void ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) { clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte); pte_update(mm, addr, ptep); } +/* + * clone_pgd_range(pgd_t *dst, pgd_t *src, int count); + * + * dst - pointer to pgd range anwhere on a pgd page + * src - "" + * count - the number of pgds to copy. + * + * dst and src can be on the same page, but the range must not overlap, + * and must not cross a page boundary. + */ +static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) +{ + memcpy(dst, src, count * sizeof(pgd_t)); +} + + #include <asm-generic/pgtable.h> #endif /* __ASSEMBLY__ */ diff --git a/include/asm-x86/pgtable_32.h b/include/asm-x86/pgtable_32.h index 4e6a0fca0b4..577ab79c4c2 100644 --- a/include/asm-x86/pgtable_32.h +++ b/include/asm-x86/pgtable_32.h @@ -40,16 +40,13 @@ void paging_init(void); #ifdef CONFIG_X86_PAE # include <asm/pgtable-3level-defs.h> # define PMD_SIZE (1UL << PMD_SHIFT) -# define PMD_MASK (~(PMD_SIZE-1)) +# define PMD_MASK (~(PMD_SIZE - 1)) #else # include <asm/pgtable-2level-defs.h> #endif #define PGDIR_SIZE (1UL << PGDIR_SHIFT) -#define PGDIR_MASK (~(PGDIR_SIZE-1)) - -#define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT) -#define KERNEL_PGD_PTRS (PTRS_PER_PGD-USER_PGD_PTRS) +#define PGDIR_MASK (~(PGDIR_SIZE - 1)) /* Just any arbitrary offset to the start of the vmalloc VM area: the * current 8MB value just means that there will be a 8MB "hole" after the @@ -58,21 +55,22 @@ void paging_init(void); * The vmalloc() routines leaves a hole of 4kB between each vmalloced * area for the same reason. ;) */ -#define VMALLOC_OFFSET (8*1024*1024) -#define VMALLOC_START (((unsigned long) high_memory + \ - 2*VMALLOC_OFFSET-1) & ~(VMALLOC_OFFSET-1)) +#define VMALLOC_OFFSET (8 * 1024 * 1024) +#define VMALLOC_START (((unsigned long)high_memory + 2 * VMALLOC_OFFSET - 1) \ + & ~(VMALLOC_OFFSET - 1)) #ifdef CONFIG_X86_PAE #define LAST_PKMAP 512 #else #define LAST_PKMAP 1024 #endif -#define PKMAP_BASE ((FIXADDR_BOOT_START - PAGE_SIZE*(LAST_PKMAP + 1)) & PMD_MASK) +#define PKMAP_BASE ((FIXADDR_BOOT_START - PAGE_SIZE * (LAST_PKMAP + 1)) \ + & PMD_MASK) #ifdef CONFIG_HIGHMEM -# define VMALLOC_END (PKMAP_BASE-2*PAGE_SIZE) +# define VMALLOC_END (PKMAP_BASE - 2 * PAGE_SIZE) #else -# define VMALLOC_END (FIXADDR_START-2*PAGE_SIZE) +# define VMALLOC_END (FIXADDR_START - 2 * PAGE_SIZE) #endif /* @@ -88,10 +86,16 @@ extern unsigned long pg0[]; #define pte_present(x) ((x).pte_low & (_PAGE_PRESENT | _PAGE_PROTNONE)) /* To avoid harmful races, pmd_none(x) should check only the lower when PAE */ -#define pmd_none(x) (!(unsigned long)pmd_val(x)) -#define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT) -#define pmd_bad(x) ((pmd_val(x) & (~PAGE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE) +#define pmd_none(x) (!(unsigned long)pmd_val((x))) +#define pmd_present(x) (pmd_val((x)) & _PAGE_PRESENT) + +extern int pmd_bad(pmd_t pmd); +#define pmd_bad_v1(x) \ + (_KERNPG_TABLE != (pmd_val((x)) & ~(PAGE_MASK | _PAGE_USER))) +#define pmd_bad_v2(x) \ + (_KERNPG_TABLE != (pmd_val((x)) & ~(PAGE_MASK | _PAGE_USER | \ + _PAGE_PSE | _PAGE_NX))) #define pages_to_mb(x) ((x) >> (20-PAGE_SHIFT)) @@ -102,32 +106,18 @@ extern unsigned long pg0[]; #endif /* - * clone_pgd_range(pgd_t *dst, pgd_t *src, int count); - * - * dst - pointer to pgd range anwhere on a pgd page - * src - "" - * count - the number of pgds to copy. - * - * dst and src can be on the same page, but the range must not overlap, - * and must not cross a page boundary. - */ -static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) -{ - memcpy(dst, src, count * sizeof(pgd_t)); -} - -/* - * Macro to mark a page protection value as "uncacheable". On processors which do not support - * it, this is a no-op. + * Macro to mark a page protection value as "uncacheable". + * On processors which do not support it, this is a no-op. */ -#define pgprot_noncached(prot) ((boot_cpu_data.x86 > 3) \ - ? (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT)) : (prot)) +#define pgprot_noncached(prot) \ + ((boot_cpu_data.x86 > 3) \ + ? (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT)) \ + : (prot)) /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. */ - #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) /* @@ -136,20 +126,20 @@ static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) * this macro returns the index of the entry in the pgd page which would * control the given virtual address */ -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) -#define pgd_index_k(addr) pgd_index(addr) +#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) +#define pgd_index_k(addr) pgd_index((addr)) /* * pgd_offset() returns a (pgd_t *) * pgd_index() is used get the offset into the pgd page's array of pgd_t's; */ -#define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address)) +#define pgd_offset(mm, address) ((mm)->pgd + pgd_index((address))) /* * a shortcut which implies the use of the kernel's pgd, instead * of a process's */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) +#define pgd_offset_k(address) pgd_offset(&init_mm, (address)) static inline int pud_large(pud_t pud) { return 0; } @@ -159,8 +149,8 @@ static inline int pud_large(pud_t pud) { return 0; } * this macro returns the index of the entry in the pmd page which would * control the given virtual address */ -#define pmd_index(address) \ - (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) +#define pmd_index(address) \ + (((address) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) /* * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE] @@ -168,54 +158,56 @@ static inline int pud_large(pud_t pud) { return 0; } * this macro returns the index of the entry in the pte page which would * control the given virtual address */ -#define pte_index(address) \ - (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, address) \ - ((pte_t *) pmd_page_vaddr(*(dir)) + pte_index(address)) +#define pte_index(address) \ + (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +#define pte_offset_kernel(dir, address) \ + ((pte_t *)pmd_page_vaddr(*(dir)) + pte_index((address))) -#define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)) +#define pmd_page(pmd) (pfn_to_page(pmd_val((pmd)) >> PAGE_SHIFT)) -#define pmd_page_vaddr(pmd) \ - ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK)) +#define pmd_page_vaddr(pmd) \ + ((unsigned long)__va(pmd_val((pmd)) & PAGE_MASK)) #if defined(CONFIG_HIGHPTE) -#define pte_offset_map(dir, address) \ - ((pte_t *)kmap_atomic_pte(pmd_page(*(dir)),KM_PTE0) + pte_index(address)) -#define pte_offset_map_nested(dir, address) \ - ((pte_t *)kmap_atomic_pte(pmd_page(*(dir)),KM_PTE1) + pte_index(address)) -#define pte_unmap(pte) kunmap_atomic(pte, KM_PTE0) -#define pte_unmap_nested(pte) kunmap_atomic(pte, KM_PTE1) +#define pte_offset_map(dir, address) \ + ((pte_t *)kmap_atomic_pte(pmd_page(*(dir)), KM_PTE0) + \ + pte_index((address))) +#define pte_offset_map_nested(dir, address) \ + ((pte_t *)kmap_atomic_pte(pmd_page(*(dir)), KM_PTE1) + \ + pte_index((address))) +#define pte_unmap(pte) kunmap_atomic((pte), KM_PTE0) +#define pte_unmap_nested(pte) kunmap_atomic((pte), KM_PTE1) #else -#define pte_offset_map(dir, address) \ - ((pte_t *)page_address(pmd_page(*(dir))) + pte_index(address)) -#define pte_offset_map_nested(dir, address) pte_offset_map(dir, address) +#define pte_offset_map(dir, address) \ + ((pte_t *)page_address(pmd_page(*(dir))) + pte_index((address))) +#define pte_offset_map_nested(dir, address) pte_offset_map((dir), (address)) #define pte_unmap(pte) do { } while (0) #define pte_unmap_nested(pte) do { } while (0) #endif /* Clear a kernel PTE and flush it from the TLB */ -#define kpte_clear_flush(ptep, vaddr) \ -do { \ - pte_clear(&init_mm, vaddr, ptep); \ - __flush_tlb_one(vaddr); \ +#define kpte_clear_flush(ptep, vaddr) \ +do { \ + pte_clear(&init_mm, (vaddr), (ptep)); \ + __flush_tlb_one((vaddr)); \ } while (0) /* * The i386 doesn't have any external MMU info: the kernel page * tables contain all the necessary information. */ -#define update_mmu_cache(vma,address,pte) do { } while (0) +#define update_mmu_cache(vma, address, pte) do { } while (0) -void native_pagetable_setup_start(pgd_t *base); -void native_pagetable_setup_done(pgd_t *base); +extern void native_pagetable_setup_start(pgd_t *base); +extern void native_pagetable_setup_done(pgd_t *base); #ifndef CONFIG_PARAVIRT -static inline void paravirt_pagetable_setup_start(pgd_t *base) +static inline void __init paravirt_pagetable_setup_start(pgd_t *base) { native_pagetable_setup_start(base); } -static inline void paravirt_pagetable_setup_done(pgd_t *base) +static inline void __init paravirt_pagetable_setup_done(pgd_t *base) { native_pagetable_setup_done(base); } @@ -233,7 +225,7 @@ static inline void paravirt_pagetable_setup_done(pgd_t *base) #define kern_addr_valid(kaddr) (0) #endif -#define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \ - remap_pfn_range(vma, vaddr, pfn, size, prot) +#define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \ + remap_pfn_range(vma, vaddr, pfn, size, prot) #endif /* _I386_PGTABLE_H */ diff --git a/include/asm-x86/pgtable_64.h b/include/asm-x86/pgtable_64.h index 0a0b77bc736..a3bbf8766c1 100644 --- a/include/asm-x86/pgtable_64.h +++ b/include/asm-x86/pgtable_64.h @@ -24,7 +24,7 @@ extern void paging_init(void); #endif /* !__ASSEMBLY__ */ -#define SHARED_KERNEL_PMD 1 +#define SHARED_KERNEL_PMD 0 /* * PGDIR_SHIFT determines what a top-level page table entry can map @@ -52,14 +52,18 @@ extern void paging_init(void); #ifndef __ASSEMBLY__ -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %p(%016lx).\n", __FILE__, __LINE__, &(e), pte_val(e)) -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %p(%016lx).\n", __FILE__, __LINE__, &(e), pmd_val(e)) -#define pud_ERROR(e) \ - printk("%s:%d: bad pud %p(%016lx).\n", __FILE__, __LINE__, &(e), pud_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %p(%016lx).\n", __FILE__, __LINE__, &(e), pgd_val(e)) +#define pte_ERROR(e) \ + printk("%s:%d: bad pte %p(%016lx).\n", \ + __FILE__, __LINE__, &(e), pte_val(e)) +#define pmd_ERROR(e) \ + printk("%s:%d: bad pmd %p(%016lx).\n", \ + __FILE__, __LINE__, &(e), pmd_val(e)) +#define pud_ERROR(e) \ + printk("%s:%d: bad pud %p(%016lx).\n", \ + __FILE__, __LINE__, &(e), pud_val(e)) +#define pgd_ERROR(e) \ + printk("%s:%d: bad pgd %p(%016lx).\n", \ + __FILE__, __LINE__, &(e), pgd_val(e)) #define pgd_none(x) (!pgd_val(x)) #define pud_none(x) (!pud_val(x)) @@ -87,7 +91,8 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp) #ifdef CONFIG_SMP return native_make_pte(xchg(&xp->pte, 0)); #else - /* native_local_ptep_get_and_clear, but duplicated because of cyclic dependency */ + /* native_local_ptep_get_and_clear, + but duplicated because of cyclic dependency */ pte_t ret = *xp; native_pte_clear(NULL, 0, xp); return ret; @@ -119,7 +124,7 @@ static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) *pgdp = pgd; } -static inline void native_pgd_clear(pgd_t * pgd) +static inline void native_pgd_clear(pgd_t *pgd) { native_set_pgd(pgd, native_make_pgd(0)); } @@ -128,19 +133,19 @@ static inline void native_pgd_clear(pgd_t * pgd) #endif /* !__ASSEMBLY__ */ -#define PMD_SIZE (_AC(1,UL) << PMD_SHIFT) -#define PMD_MASK (~(PMD_SIZE-1)) -#define PUD_SIZE (_AC(1,UL) << PUD_SHIFT) -#define PUD_MASK (~(PUD_SIZE-1)) -#define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT) -#define PGDIR_MASK (~(PGDIR_SIZE-1)) +#define PMD_SIZE (_AC(1, UL) << PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE - 1)) +#define PUD_SIZE (_AC(1, UL) << PUD_SHIFT) +#define PUD_MASK (~(PUD_SIZE - 1)) +#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE - 1)) -#define MAXMEM _AC(0x3fffffffffff, UL) +#define MAXMEM _AC(0x00003fffffffffff, UL) #define VMALLOC_START _AC(0xffffc20000000000, UL) #define VMALLOC_END _AC(0xffffe1ffffffffff, UL) #define VMEMMAP_START _AC(0xffffe20000000000, UL) -#define MODULES_VADDR _AC(0xffffffff88000000, UL) +#define MODULES_VADDR _AC(0xffffffffa0000000, UL) #define MODULES_END _AC(0xfffffffffff00000, UL) #define MODULES_LEN (MODULES_END - MODULES_VADDR) @@ -153,26 +158,28 @@ static inline unsigned long pgd_bad(pgd_t pgd) static inline unsigned long pud_bad(pud_t pud) { - return pud_val(pud) & ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER); + return pud_val(pud) & + ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER | _PAGE_PSE | _PAGE_NX); } static inline unsigned long pmd_bad(pmd_t pmd) { - return pmd_val(pmd) & ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER); + return pmd_val(pmd) & + ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER | _PAGE_PSE | _PAGE_NX); } -#define pte_none(x) (!pte_val(x)) -#define pte_present(x) (pte_val(x) & (_PAGE_PRESENT | _PAGE_PROTNONE)) +#define pte_none(x) (!pte_val((x))) +#define pte_present(x) (pte_val((x)) & (_PAGE_PRESENT | _PAGE_PROTNONE)) -#define pages_to_mb(x) ((x) >> (20-PAGE_SHIFT)) /* FIXME: is this right? */ -#define pte_page(x) pfn_to_page(pte_pfn(x)) -#define pte_pfn(x) ((pte_val(x) & __PHYSICAL_MASK) >> PAGE_SHIFT) +#define pages_to_mb(x) ((x) >> (20 - PAGE_SHIFT)) /* FIXME: is this right? */ +#define pte_page(x) pfn_to_page(pte_pfn((x))) +#define pte_pfn(x) ((pte_val((x)) & __PHYSICAL_MASK) >> PAGE_SHIFT) /* * Macro to mark a page protection value as "uncacheable". */ -#define pgprot_noncached(prot) (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT)) - +#define pgprot_noncached(prot) \ + (__pgprot(pgprot_val((prot)) | _PAGE_PCD | _PAGE_PWT)) /* * Conversion functions: convert a page and protection to a page entry, @@ -182,75 +189,81 @@ static inline unsigned long pmd_bad(pmd_t pmd) /* * Level 4 access. */ -#define pgd_page_vaddr(pgd) ((unsigned long) __va((unsigned long)pgd_val(pgd) & PTE_MASK)) -#define pgd_page(pgd) (pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT)) -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) -#define pgd_offset(mm, addr) ((mm)->pgd + pgd_index(addr)) -#define pgd_offset_k(address) (init_level4_pgt + pgd_index(address)) +#define pgd_page_vaddr(pgd) \ + ((unsigned long)__va((unsigned long)pgd_val((pgd)) & PTE_MASK)) +#define pgd_page(pgd) (pfn_to_page(pgd_val((pgd)) >> PAGE_SHIFT)) +#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) +#define pgd_offset(mm, address) ((mm)->pgd + pgd_index((address))) +#define pgd_offset_k(address) (init_level4_pgt + pgd_index((address))) #define pgd_present(pgd) (pgd_val(pgd) & _PAGE_PRESENT) static inline int pgd_large(pgd_t pgd) { return 0; } #define mk_kernel_pgd(address) ((pgd_t){ (address) | _KERNPG_TABLE }) /* PUD - Level3 access */ /* to find an entry in a page-table-directory. */ -#define pud_page_vaddr(pud) ((unsigned long) __va(pud_val(pud) & PHYSICAL_PAGE_MASK)) -#define pud_page(pud) (pfn_to_page(pud_val(pud) >> PAGE_SHIFT)) -#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) -#define pud_offset(pgd, address) ((pud_t *) pgd_page_vaddr(*(pgd)) + pud_index(address)) -#define pud_present(pud) (pud_val(pud) & _PAGE_PRESENT) +#define pud_page_vaddr(pud) \ + ((unsigned long)__va(pud_val((pud)) & PHYSICAL_PAGE_MASK)) +#define pud_page(pud) (pfn_to_page(pud_val((pud)) >> PAGE_SHIFT)) +#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) +#define pud_offset(pgd, address) \ + ((pud_t *)pgd_page_vaddr(*(pgd)) + pud_index((address))) +#define pud_present(pud) (pud_val((pud)) & _PAGE_PRESENT) static inline int pud_large(pud_t pte) { - return (pud_val(pte) & (_PAGE_PSE|_PAGE_PRESENT)) == - (_PAGE_PSE|_PAGE_PRESENT); + return (pud_val(pte) & (_PAGE_PSE | _PAGE_PRESENT)) == + (_PAGE_PSE | _PAGE_PRESENT); } /* PMD - Level 2 access */ -#define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_val(pmd) & PTE_MASK)) -#define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)) - -#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) -#define pmd_offset(dir, address) ((pmd_t *) pud_page_vaddr(*(dir)) + \ - pmd_index(address)) -#define pmd_none(x) (!pmd_val(x)) -#define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT) -#define pfn_pmd(nr,prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val(prot))) -#define pmd_pfn(x) ((pmd_val(x) & __PHYSICAL_MASK) >> PAGE_SHIFT) - -#define pte_to_pgoff(pte) ((pte_val(pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT) -#define pgoff_to_pte(off) ((pte_t) { .pte = ((off) << PAGE_SHIFT) | _PAGE_FILE }) +#define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_val((pmd)) & PTE_MASK)) +#define pmd_page(pmd) (pfn_to_page(pmd_val((pmd)) >> PAGE_SHIFT)) + +#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) +#define pmd_offset(dir, address) ((pmd_t *)pud_page_vaddr(*(dir)) + \ + pmd_index(address)) +#define pmd_none(x) (!pmd_val((x))) +#define pmd_present(x) (pmd_val((x)) & _PAGE_PRESENT) +#define pfn_pmd(nr, prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val((prot)))) +#define pmd_pfn(x) ((pmd_val((x)) & __PHYSICAL_MASK) >> PAGE_SHIFT) + +#define pte_to_pgoff(pte) ((pte_val((pte)) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT) +#define pgoff_to_pte(off) ((pte_t) { .pte = ((off) << PAGE_SHIFT) | \ + _PAGE_FILE }) #define PTE_FILE_MAX_BITS __PHYSICAL_MASK_SHIFT /* PTE - Level 1 access. */ /* page, protection -> pte */ -#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) - -#define pte_index(address) \ - (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +#define mk_pte(page, pgprot) pfn_pte(page_to_pfn((page)), (pgprot)) + +#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) #define pte_offset_kernel(dir, address) ((pte_t *) pmd_page_vaddr(*(dir)) + \ - pte_index(address)) + pte_index((address))) /* x86-64 always has all page tables mapped. */ -#define pte_offset_map(dir,address) pte_offset_kernel(dir,address) -#define pte_offset_map_nested(dir,address) pte_offset_kernel(dir,address) +#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) +#define pte_offset_map_nested(dir, address) pte_offset_kernel((dir), (address)) #define pte_unmap(pte) /* NOP */ -#define pte_unmap_nested(pte) /* NOP */ +#define pte_unmap_nested(pte) /* NOP */ + +#define update_mmu_cache(vma, address, pte) do { } while (0) -#define update_mmu_cache(vma,address,pte) do { } while (0) +extern int direct_gbpages; /* Encode and de-code a swap entry */ #define __swp_type(x) (((x).val >> 1) & 0x3f) #define __swp_offset(x) ((x).val >> 8) -#define __swp_entry(type, offset) ((swp_entry_t) { ((type) << 1) | ((offset) << 8) }) -#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) +#define __swp_entry(type, offset) ((swp_entry_t) { ((type) << 1) | \ + ((offset) << 8) }) +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) }) #define __swp_entry_to_pte(x) ((pte_t) { .pte = (x).val }) -extern int kern_addr_valid(unsigned long addr); +extern int kern_addr_valid(unsigned long addr); extern void cleanup_highmap(void); -#define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \ - remap_pfn_range(vma, vaddr, pfn, size, prot) +#define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \ + remap_pfn_range(vma, vaddr, pfn, size, prot) #define HAVE_ARCH_UNMAPPED_AREA #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN @@ -263,8 +276,10 @@ extern void cleanup_highmap(void); /* fs/proc/kcore.c */ #define kc_vaddr_to_offset(v) ((v) & __VIRTUAL_MASK) -#define kc_offset_to_vaddr(o) \ - (((o) & (1UL << (__VIRTUAL_MASK_SHIFT-1))) ? ((o) | (~__VIRTUAL_MASK)) : (o)) +#define kc_offset_to_vaddr(o) \ + (((o) & (1UL << (__VIRTUAL_MASK_SHIFT - 1))) \ + ? ((o) | ~__VIRTUAL_MASK) \ + : (o)) #define __HAVE_ARCH_PTE_SAME #endif /* !__ASSEMBLY__ */ diff --git a/include/asm-x86/posix_types_32.h b/include/asm-x86/posix_types_32.h index 015e539cdef..b031efda37e 100644 --- a/include/asm-x86/posix_types_32.h +++ b/include/asm-x86/posix_types_32.h @@ -45,32 +45,39 @@ typedef struct { #if defined(__KERNEL__) #undef __FD_SET -#define __FD_SET(fd,fdsetp) \ - __asm__ __volatile__("btsl %1,%0": \ - "+m" (*(__kernel_fd_set *) (fdsetp)):"r" ((int) (fd))) +#define __FD_SET(fd,fdsetp) \ + asm volatile("btsl %1,%0": \ + "+m" (*(__kernel_fd_set *)(fdsetp)) \ + : "r" ((int)(fd))) #undef __FD_CLR -#define __FD_CLR(fd,fdsetp) \ - __asm__ __volatile__("btrl %1,%0": \ - "+m" (*(__kernel_fd_set *) (fdsetp)):"r" ((int) (fd))) +#define __FD_CLR(fd,fdsetp) \ + asm volatile("btrl %1,%0": \ + "+m" (*(__kernel_fd_set *)(fdsetp)) \ + : "r" ((int) (fd))) #undef __FD_ISSET -#define __FD_ISSET(fd,fdsetp) (__extension__ ({ \ - unsigned char __result; \ - __asm__ __volatile__("btl %1,%2 ; setb %0" \ - :"=q" (__result) :"r" ((int) (fd)), \ - "m" (*(__kernel_fd_set *) (fdsetp))); \ - __result; })) +#define __FD_ISSET(fd,fdsetp) \ + (__extension__ \ + ({ \ + unsigned char __result; \ + asm volatile("btl %1,%2 ; setb %0" \ + : "=q" (__result) \ + : "r" ((int)(fd)), \ + "m" (*(__kernel_fd_set *)(fdsetp))); \ + __result; \ +})) #undef __FD_ZERO -#define __FD_ZERO(fdsetp) \ -do { \ - int __d0, __d1; \ - __asm__ __volatile__("cld ; rep ; stosl" \ - :"=m" (*(__kernel_fd_set *) (fdsetp)), \ - "=&c" (__d0), "=&D" (__d1) \ - :"a" (0), "1" (__FDSET_LONGS), \ - "2" ((__kernel_fd_set *) (fdsetp)) : "memory"); \ +#define __FD_ZERO(fdsetp) \ +do { \ + int __d0, __d1; \ + asm volatile("cld ; rep ; stosl" \ + : "=m" (*(__kernel_fd_set *)(fdsetp)), \ + "=&c" (__d0), "=&D" (__d1) \ + : "a" (0), "1" (__FDSET_LONGS), \ + "2" ((__kernel_fd_set *)(fdsetp)) \ + : "memory"); \ } while (0) #endif /* defined(__KERNEL__) */ diff --git a/include/asm-x86/posix_types_64.h b/include/asm-x86/posix_types_64.h index 9926aa43775..d6624c95854 100644 --- a/include/asm-x86/posix_types_64.h +++ b/include/asm-x86/posix_types_64.h @@ -46,7 +46,7 @@ typedef unsigned long __kernel_old_dev_t; #ifdef __KERNEL__ #undef __FD_SET -static __inline__ void __FD_SET(unsigned long fd, __kernel_fd_set *fdsetp) +static inline void __FD_SET(unsigned long fd, __kernel_fd_set *fdsetp) { unsigned long _tmp = fd / __NFDBITS; unsigned long _rem = fd % __NFDBITS; @@ -54,7 +54,7 @@ static __inline__ void __FD_SET(unsigned long fd, __kernel_fd_set *fdsetp) } #undef __FD_CLR -static __inline__ void __FD_CLR(unsigned long fd, __kernel_fd_set *fdsetp) +static inline void __FD_CLR(unsigned long fd, __kernel_fd_set *fdsetp) { unsigned long _tmp = fd / __NFDBITS; unsigned long _rem = fd % __NFDBITS; @@ -62,7 +62,7 @@ static __inline__ void __FD_CLR(unsigned long fd, __kernel_fd_set *fdsetp) } #undef __FD_ISSET -static __inline__ int __FD_ISSET(unsigned long fd, __const__ __kernel_fd_set *p) +static inline int __FD_ISSET(unsigned long fd, __const__ __kernel_fd_set *p) { unsigned long _tmp = fd / __NFDBITS; unsigned long _rem = fd % __NFDBITS; @@ -74,36 +74,36 @@ static __inline__ int __FD_ISSET(unsigned long fd, __const__ __kernel_fd_set *p) * for 256 and 1024-bit fd_sets respectively) */ #undef __FD_ZERO -static __inline__ void __FD_ZERO(__kernel_fd_set *p) +static inline void __FD_ZERO(__kernel_fd_set *p) { unsigned long *tmp = p->fds_bits; int i; if (__builtin_constant_p(__FDSET_LONGS)) { switch (__FDSET_LONGS) { - case 32: - tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; - tmp[ 4] = 0; tmp[ 5] = 0; tmp[ 6] = 0; tmp[ 7] = 0; - tmp[ 8] = 0; tmp[ 9] = 0; tmp[10] = 0; tmp[11] = 0; - tmp[12] = 0; tmp[13] = 0; tmp[14] = 0; tmp[15] = 0; - tmp[16] = 0; tmp[17] = 0; tmp[18] = 0; tmp[19] = 0; - tmp[20] = 0; tmp[21] = 0; tmp[22] = 0; tmp[23] = 0; - tmp[24] = 0; tmp[25] = 0; tmp[26] = 0; tmp[27] = 0; - tmp[28] = 0; tmp[29] = 0; tmp[30] = 0; tmp[31] = 0; - return; - case 16: - tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; - tmp[ 4] = 0; tmp[ 5] = 0; tmp[ 6] = 0; tmp[ 7] = 0; - tmp[ 8] = 0; tmp[ 9] = 0; tmp[10] = 0; tmp[11] = 0; - tmp[12] = 0; tmp[13] = 0; tmp[14] = 0; tmp[15] = 0; - return; - case 8: - tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; - tmp[ 4] = 0; tmp[ 5] = 0; tmp[ 6] = 0; tmp[ 7] = 0; - return; - case 4: - tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; - return; + case 32: + tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; + tmp[ 4] = 0; tmp[ 5] = 0; tmp[ 6] = 0; tmp[ 7] = 0; + tmp[ 8] = 0; tmp[ 9] = 0; tmp[10] = 0; tmp[11] = 0; + tmp[12] = 0; tmp[13] = 0; tmp[14] = 0; tmp[15] = 0; + tmp[16] = 0; tmp[17] = 0; tmp[18] = 0; tmp[19] = 0; + tmp[20] = 0; tmp[21] = 0; tmp[22] = 0; tmp[23] = 0; + tmp[24] = 0; tmp[25] = 0; tmp[26] = 0; tmp[27] = 0; + tmp[28] = 0; tmp[29] = 0; tmp[30] = 0; tmp[31] = 0; + return; + case 16: + tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; + tmp[ 4] = 0; tmp[ 5] = 0; tmp[ 6] = 0; tmp[ 7] = 0; + tmp[ 8] = 0; tmp[ 9] = 0; tmp[10] = 0; tmp[11] = 0; + tmp[12] = 0; tmp[13] = 0; tmp[14] = 0; tmp[15] = 0; + return; + case 8: + tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; + tmp[ 4] = 0; tmp[ 5] = 0; tmp[ 6] = 0; tmp[ 7] = 0; + return; + case 4: + tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; + return; } } i = __FDSET_LONGS; diff --git a/include/asm-x86/processor.h b/include/asm-x86/processor.h index 45a2f0ab33d..559105220a4 100644 --- a/include/asm-x86/processor.h +++ b/include/asm-x86/processor.h @@ -3,10 +3,6 @@ #include <asm/processor-flags.h> -/* migration helpers, for KVM - will be removed in 2.6.25: */ -#include <asm/vm86.h> -#define Xgt_desc_struct desc_ptr - /* Forward declaration, a strange C thing */ struct task_struct; struct mm_struct; @@ -24,6 +20,7 @@ struct mm_struct; #include <asm/msr.h> #include <asm/desc_defs.h> #include <asm/nops.h> + #include <linux/personality.h> #include <linux/cpumask.h> #include <linux/cache.h> @@ -37,16 +34,18 @@ struct mm_struct; static inline void *current_text_addr(void) { void *pc; - asm volatile("mov $1f,%0\n1:":"=r" (pc)); + + asm volatile("mov $1f, %0; 1:":"=r" (pc)); + return pc; } #ifdef CONFIG_X86_VSMP -#define ARCH_MIN_TASKALIGN (1 << INTERNODE_CACHE_SHIFT) -#define ARCH_MIN_MMSTRUCT_ALIGN (1 << INTERNODE_CACHE_SHIFT) +# define ARCH_MIN_TASKALIGN (1 << INTERNODE_CACHE_SHIFT) +# define ARCH_MIN_MMSTRUCT_ALIGN (1 << INTERNODE_CACHE_SHIFT) #else -#define ARCH_MIN_TASKALIGN 16 -#define ARCH_MIN_MMSTRUCT_ALIGN 0 +# define ARCH_MIN_TASKALIGN 16 +# define ARCH_MIN_MMSTRUCT_ALIGN 0 #endif /* @@ -56,69 +55,81 @@ static inline void *current_text_addr(void) */ struct cpuinfo_x86 { - __u8 x86; /* CPU family */ - __u8 x86_vendor; /* CPU vendor */ - __u8 x86_model; - __u8 x86_mask; + __u8 x86; /* CPU family */ + __u8 x86_vendor; /* CPU vendor */ + __u8 x86_model; + __u8 x86_mask; #ifdef CONFIG_X86_32 - char wp_works_ok; /* It doesn't on 386's */ - char hlt_works_ok; /* Problems on some 486Dx4's and old 386's */ - char hard_math; - char rfu; - char fdiv_bug; - char f00f_bug; - char coma_bug; - char pad0; + char wp_works_ok; /* It doesn't on 386's */ + + /* Problems on some 486Dx4's and old 386's: */ + char hlt_works_ok; + char hard_math; + char rfu; + char fdiv_bug; + char f00f_bug; + char coma_bug; + char pad0; #else - /* number of 4K pages in DTLB/ITLB combined(in pages)*/ - int x86_tlbsize; - __u8 x86_virt_bits, x86_phys_bits; - /* cpuid returned core id bits */ - __u8 x86_coreid_bits; - /* Max extended CPUID function supported */ - __u32 extended_cpuid_level; + /* Number of 4K pages in DTLB/ITLB combined(in pages): */ + int x86_tlbsize; + __u8 x86_virt_bits; + __u8 x86_phys_bits; + /* CPUID returned core id bits: */ + __u8 x86_coreid_bits; + /* Max extended CPUID function supported: */ + __u32 extended_cpuid_level; #endif - int cpuid_level; /* Maximum supported CPUID level, -1=no CPUID */ - __u32 x86_capability[NCAPINTS]; - char x86_vendor_id[16]; - char x86_model_id[64]; - int x86_cache_size; /* in KB - valid for CPUS which support this - call */ - int x86_cache_alignment; /* In bytes */ - int x86_power; - unsigned long loops_per_jiffy; + /* Maximum supported CPUID level, -1=no CPUID: */ + int cpuid_level; + __u32 x86_capability[NCAPINTS]; + char x86_vendor_id[16]; + char x86_model_id[64]; + /* in KB - valid for CPUS which support this call: */ + int x86_cache_size; + int x86_cache_alignment; /* In bytes */ + int x86_power; + unsigned long loops_per_jiffy; #ifdef CONFIG_SMP - cpumask_t llc_shared_map; /* cpus sharing the last level cache */ + /* cpus sharing the last level cache: */ + cpumask_t llc_shared_map; #endif - u16 x86_max_cores; /* cpuid returned max cores value */ - u16 apicid; - u16 x86_clflush_size; + /* cpuid returned max cores value: */ + u16 x86_max_cores; + u16 apicid; + u16 initial_apicid; + u16 x86_clflush_size; #ifdef CONFIG_SMP - u16 booted_cores; /* number of cores as seen by OS */ - u16 phys_proc_id; /* Physical processor id. */ - u16 cpu_core_id; /* Core id */ - u16 cpu_index; /* index into per_cpu list */ + /* number of cores as seen by the OS: */ + u16 booted_cores; + /* Physical processor id: */ + u16 phys_proc_id; + /* Core id: */ + u16 cpu_core_id; + /* Index into per_cpu list: */ + u16 cpu_index; #endif } __attribute__((__aligned__(SMP_CACHE_BYTES))); -#define X86_VENDOR_INTEL 0 -#define X86_VENDOR_CYRIX 1 -#define X86_VENDOR_AMD 2 -#define X86_VENDOR_UMC 3 -#define X86_VENDOR_NEXGEN 4 -#define X86_VENDOR_CENTAUR 5 -#define X86_VENDOR_TRANSMETA 7 -#define X86_VENDOR_NSC 8 -#define X86_VENDOR_NUM 9 -#define X86_VENDOR_UNKNOWN 0xff +#define X86_VENDOR_INTEL 0 +#define X86_VENDOR_CYRIX 1 +#define X86_VENDOR_AMD 2 +#define X86_VENDOR_UMC 3 +#define X86_VENDOR_CENTAUR 5 +#define X86_VENDOR_TRANSMETA 7 +#define X86_VENDOR_NSC 8 +#define X86_VENDOR_NUM 9 + +#define X86_VENDOR_UNKNOWN 0xff /* * capabilities of CPUs */ -extern struct cpuinfo_x86 boot_cpu_data; -extern struct cpuinfo_x86 new_cpu_data; -extern struct tss_struct doublefault_tss; -extern __u32 cleared_cpu_caps[NCAPINTS]; +extern struct cpuinfo_x86 boot_cpu_data; +extern struct cpuinfo_x86 new_cpu_data; + +extern struct tss_struct doublefault_tss; +extern __u32 cleared_cpu_caps[NCAPINTS]; #ifdef CONFIG_SMP DECLARE_PER_CPU(struct cpuinfo_x86, cpu_info); @@ -129,7 +140,18 @@ DECLARE_PER_CPU(struct cpuinfo_x86, cpu_info); #define current_cpu_data boot_cpu_data #endif -void cpu_detect(struct cpuinfo_x86 *c); +static inline int hlt_works(int cpu) +{ +#ifdef CONFIG_X86_32 + return cpu_data(cpu).hlt_works_ok; +#else + return 1; +#endif +} + +#define cache_line_size() (boot_cpu_data.x86_cache_alignment) + +extern void cpu_detect(struct cpuinfo_x86 *c); extern void identify_cpu(struct cpuinfo_x86 *); extern void identify_boot_cpu(void); @@ -146,15 +168,15 @@ static inline void detect_ht(struct cpuinfo_x86 *c) {} #endif static inline void native_cpuid(unsigned int *eax, unsigned int *ebx, - unsigned int *ecx, unsigned int *edx) + unsigned int *ecx, unsigned int *edx) { /* ecx is often an input as well as an output. */ - __asm__("cpuid" - : "=a" (*eax), - "=b" (*ebx), - "=c" (*ecx), - "=d" (*edx) - : "0" (*eax), "2" (*ecx)); + asm("cpuid" + : "=a" (*eax), + "=b" (*ebx), + "=c" (*ecx), + "=d" (*edx) + : "0" (*eax), "2" (*ecx)); } static inline void load_cr3(pgd_t *pgdir) @@ -165,54 +187,67 @@ static inline void load_cr3(pgd_t *pgdir) #ifdef CONFIG_X86_32 /* This is the TSS defined by the hardware. */ struct x86_hw_tss { - unsigned short back_link, __blh; - unsigned long sp0; - unsigned short ss0, __ss0h; - unsigned long sp1; - unsigned short ss1, __ss1h; /* ss1 caches MSR_IA32_SYSENTER_CS */ - unsigned long sp2; - unsigned short ss2, __ss2h; - unsigned long __cr3; - unsigned long ip; - unsigned long flags; - unsigned long ax, cx, dx, bx; - unsigned long sp, bp, si, di; - unsigned short es, __esh; - unsigned short cs, __csh; - unsigned short ss, __ssh; - unsigned short ds, __dsh; - unsigned short fs, __fsh; - unsigned short gs, __gsh; - unsigned short ldt, __ldth; - unsigned short trace, io_bitmap_base; + unsigned short back_link, __blh; + unsigned long sp0; + unsigned short ss0, __ss0h; + unsigned long sp1; + /* ss1 caches MSR_IA32_SYSENTER_CS: */ + unsigned short ss1, __ss1h; + unsigned long sp2; + unsigned short ss2, __ss2h; + unsigned long __cr3; + unsigned long ip; + unsigned long flags; + unsigned long ax; + unsigned long cx; + unsigned long dx; + unsigned long bx; + unsigned long sp; + unsigned long bp; + unsigned long si; + unsigned long di; + unsigned short es, __esh; + unsigned short cs, __csh; + unsigned short ss, __ssh; + unsigned short ds, __dsh; + unsigned short fs, __fsh; + unsigned short gs, __gsh; + unsigned short ldt, __ldth; + unsigned short trace; + unsigned short io_bitmap_base; + } __attribute__((packed)); #else struct x86_hw_tss { - u32 reserved1; - u64 sp0; - u64 sp1; - u64 sp2; - u64 reserved2; - u64 ist[7]; - u32 reserved3; - u32 reserved4; - u16 reserved5; - u16 io_bitmap_base; + u32 reserved1; + u64 sp0; + u64 sp1; + u64 sp2; + u64 reserved2; + u64 ist[7]; + u32 reserved3; + u32 reserved4; + u16 reserved5; + u16 io_bitmap_base; + } __attribute__((packed)) ____cacheline_aligned; #endif /* - * Size of io_bitmap. + * IO-bitmap sizes: */ -#define IO_BITMAP_BITS 65536 -#define IO_BITMAP_BYTES (IO_BITMAP_BITS/8) -#define IO_BITMAP_LONGS (IO_BITMAP_BYTES/sizeof(long)) -#define IO_BITMAP_OFFSET offsetof(struct tss_struct, io_bitmap) -#define INVALID_IO_BITMAP_OFFSET 0x8000 -#define INVALID_IO_BITMAP_OFFSET_LAZY 0x9000 +#define IO_BITMAP_BITS 65536 +#define IO_BITMAP_BYTES (IO_BITMAP_BITS/8) +#define IO_BITMAP_LONGS (IO_BITMAP_BYTES/sizeof(long)) +#define IO_BITMAP_OFFSET offsetof(struct tss_struct, io_bitmap) +#define INVALID_IO_BITMAP_OFFSET 0x8000 +#define INVALID_IO_BITMAP_OFFSET_LAZY 0x9000 struct tss_struct { - struct x86_hw_tss x86_tss; + /* + * The hardware state: + */ + struct x86_hw_tss x86_tss; /* * The extra 1 is there because the CPU will access an @@ -220,135 +255,164 @@ struct tss_struct { * bitmap. The extra byte must be all 1 bits, and must * be within the limit. */ - unsigned long io_bitmap[IO_BITMAP_LONGS + 1]; + unsigned long io_bitmap[IO_BITMAP_LONGS + 1]; /* * Cache the current maximum and the last task that used the bitmap: */ - unsigned long io_bitmap_max; - struct thread_struct *io_bitmap_owner; + unsigned long io_bitmap_max; + struct thread_struct *io_bitmap_owner; + /* - * pads the TSS to be cacheline-aligned (size is 0x100) + * Pad the TSS to be cacheline-aligned (size is 0x100): */ - unsigned long __cacheline_filler[35]; + unsigned long __cacheline_filler[35]; /* - * .. and then another 0x100 bytes for emergency kernel stack + * .. and then another 0x100 bytes for the emergency kernel stack: */ - unsigned long stack[64]; + unsigned long stack[64]; + } __attribute__((packed)); DECLARE_PER_CPU(struct tss_struct, init_tss); -/* Save the original ist values for checking stack pointers during debugging */ +/* + * Save the original ist values for checking stack pointers during debugging + */ struct orig_ist { - unsigned long ist[7]; + unsigned long ist[7]; }; #define MXCSR_DEFAULT 0x1f80 struct i387_fsave_struct { - u32 cwd; - u32 swd; - u32 twd; - u32 fip; - u32 fcs; - u32 foo; - u32 fos; - u32 st_space[20]; /* 8*10 bytes for each FP-reg = 80 bytes */ - u32 status; /* software status information */ + u32 cwd; /* FPU Control Word */ + u32 swd; /* FPU Status Word */ + u32 twd; /* FPU Tag Word */ + u32 fip; /* FPU IP Offset */ + u32 fcs; /* FPU IP Selector */ + u32 foo; /* FPU Operand Pointer Offset */ + u32 fos; /* FPU Operand Pointer Selector */ + + /* 8*10 bytes for each FP-reg = 80 bytes: */ + u32 st_space[20]; + + /* Software status information [not touched by FSAVE ]: */ + u32 status; }; struct i387_fxsave_struct { - u16 cwd; - u16 swd; - u16 twd; - u16 fop; + u16 cwd; /* Control Word */ + u16 swd; /* Status Word */ + u16 twd; /* Tag Word */ + u16 fop; /* Last Instruction Opcode */ union { struct { - u64 rip; - u64 rdp; + u64 rip; /* Instruction Pointer */ + u64 rdp; /* Data Pointer */ }; struct { - u32 fip; - u32 fcs; - u32 foo; - u32 fos; + u32 fip; /* FPU IP Offset */ + u32 fcs; /* FPU IP Selector */ + u32 foo; /* FPU Operand Offset */ + u32 fos; /* FPU Operand Selector */ }; }; - u32 mxcsr; - u32 mxcsr_mask; - u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */ - u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */ - u32 padding[24]; + u32 mxcsr; /* MXCSR Register State */ + u32 mxcsr_mask; /* MXCSR Mask */ + + /* 8*16 bytes for each FP-reg = 128 bytes: */ + u32 st_space[32]; + + /* 16*16 bytes for each XMM-reg = 256 bytes: */ + u32 xmm_space[64]; + + u32 padding[24]; + } __attribute__((aligned(16))); struct i387_soft_struct { - u32 cwd; - u32 swd; - u32 twd; - u32 fip; - u32 fcs; - u32 foo; - u32 fos; - u32 st_space[20]; /* 8*10 bytes for each FP-reg = 80 bytes */ - u8 ftop, changed, lookahead, no_update, rm, alimit; - struct info *info; - u32 entry_eip; + u32 cwd; + u32 swd; + u32 twd; + u32 fip; + u32 fcs; + u32 foo; + u32 fos; + /* 8*10 bytes for each FP-reg = 80 bytes: */ + u32 st_space[20]; + u8 ftop; + u8 changed; + u8 lookahead; + u8 no_update; + u8 rm; + u8 alimit; + struct info *info; + u32 entry_eip; }; -union i387_union { +union thread_xstate { struct i387_fsave_struct fsave; struct i387_fxsave_struct fxsave; - struct i387_soft_struct soft; + struct i387_soft_struct soft; }; -#ifdef CONFIG_X86_32 -DECLARE_PER_CPU(u8, cpu_llc_id); -#else +#ifdef CONFIG_X86_64 DECLARE_PER_CPU(struct orig_ist, orig_ist); #endif extern void print_cpu_info(struct cpuinfo_x86 *); +extern unsigned int xstate_size; +extern void free_thread_xstate(struct task_struct *); +extern struct kmem_cache *task_xstate_cachep; extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c); extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c); extern unsigned short num_cache_leaves; struct thread_struct { -/* cached TLS descriptors. */ - struct desc_struct tls_array[GDT_ENTRY_TLS_ENTRIES]; - unsigned long sp0; - unsigned long sp; + /* Cached TLS descriptors: */ + struct desc_struct tls_array[GDT_ENTRY_TLS_ENTRIES]; + unsigned long sp0; + unsigned long sp; #ifdef CONFIG_X86_32 - unsigned long sysenter_cs; + unsigned long sysenter_cs; #else - unsigned long usersp; /* Copy from PDA */ - unsigned short es, ds, fsindex, gsindex; + unsigned long usersp; /* Copy from PDA */ + unsigned short es; + unsigned short ds; + unsigned short fsindex; + unsigned short gsindex; #endif - unsigned long ip; - unsigned long fs; - unsigned long gs; -/* Hardware debugging registers */ - unsigned long debugreg0; - unsigned long debugreg1; - unsigned long debugreg2; - unsigned long debugreg3; - unsigned long debugreg6; - unsigned long debugreg7; -/* fault info */ - unsigned long cr2, trap_no, error_code; -/* floating point info */ - union i387_union i387 __attribute__((aligned(16)));; + unsigned long ip; + unsigned long fs; + unsigned long gs; + /* Hardware debugging registers: */ + unsigned long debugreg0; + unsigned long debugreg1; + unsigned long debugreg2; + unsigned long debugreg3; + unsigned long debugreg6; + unsigned long debugreg7; + /* Fault info: */ + unsigned long cr2; + unsigned long trap_no; + unsigned long error_code; + /* floating point and extended processor state */ + union thread_xstate *xstate; #ifdef CONFIG_X86_32 -/* virtual 86 mode info */ + /* Virtual 86 mode info */ struct vm86_struct __user *vm86_info; unsigned long screen_bitmap; - unsigned long v86flags, v86mask, saved_sp0; - unsigned int saved_fs, saved_gs; + unsigned long v86flags; + unsigned long v86mask; + unsigned long saved_sp0; + unsigned int saved_fs; + unsigned int saved_gs; #endif -/* IO permissions */ - unsigned long *io_bitmap_ptr; - unsigned long iopl; -/* max allowed port in the bitmap, in bytes: */ - unsigned io_bitmap_max; + /* IO permissions: */ + unsigned long *io_bitmap_ptr; + unsigned long iopl; + /* Max allowed port in the bitmap, in bytes: */ + unsigned io_bitmap_max; /* MSR_IA32_DEBUGCTLMSR value to switch in if TIF_DEBUGCTLMSR is set. */ unsigned long debugctlmsr; /* Debug Store - if not 0 points to a DS Save Area configuration; @@ -358,21 +422,27 @@ struct thread_struct { static inline unsigned long native_get_debugreg(int regno) { - unsigned long val = 0; /* Damn you, gcc! */ + unsigned long val = 0; /* Damn you, gcc! */ switch (regno) { case 0: - asm("mov %%db0, %0" :"=r" (val)); break; + asm("mov %%db0, %0" :"=r" (val)); + break; case 1: - asm("mov %%db1, %0" :"=r" (val)); break; + asm("mov %%db1, %0" :"=r" (val)); + break; case 2: - asm("mov %%db2, %0" :"=r" (val)); break; + asm("mov %%db2, %0" :"=r" (val)); + break; case 3: - asm("mov %%db3, %0" :"=r" (val)); break; + asm("mov %%db3, %0" :"=r" (val)); + break; case 6: - asm("mov %%db6, %0" :"=r" (val)); break; + asm("mov %%db6, %0" :"=r" (val)); + break; case 7: - asm("mov %%db7, %0" :"=r" (val)); break; + asm("mov %%db7, %0" :"=r" (val)); + break; default: BUG(); } @@ -383,22 +453,22 @@ static inline void native_set_debugreg(int regno, unsigned long value) { switch (regno) { case 0: - asm("mov %0,%%db0" : /* no output */ :"r" (value)); + asm("mov %0, %%db0" ::"r" (value)); break; case 1: - asm("mov %0,%%db1" : /* no output */ :"r" (value)); + asm("mov %0, %%db1" ::"r" (value)); break; case 2: - asm("mov %0,%%db2" : /* no output */ :"r" (value)); + asm("mov %0, %%db2" ::"r" (value)); break; case 3: - asm("mov %0,%%db3" : /* no output */ :"r" (value)); + asm("mov %0, %%db3" ::"r" (value)); break; case 6: - asm("mov %0,%%db6" : /* no output */ :"r" (value)); + asm("mov %0, %%db6" ::"r" (value)); break; case 7: - asm("mov %0,%%db7" : /* no output */ :"r" (value)); + asm("mov %0, %%db7" ::"r" (value)); break; default: BUG(); @@ -412,23 +482,24 @@ static inline void native_set_iopl_mask(unsigned mask) { #ifdef CONFIG_X86_32 unsigned int reg; - __asm__ __volatile__ ("pushfl;" - "popl %0;" - "andl %1, %0;" - "orl %2, %0;" - "pushl %0;" - "popfl" - : "=&r" (reg) - : "i" (~X86_EFLAGS_IOPL), "r" (mask)); + + asm volatile ("pushfl;" + "popl %0;" + "andl %1, %0;" + "orl %2, %0;" + "pushl %0;" + "popfl" + : "=&r" (reg) + : "i" (~X86_EFLAGS_IOPL), "r" (mask)); #endif } -static inline void native_load_sp0(struct tss_struct *tss, - struct thread_struct *thread) +static inline void +native_load_sp0(struct tss_struct *tss, struct thread_struct *thread) { tss->x86_tss.sp0 = thread->sp0; #ifdef CONFIG_X86_32 - /* Only happens when SEP is enabled, no need to test "SEP"arately */ + /* Only happens when SEP is enabled, no need to test "SEP"arately: */ if (unlikely(tss->x86_tss.ss1 != thread->sysenter_cs)) { tss->x86_tss.ss1 = thread->sysenter_cs; wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0); @@ -446,8 +517,8 @@ static inline void native_swapgs(void) #ifdef CONFIG_PARAVIRT #include <asm/paravirt.h> #else -#define __cpuid native_cpuid -#define paravirt_enabled() 0 +#define __cpuid native_cpuid +#define paravirt_enabled() 0 /* * These special macros can be used to get or set a debugging register @@ -473,11 +544,12 @@ static inline void load_sp0(struct tss_struct *tss, * enable), so that any CPU's that boot up * after us can get the correct flags. */ -extern unsigned long mmu_cr4_features; +extern unsigned long mmu_cr4_features; static inline void set_in_cr4(unsigned long mask) { unsigned cr4; + mmu_cr4_features |= mask; cr4 = read_cr4(); cr4 |= mask; @@ -487,6 +559,7 @@ static inline void set_in_cr4(unsigned long mask) static inline void clear_in_cr4(unsigned long mask) { unsigned cr4; + mmu_cr4_features &= ~mask; cr4 = read_cr4(); cr4 &= ~mask; @@ -494,42 +567,42 @@ static inline void clear_in_cr4(unsigned long mask) } struct microcode_header { - unsigned int hdrver; - unsigned int rev; - unsigned int date; - unsigned int sig; - unsigned int cksum; - unsigned int ldrver; - unsigned int pf; - unsigned int datasize; - unsigned int totalsize; - unsigned int reserved[3]; + unsigned int hdrver; + unsigned int rev; + unsigned int date; + unsigned int sig; + unsigned int cksum; + unsigned int ldrver; + unsigned int pf; + unsigned int datasize; + unsigned int totalsize; + unsigned int reserved[3]; }; struct microcode { - struct microcode_header hdr; - unsigned int bits[0]; + struct microcode_header hdr; + unsigned int bits[0]; }; -typedef struct microcode microcode_t; -typedef struct microcode_header microcode_header_t; +typedef struct microcode microcode_t; +typedef struct microcode_header microcode_header_t; /* microcode format is extended from prescott processors */ struct extended_signature { - unsigned int sig; - unsigned int pf; - unsigned int cksum; + unsigned int sig; + unsigned int pf; + unsigned int cksum; }; struct extended_sigtable { - unsigned int count; - unsigned int cksum; - unsigned int reserved[3]; + unsigned int count; + unsigned int cksum; + unsigned int reserved[3]; struct extended_signature sigs[0]; }; typedef struct { - unsigned long seg; + unsigned long seg; } mm_segment_t; @@ -541,7 +614,7 @@ extern int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); /* Free all resources held by a thread. */ extern void release_thread(struct task_struct *); -/* Prepare to copy thread state - unlazy all lazy status */ +/* Prepare to copy thread state - unlazy all lazy state */ extern void prepare_to_copy(struct task_struct *tsk); unsigned long get_wchan(struct task_struct *p); @@ -578,118 +651,138 @@ static inline unsigned int cpuid_eax(unsigned int op) unsigned int eax, ebx, ecx, edx; cpuid(op, &eax, &ebx, &ecx, &edx); + return eax; } + static inline unsigned int cpuid_ebx(unsigned int op) { unsigned int eax, ebx, ecx, edx; cpuid(op, &eax, &ebx, &ecx, &edx); + return ebx; } + static inline unsigned int cpuid_ecx(unsigned int op) { unsigned int eax, ebx, ecx, edx; cpuid(op, &eax, &ebx, &ecx, &edx); + return ecx; } + static inline unsigned int cpuid_edx(unsigned int op) { unsigned int eax, ebx, ecx, edx; cpuid(op, &eax, &ebx, &ecx, &edx); + return edx; } /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ static inline void rep_nop(void) { - __asm__ __volatile__("rep;nop": : :"memory"); + asm volatile("rep; nop" ::: "memory"); +} + +static inline void cpu_relax(void) +{ + rep_nop(); } -/* Stop speculative execution */ +/* Stop speculative execution: */ static inline void sync_core(void) { int tmp; + asm volatile("cpuid" : "=a" (tmp) : "0" (1) - : "ebx", "ecx", "edx", "memory"); + : "ebx", "ecx", "edx", "memory"); } -#define cpu_relax() rep_nop() - static inline void __monitor(const void *eax, unsigned long ecx, - unsigned long edx) + unsigned long edx) { - /* "monitor %eax,%ecx,%edx;" */ - asm volatile( - ".byte 0x0f,0x01,0xc8;" - : :"a" (eax), "c" (ecx), "d"(edx)); + /* "monitor %eax, %ecx, %edx;" */ + asm volatile(".byte 0x0f, 0x01, 0xc8;" + :: "a" (eax), "c" (ecx), "d"(edx)); } static inline void __mwait(unsigned long eax, unsigned long ecx) { - /* "mwait %eax,%ecx;" */ - asm volatile( - ".byte 0x0f,0x01,0xc9;" - : :"a" (eax), "c" (ecx)); + /* "mwait %eax, %ecx;" */ + asm volatile(".byte 0x0f, 0x01, 0xc9;" + :: "a" (eax), "c" (ecx)); } static inline void __sti_mwait(unsigned long eax, unsigned long ecx) { - /* "mwait %eax,%ecx;" */ - asm volatile( - "sti; .byte 0x0f,0x01,0xc9;" - : :"a" (eax), "c" (ecx)); + trace_hardirqs_on(); + /* "mwait %eax, %ecx;" */ + asm volatile("sti; .byte 0x0f, 0x01, 0xc9;" + :: "a" (eax), "c" (ecx)); } extern void mwait_idle_with_hints(unsigned long eax, unsigned long ecx); -extern int force_mwait; +extern int force_mwait; extern void select_idle_routine(const struct cpuinfo_x86 *c); -extern unsigned long boot_option_idle_override; +extern unsigned long boot_option_idle_override; extern void enable_sep_cpu(void); extern int sysenter_setup(void); /* Defined in head.S */ -extern struct desc_ptr early_gdt_descr; +extern struct desc_ptr early_gdt_descr; extern void cpu_set_gdt(int); extern void switch_to_new_gdt(void); extern void cpu_init(void); extern void init_gdt(int cpu); -/* from system description table in BIOS. Mostly for MCA use, but - * others may find it useful. */ -extern unsigned int machine_id; -extern unsigned int machine_submodel_id; -extern unsigned int BIOS_revision; +static inline void update_debugctlmsr(unsigned long debugctlmsr) +{ +#ifndef CONFIG_X86_DEBUGCTLMSR + if (boot_cpu_data.x86 < 6) + return; +#endif + wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr); +} + +/* + * from system description table in BIOS. Mostly for MCA use, but + * others may find it useful: + */ +extern unsigned int machine_id; +extern unsigned int machine_submodel_id; +extern unsigned int BIOS_revision; -/* Boot loader type from the setup header */ -extern int bootloader_type; +/* Boot loader type from the setup header: */ +extern int bootloader_type; -extern char ignore_fpu_irq; -#define cache_line_size() (boot_cpu_data.x86_cache_alignment) +extern char ignore_fpu_irq; #define HAVE_ARCH_PICK_MMAP_LAYOUT 1 #define ARCH_HAS_PREFETCHW #define ARCH_HAS_SPINLOCK_PREFETCH #ifdef CONFIG_X86_32 -#define BASE_PREFETCH ASM_NOP4 -#define ARCH_HAS_PREFETCH +# define BASE_PREFETCH ASM_NOP4 +# define ARCH_HAS_PREFETCH #else -#define BASE_PREFETCH "prefetcht0 (%1)" +# define BASE_PREFETCH "prefetcht0 (%1)" #endif -/* Prefetch instructions for Pentium III and AMD Athlon */ -/* It's not worth to care about 3dnow! prefetches for the K6 - because they are microcoded there and very slow. - However we don't do prefetches for pre XP Athlons currently - That should be fixed. */ +/* + * Prefetch instructions for Pentium III (+) and AMD Athlon (+) + * + * It's not worth to care about 3dnow prefetches for the K6 + * because they are microcoded there and very slow. + */ static inline void prefetch(const void *x) { alternative_input(BASE_PREFETCH, @@ -698,8 +791,11 @@ static inline void prefetch(const void *x) "r" (x)); } -/* 3dnow! prefetch to get an exclusive cache line. Useful for - spinlocks to avoid one state transition in the cache coherency protocol. */ +/* + * 3dnow prefetch to get an exclusive cache line. + * Useful for spinlocks to avoid one state transition in the + * cache coherency protocol: + */ static inline void prefetchw(const void *x) { alternative_input(BASE_PREFETCH, @@ -708,21 +804,25 @@ static inline void prefetchw(const void *x) "r" (x)); } -#define spin_lock_prefetch(x) prefetchw(x) +static inline void spin_lock_prefetch(const void *x) +{ + prefetchw(x); +} + #ifdef CONFIG_X86_32 /* * User space process size: 3GB (default). */ -#define TASK_SIZE (PAGE_OFFSET) -#define STACK_TOP TASK_SIZE -#define STACK_TOP_MAX STACK_TOP - -#define INIT_THREAD { \ - .sp0 = sizeof(init_stack) + (long)&init_stack, \ - .vm86_info = NULL, \ - .sysenter_cs = __KERNEL_CS, \ - .io_bitmap_ptr = NULL, \ - .fs = __KERNEL_PERCPU, \ +#define TASK_SIZE PAGE_OFFSET +#define STACK_TOP TASK_SIZE +#define STACK_TOP_MAX STACK_TOP + +#define INIT_THREAD { \ + .sp0 = sizeof(init_stack) + (long)&init_stack, \ + .vm86_info = NULL, \ + .sysenter_cs = __KERNEL_CS, \ + .io_bitmap_ptr = NULL, \ + .fs = __KERNEL_PERCPU, \ } /* @@ -731,28 +831,15 @@ static inline void prefetchw(const void *x) * permission bitmap. The extra byte must be all 1 bits, and must * be within the limit. */ -#define INIT_TSS { \ - .x86_tss = { \ +#define INIT_TSS { \ + .x86_tss = { \ .sp0 = sizeof(init_stack) + (long)&init_stack, \ - .ss0 = __KERNEL_DS, \ - .ss1 = __KERNEL_CS, \ - .io_bitmap_base = INVALID_IO_BITMAP_OFFSET, \ - }, \ - .io_bitmap = { [0 ... IO_BITMAP_LONGS] = ~0 }, \ -} - -#define start_thread(regs, new_eip, new_esp) do { \ - __asm__("movl %0,%%gs": :"r" (0)); \ - regs->fs = 0; \ - set_fs(USER_DS); \ - regs->ds = __USER_DS; \ - regs->es = __USER_DS; \ - regs->ss = __USER_DS; \ - regs->cs = __USER_CS; \ - regs->ip = new_eip; \ - regs->sp = new_esp; \ -} while (0) - + .ss0 = __KERNEL_DS, \ + .ss1 = __KERNEL_CS, \ + .io_bitmap_base = INVALID_IO_BITMAP_OFFSET, \ + }, \ + .io_bitmap = { [0 ... IO_BITMAP_LONGS] = ~0 }, \ +} extern unsigned long thread_saved_pc(struct task_struct *tsk); @@ -780,24 +867,24 @@ extern unsigned long thread_saved_pc(struct task_struct *tsk); __regs__ - 1; \ }) -#define KSTK_ESP(task) (task_pt_regs(task)->sp) +#define KSTK_ESP(task) (task_pt_regs(task)->sp) #else /* * User space process size. 47bits minus one guard page. */ -#define TASK_SIZE64 (0x800000000000UL - 4096) +#define TASK_SIZE64 ((1UL << 47) - PAGE_SIZE) /* This decides where the kernel will search for a free chunk of vm * space during mmap's. */ -#define IA32_PAGE_OFFSET ((current->personality & ADDR_LIMIT_3GB) ? \ - 0xc0000000 : 0xFFFFe000) +#define IA32_PAGE_OFFSET ((current->personality & ADDR_LIMIT_3GB) ? \ + 0xc0000000 : 0xFFFFe000) -#define TASK_SIZE (test_thread_flag(TIF_IA32) ? \ - IA32_PAGE_OFFSET : TASK_SIZE64) -#define TASK_SIZE_OF(child) ((test_tsk_thread_flag(child, TIF_IA32)) ? \ - IA32_PAGE_OFFSET : TASK_SIZE64) +#define TASK_SIZE (test_thread_flag(TIF_IA32) ? \ + IA32_PAGE_OFFSET : TASK_SIZE64) +#define TASK_SIZE_OF(child) ((test_tsk_thread_flag(child, TIF_IA32)) ? \ + IA32_PAGE_OFFSET : TASK_SIZE64) #define STACK_TOP TASK_SIZE #define STACK_TOP_MAX TASK_SIZE64 @@ -810,33 +897,32 @@ extern unsigned long thread_saved_pc(struct task_struct *tsk); .x86_tss.sp0 = (unsigned long)&init_stack + sizeof(init_stack) \ } -#define start_thread(regs, new_rip, new_rsp) do { \ - asm volatile("movl %0,%%fs; movl %0,%%es; movl %0,%%ds": :"r" (0)); \ - load_gs_index(0); \ - (regs)->ip = (new_rip); \ - (regs)->sp = (new_rsp); \ - write_pda(oldrsp, (new_rsp)); \ - (regs)->cs = __USER_CS; \ - (regs)->ss = __USER_DS; \ - (regs)->flags = 0x200; \ - set_fs(USER_DS); \ -} while (0) - /* * Return saved PC of a blocked thread. * What is this good for? it will be always the scheduler or ret_from_fork. */ -#define thread_saved_pc(t) (*(unsigned long *)((t)->thread.sp - 8)) +#define thread_saved_pc(t) (*(unsigned long *)((t)->thread.sp - 8)) -#define task_pt_regs(tsk) ((struct pt_regs *)(tsk)->thread.sp0 - 1) -#define KSTK_ESP(tsk) -1 /* sorry. doesn't work for syscall. */ +#define task_pt_regs(tsk) ((struct pt_regs *)(tsk)->thread.sp0 - 1) +#define KSTK_ESP(tsk) -1 /* sorry. doesn't work for syscall. */ #endif /* CONFIG_X86_64 */ -/* This decides where the kernel will search for a free chunk of vm +extern void start_thread(struct pt_regs *regs, unsigned long new_ip, + unsigned long new_sp); + +/* + * This decides where the kernel will search for a free chunk of vm * space during mmap's. */ #define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 3)) -#define KSTK_EIP(task) (task_pt_regs(task)->ip) +#define KSTK_EIP(task) (task_pt_regs(task)->ip) + +/* Get/set a process' ability to use the timestamp counter instruction */ +#define GET_TSC_CTL(adr) get_tsc_mode((adr)) +#define SET_TSC_CTL(val) set_tsc_mode((val)) + +extern int get_tsc_mode(unsigned long adr); +extern int set_tsc_mode(unsigned int val); #endif diff --git a/include/asm-x86/proto.h b/include/asm-x86/proto.h index 68563c0709a..6c8b41b03f6 100644 --- a/include/asm-x86/proto.h +++ b/include/asm-x86/proto.h @@ -7,8 +7,6 @@ extern void early_idt_handler(void); -extern void init_memory_mapping(unsigned long start, unsigned long end); - extern void system_call(void); extern void syscall_init(void); @@ -22,11 +20,15 @@ extern void syscall32_cpu_init(void); extern void check_efer(void); +#ifdef CONFIG_X86_BIOS_REBOOT extern int reboot_force; +#else +static const int reboot_force = 0; +#endif long do_arch_prctl(struct task_struct *task, int code, unsigned long addr); -#define round_up(x,y) (((x) + (y) - 1) & ~((y)-1)) -#define round_down(x,y) ((x) & ~((y)-1)) +#define round_up(x, y) (((x) + (y) - 1) & ~((y) - 1)) +#define round_down(x, y) ((x) & ~((y) - 1)) #endif diff --git a/include/asm-x86/ptrace.h b/include/asm-x86/ptrace.h index d9e04b46a44..9f922b0b95d 100644 --- a/include/asm-x86/ptrace.h +++ b/include/asm-x86/ptrace.h @@ -36,23 +36,23 @@ struct pt_regs { #else /* __KERNEL__ */ struct pt_regs { - long bx; - long cx; - long dx; - long si; - long di; - long bp; - long ax; - int ds; - int es; - int fs; + unsigned long bx; + unsigned long cx; + unsigned long dx; + unsigned long si; + unsigned long di; + unsigned long bp; + unsigned long ax; + unsigned long ds; + unsigned long es; + unsigned long fs; /* int gs; */ - long orig_ax; - long ip; - int cs; - long flags; - long sp; - int ss; + unsigned long orig_ax; + unsigned long ip; + unsigned long cs; + unsigned long flags; + unsigned long sp; + unsigned long ss; }; #include <asm/vm86.h> @@ -140,12 +140,16 @@ extern unsigned long convert_ip_to_linear(struct task_struct *child, struct pt_regs *regs); #ifdef CONFIG_X86_32 -extern void send_sigtrap(struct task_struct *tsk, struct pt_regs *regs, int error_code); +extern void send_sigtrap(struct task_struct *tsk, struct pt_regs *regs, + int error_code); #else void signal_fault(struct pt_regs *regs, void __user *frame, char *where); #endif -#define regs_return_value(regs) ((regs)->ax) +static inline unsigned long regs_return_value(struct pt_regs *regs) +{ + return regs->ax; +} /* * user_mode_vm(regs) determines whether a register set came from user mode. @@ -166,8 +170,8 @@ static inline int user_mode(struct pt_regs *regs) static inline int user_mode_vm(struct pt_regs *regs) { #ifdef CONFIG_X86_32 - return ((regs->cs & SEGMENT_RPL_MASK) | - (regs->flags & VM_MASK)) >= USER_RPL; + return ((regs->cs & SEGMENT_RPL_MASK) | (regs->flags & X86_VM_MASK)) >= + USER_RPL; #else return user_mode(regs); #endif @@ -176,7 +180,7 @@ static inline int user_mode_vm(struct pt_regs *regs) static inline int v8086_mode(struct pt_regs *regs) { #ifdef CONFIG_X86_32 - return (regs->flags & VM_MASK); + return (regs->flags & X86_VM_MASK); #else return 0; /* No V86 mode support in long mode */ #endif @@ -227,6 +231,8 @@ extern int do_get_thread_area(struct task_struct *p, int idx, extern int do_set_thread_area(struct task_struct *p, int idx, struct user_desc __user *info, int can_allocate); +#define __ARCH_WANT_COMPAT_SYS_PTRACE + #endif /* __KERNEL__ */ #endif /* !__ASSEMBLY__ */ diff --git a/include/asm-x86/reboot.h b/include/asm-x86/reboot.h index e9e3ffc22c0..e63741f1939 100644 --- a/include/asm-x86/reboot.h +++ b/include/asm-x86/reboot.h @@ -3,8 +3,7 @@ struct pt_regs; -struct machine_ops -{ +struct machine_ops { void (*restart)(char *cmd); void (*halt)(void); void (*power_off)(void); @@ -16,5 +15,7 @@ struct machine_ops extern struct machine_ops machine_ops; void machine_real_restart(unsigned char *code, int length); +void native_machine_crash_shutdown(struct pt_regs *regs); +void native_machine_shutdown(void); #endif /* _ASM_REBOOT_H */ diff --git a/include/asm-x86/resume-trace.h b/include/asm-x86/resume-trace.h index 46f725b0bc8..2557514d7ef 100644 --- a/include/asm-x86/resume-trace.h +++ b/include/asm-x86/resume-trace.h @@ -3,16 +3,17 @@ #include <asm/asm.h> -#define TRACE_RESUME(user) do { \ +#define TRACE_RESUME(user) \ +do { \ if (pm_trace_enabled) { \ void *tracedata; \ asm volatile(_ASM_MOV_UL " $1f,%0\n" \ - ".section .tracedata,\"a\"\n" \ - "1:\t.word %c1\n\t" \ - _ASM_PTR " %c2\n" \ - ".previous" \ - :"=r" (tracedata) \ - : "i" (__LINE__), "i" (__FILE__)); \ + ".section .tracedata,\"a\"\n" \ + "1:\t.word %c1\n\t" \ + _ASM_PTR " %c2\n" \ + ".previous" \ + :"=r" (tracedata) \ + : "i" (__LINE__), "i" (__FILE__)); \ generate_resume_trace(tracedata, user); \ } \ } while (0) diff --git a/include/asm-x86/rio.h b/include/asm-x86/rio.h index 97cdcc9887b..c9448bd8968 100644 --- a/include/asm-x86/rio.h +++ b/include/asm-x86/rio.h @@ -11,64 +11,53 @@ #define RIO_TABLE_VERSION 3 struct rio_table_hdr { - u8 version; /* Version number of this data structure */ - u8 num_scal_dev; /* # of Scalability devices */ - u8 num_rio_dev; /* # of RIO I/O devices */ + u8 version; /* Version number of this data structure */ + u8 num_scal_dev; /* # of Scalability devices */ + u8 num_rio_dev; /* # of RIO I/O devices */ } __attribute__((packed)); struct scal_detail { - u8 node_id; /* Scalability Node ID */ - u32 CBAR; /* Address of 1MB register space */ - u8 port0node; /* Node ID port connected to: 0xFF=None */ - u8 port0port; /* Port num port connected to: 0,1,2, or */ - /* 0xFF=None */ - u8 port1node; /* Node ID port connected to: 0xFF = None */ - u8 port1port; /* Port num port connected to: 0,1,2, or */ - /* 0xFF=None */ - u8 port2node; /* Node ID port connected to: 0xFF = None */ - u8 port2port; /* Port num port connected to: 0,1,2, or */ - /* 0xFF=None */ - u8 chassis_num; /* 1 based Chassis number (1 = boot node) */ + u8 node_id; /* Scalability Node ID */ + u32 CBAR; /* Address of 1MB register space */ + u8 port0node; /* Node ID port connected to: 0xFF=None */ + u8 port0port; /* Port num port connected to: 0,1,2, or */ + /* 0xFF=None */ + u8 port1node; /* Node ID port connected to: 0xFF = None */ + u8 port1port; /* Port num port connected to: 0,1,2, or */ + /* 0xFF=None */ + u8 port2node; /* Node ID port connected to: 0xFF = None */ + u8 port2port; /* Port num port connected to: 0,1,2, or */ + /* 0xFF=None */ + u8 chassis_num; /* 1 based Chassis number (1 = boot node) */ } __attribute__((packed)); struct rio_detail { - u8 node_id; /* RIO Node ID */ - u32 BBAR; /* Address of 1MB register space */ - u8 type; /* Type of device */ - u8 owner_id; /* Node ID of Hurricane that owns this */ - /* node */ - u8 port0node; /* Node ID port connected to: 0xFF=None */ - u8 port0port; /* Port num port connected to: 0,1,2, or */ - /* 0xFF=None */ - u8 port1node; /* Node ID port connected to: 0xFF=None */ - u8 port1port; /* Port num port connected to: 0,1,2, or */ - /* 0xFF=None */ - u8 first_slot; /* Lowest slot number below this Calgary */ - u8 status; /* Bit 0 = 1 : the XAPIC is used */ - /* = 0 : the XAPIC is not used, ie: */ - /* ints fwded to another XAPIC */ - /* Bits1:7 Reserved */ - u8 WP_index; /* instance index - lower ones have */ - /* lower slot numbers/PCI bus numbers */ - u8 chassis_num; /* 1 based Chassis number */ + u8 node_id; /* RIO Node ID */ + u32 BBAR; /* Address of 1MB register space */ + u8 type; /* Type of device */ + u8 owner_id; /* Node ID of Hurricane that owns this */ + /* node */ + u8 port0node; /* Node ID port connected to: 0xFF=None */ + u8 port0port; /* Port num port connected to: 0,1,2, or */ + /* 0xFF=None */ + u8 port1node; /* Node ID port connected to: 0xFF=None */ + u8 port1port; /* Port num port connected to: 0,1,2, or */ + /* 0xFF=None */ + u8 first_slot; /* Lowest slot number below this Calgary */ + u8 status; /* Bit 0 = 1 : the XAPIC is used */ + /* = 0 : the XAPIC is not used, ie: */ + /* ints fwded to another XAPIC */ + /* Bits1:7 Reserved */ + u8 WP_index; /* instance index - lower ones have */ + /* lower slot numbers/PCI bus numbers */ + u8 chassis_num; /* 1 based Chassis number */ } __attribute__((packed)); enum { - HURR_SCALABILTY = 0, /* Hurricane Scalability info */ - HURR_RIOIB = 2, /* Hurricane RIOIB info */ - COMPAT_CALGARY = 4, /* Compatibility Calgary */ - ALT_CALGARY = 5, /* Second Planar Calgary */ + HURR_SCALABILTY = 0, /* Hurricane Scalability info */ + HURR_RIOIB = 2, /* Hurricane RIOIB info */ + COMPAT_CALGARY = 4, /* Compatibility Calgary */ + ALT_CALGARY = 5, /* Second Planar Calgary */ }; -/* - * there is a real-mode segmented pointer pointing to the - * 4K EBDA area at 0x40E. - */ -static inline unsigned long get_bios_ebda(void) -{ - unsigned long address = *(unsigned short *)phys_to_virt(0x40EUL); - address <<= 4; - return address; -} - #endif /* __ASM_RIO_H */ diff --git a/include/asm-x86/rwsem.h b/include/asm-x86/rwsem.h index 520a379f4b8..750f2a3542b 100644 --- a/include/asm-x86/rwsem.h +++ b/include/asm-x86/rwsem.h @@ -56,14 +56,16 @@ extern asmregparm struct rw_semaphore * /* * the semaphore definition */ -struct rw_semaphore { - signed long count; + #define RWSEM_UNLOCKED_VALUE 0x00000000 #define RWSEM_ACTIVE_BIAS 0x00000001 #define RWSEM_ACTIVE_MASK 0x0000ffff #define RWSEM_WAITING_BIAS (-0x00010000) #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) + +struct rw_semaphore { + signed long count; spinlock_t wait_lock; struct list_head wait_list; #ifdef CONFIG_DEBUG_LOCK_ALLOC @@ -78,11 +80,13 @@ struct rw_semaphore { #endif -#define __RWSEM_INITIALIZER(name) \ -{ RWSEM_UNLOCKED_VALUE, __SPIN_LOCK_UNLOCKED((name).wait_lock), \ - LIST_HEAD_INIT((name).wait_list) __RWSEM_DEP_MAP_INIT(name) } +#define __RWSEM_INITIALIZER(name) \ +{ \ + RWSEM_UNLOCKED_VALUE, __SPIN_LOCK_UNLOCKED((name).wait_lock), \ + LIST_HEAD_INIT((name).wait_list) __RWSEM_DEP_MAP_INIT(name) \ +} -#define DECLARE_RWSEM(name) \ +#define DECLARE_RWSEM(name) \ struct rw_semaphore name = __RWSEM_INITIALIZER(name) extern void __init_rwsem(struct rw_semaphore *sem, const char *name, @@ -100,16 +104,16 @@ do { \ */ static inline void __down_read(struct rw_semaphore *sem) { - __asm__ __volatile__( - "# beginning down_read\n\t" -LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */ - " jns 1f\n" - " call call_rwsem_down_read_failed\n" - "1:\n\t" - "# ending down_read\n\t" - : "+m" (sem->count) - : "a" (sem) - : "memory", "cc"); + asm volatile("# beginning down_read\n\t" + LOCK_PREFIX " incl (%%eax)\n\t" + /* adds 0x00000001, returns the old value */ + " jns 1f\n" + " call call_rwsem_down_read_failed\n" + "1:\n\t" + "# ending down_read\n\t" + : "+m" (sem->count) + : "a" (sem) + : "memory", "cc"); } /* @@ -118,21 +122,20 @@ LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value static inline int __down_read_trylock(struct rw_semaphore *sem) { __s32 result, tmp; - __asm__ __volatile__( - "# beginning __down_read_trylock\n\t" - " movl %0,%1\n\t" - "1:\n\t" - " movl %1,%2\n\t" - " addl %3,%2\n\t" - " jle 2f\n\t" -LOCK_PREFIX " cmpxchgl %2,%0\n\t" - " jnz 1b\n\t" - "2:\n\t" - "# ending __down_read_trylock\n\t" - : "+m" (sem->count), "=&a" (result), "=&r" (tmp) - : "i" (RWSEM_ACTIVE_READ_BIAS) - : "memory", "cc"); - return result>=0 ? 1 : 0; + asm volatile("# beginning __down_read_trylock\n\t" + " movl %0,%1\n\t" + "1:\n\t" + " movl %1,%2\n\t" + " addl %3,%2\n\t" + " jle 2f\n\t" + LOCK_PREFIX " cmpxchgl %2,%0\n\t" + " jnz 1b\n\t" + "2:\n\t" + "# ending __down_read_trylock\n\t" + : "+m" (sem->count), "=&a" (result), "=&r" (tmp) + : "i" (RWSEM_ACTIVE_READ_BIAS) + : "memory", "cc"); + return result >= 0 ? 1 : 0; } /* @@ -143,17 +146,18 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) int tmp; tmp = RWSEM_ACTIVE_WRITE_BIAS; - __asm__ __volatile__( - "# beginning down_write\n\t" -LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */ - " testl %%edx,%%edx\n\t" /* was the count 0 before? */ - " jz 1f\n" - " call call_rwsem_down_write_failed\n" - "1:\n" - "# ending down_write" - : "+m" (sem->count), "=d" (tmp) - : "a" (sem), "1" (tmp) - : "memory", "cc"); + asm volatile("# beginning down_write\n\t" + LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" + /* subtract 0x0000ffff, returns the old value */ + " testl %%edx,%%edx\n\t" + /* was the count 0 before? */ + " jz 1f\n" + " call call_rwsem_down_write_failed\n" + "1:\n" + "# ending down_write" + : "+m" (sem->count), "=d" (tmp) + : "a" (sem), "1" (tmp) + : "memory", "cc"); } static inline void __down_write(struct rw_semaphore *sem) @@ -167,7 +171,7 @@ static inline void __down_write(struct rw_semaphore *sem) static inline int __down_write_trylock(struct rw_semaphore *sem) { signed long ret = cmpxchg(&sem->count, - RWSEM_UNLOCKED_VALUE, + RWSEM_UNLOCKED_VALUE, RWSEM_ACTIVE_WRITE_BIAS); if (ret == RWSEM_UNLOCKED_VALUE) return 1; @@ -180,16 +184,16 @@ static inline int __down_write_trylock(struct rw_semaphore *sem) static inline void __up_read(struct rw_semaphore *sem) { __s32 tmp = -RWSEM_ACTIVE_READ_BIAS; - __asm__ __volatile__( - "# beginning __up_read\n\t" -LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */ - " jns 1f\n\t" - " call call_rwsem_wake\n" - "1:\n" - "# ending __up_read\n" - : "+m" (sem->count), "=d" (tmp) - : "a" (sem), "1" (tmp) - : "memory", "cc"); + asm volatile("# beginning __up_read\n\t" + LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" + /* subtracts 1, returns the old value */ + " jns 1f\n\t" + " call call_rwsem_wake\n" + "1:\n" + "# ending __up_read\n" + : "+m" (sem->count), "=d" (tmp) + : "a" (sem), "1" (tmp) + : "memory", "cc"); } /* @@ -197,17 +201,18 @@ LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old valu */ static inline void __up_write(struct rw_semaphore *sem) { - __asm__ __volatile__( - "# beginning __up_write\n\t" - " movl %2,%%edx\n\t" -LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */ - " jz 1f\n" - " call call_rwsem_wake\n" - "1:\n\t" - "# ending __up_write\n" - : "+m" (sem->count) - : "a" (sem), "i" (-RWSEM_ACTIVE_WRITE_BIAS) - : "memory", "cc", "edx"); + asm volatile("# beginning __up_write\n\t" + " movl %2,%%edx\n\t" + LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" + /* tries to transition + 0xffff0001 -> 0x00000000 */ + " jz 1f\n" + " call call_rwsem_wake\n" + "1:\n\t" + "# ending __up_write\n" + : "+m" (sem->count) + : "a" (sem), "i" (-RWSEM_ACTIVE_WRITE_BIAS) + : "memory", "cc", "edx"); } /* @@ -215,16 +220,16 @@ LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> */ static inline void __downgrade_write(struct rw_semaphore *sem) { - __asm__ __volatile__( - "# beginning __downgrade_write\n\t" -LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */ - " jns 1f\n\t" - " call call_rwsem_downgrade_wake\n" - "1:\n\t" - "# ending __downgrade_write\n" - : "+m" (sem->count) - : "a" (sem), "i" (-RWSEM_WAITING_BIAS) - : "memory", "cc"); + asm volatile("# beginning __downgrade_write\n\t" + LOCK_PREFIX " addl %2,(%%eax)\n\t" + /* transitions 0xZZZZ0001 -> 0xYYYY0001 */ + " jns 1f\n\t" + " call call_rwsem_downgrade_wake\n" + "1:\n\t" + "# ending __downgrade_write\n" + : "+m" (sem->count) + : "a" (sem), "i" (-RWSEM_WAITING_BIAS) + : "memory", "cc"); } /* @@ -232,10 +237,9 @@ LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */ static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem) { - __asm__ __volatile__( -LOCK_PREFIX "addl %1,%0" - : "+m" (sem->count) - : "ir" (delta)); + asm volatile(LOCK_PREFIX "addl %1,%0" + : "+m" (sem->count) + : "ir" (delta)); } /* @@ -245,12 +249,11 @@ static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem) { int tmp = delta; - __asm__ __volatile__( -LOCK_PREFIX "xadd %0,%1" - : "+r" (tmp), "+m" (sem->count) - : : "memory"); + asm volatile(LOCK_PREFIX "xadd %0,%1" + : "+r" (tmp), "+m" (sem->count) + : : "memory"); - return tmp+delta; + return tmp + delta; } static inline int rwsem_is_locked(struct rw_semaphore *sem) diff --git a/include/asm-x86/scatterlist.h b/include/asm-x86/scatterlist.h index d13c197866d..c0432061f81 100644 --- a/include/asm-x86/scatterlist.h +++ b/include/asm-x86/scatterlist.h @@ -11,9 +11,7 @@ struct scatterlist { unsigned int offset; unsigned int length; dma_addr_t dma_address; -#ifdef CONFIG_X86_64 unsigned int dma_length; -#endif }; #define ARCH_HAS_SG_CHAIN diff --git a/include/asm-x86/segment.h b/include/asm-x86/segment.h index 23f0535fec6..ed5131dd7d9 100644 --- a/include/asm-x86/segment.h +++ b/include/asm-x86/segment.h @@ -191,13 +191,14 @@ #define SEGMENT_TI_MASK 0x4 #define IDT_ENTRIES 256 +#define NUM_EXCEPTION_VECTORS 32 #define GDT_SIZE (GDT_ENTRIES * 8) #define GDT_ENTRY_TLS_ENTRIES 3 #define TLS_SIZE (GDT_ENTRY_TLS_ENTRIES * 8) #ifdef __KERNEL__ #ifndef __ASSEMBLY__ -extern const char early_idt_handlers[IDT_ENTRIES][10]; +extern const char early_idt_handlers[NUM_EXCEPTION_VECTORS][10]; #endif #endif diff --git a/include/asm-x86/semaphore.h b/include/asm-x86/semaphore.h index 572c0b67a6b..d9b2034ed1d 100644 --- a/include/asm-x86/semaphore.h +++ b/include/asm-x86/semaphore.h @@ -1,5 +1 @@ -#ifdef CONFIG_X86_32 -# include "semaphore_32.h" -#else -# include "semaphore_64.h" -#endif +#include <linux/semaphore.h> diff --git a/include/asm-x86/semaphore_32.h b/include/asm-x86/semaphore_32.h deleted file mode 100644 index ac96d3804d0..00000000000 --- a/include/asm-x86/semaphore_32.h +++ /dev/null @@ -1,175 +0,0 @@ -#ifndef _I386_SEMAPHORE_H -#define _I386_SEMAPHORE_H - -#include <linux/linkage.h> - -#ifdef __KERNEL__ - -/* - * SMP- and interrupt-safe semaphores.. - * - * (C) Copyright 1996 Linus Torvalds - * - * Modified 1996-12-23 by Dave Grothe <dave@gcom.com> to fix bugs in - * the original code and to make semaphore waits - * interruptible so that processes waiting on - * semaphores can be killed. - * Modified 1999-02-14 by Andrea Arcangeli, split the sched.c helper - * functions in asm/sempahore-helper.h while fixing a - * potential and subtle race discovered by Ulrich Schmid - * in down_interruptible(). Since I started to play here I - * also implemented the `trylock' semaphore operation. - * 1999-07-02 Artur Skawina <skawina@geocities.com> - * Optimized "0(ecx)" -> "(ecx)" (the assembler does not - * do this). Changed calling sequences from push/jmp to - * traditional call/ret. - * Modified 2001-01-01 Andreas Franck <afranck@gmx.de> - * Some hacks to ensure compatibility with recent - * GCC snapshots, to avoid stack corruption when compiling - * with -fomit-frame-pointer. It's not sure if this will - * be fixed in GCC, as our previous implementation was a - * bit dubious. - * - * If you would like to see an analysis of this implementation, please - * ftp to gcom.com and download the file - * /pub/linux/src/semaphore/semaphore-2.0.24.tar.gz. - * - */ - -#include <asm/system.h> -#include <asm/atomic.h> -#include <linux/wait.h> -#include <linux/rwsem.h> - -struct semaphore { - atomic_t count; - int sleepers; - wait_queue_head_t wait; -}; - - -#define __SEMAPHORE_INITIALIZER(name, n) \ -{ \ - .count = ATOMIC_INIT(n), \ - .sleepers = 0, \ - .wait = __WAIT_QUEUE_HEAD_INITIALIZER((name).wait) \ -} - -#define __DECLARE_SEMAPHORE_GENERIC(name,count) \ - struct semaphore name = __SEMAPHORE_INITIALIZER(name,count) - -#define DECLARE_MUTEX(name) __DECLARE_SEMAPHORE_GENERIC(name,1) - -static inline void sema_init (struct semaphore *sem, int val) -{ -/* - * *sem = (struct semaphore)__SEMAPHORE_INITIALIZER((*sem),val); - * - * i'd rather use the more flexible initialization above, but sadly - * GCC 2.7.2.3 emits a bogus warning. EGCS doesn't. Oh well. - */ - atomic_set(&sem->count, val); - sem->sleepers = 0; - init_waitqueue_head(&sem->wait); -} - -static inline void init_MUTEX (struct semaphore *sem) -{ - sema_init(sem, 1); -} - -static inline void init_MUTEX_LOCKED (struct semaphore *sem) -{ - sema_init(sem, 0); -} - -extern asmregparm void __down_failed(atomic_t *count_ptr); -extern asmregparm int __down_failed_interruptible(atomic_t *count_ptr); -extern asmregparm int __down_failed_trylock(atomic_t *count_ptr); -extern asmregparm void __up_wakeup(atomic_t *count_ptr); - -/* - * This is ugly, but we want the default case to fall through. - * "__down_failed" is a special asm handler that calls the C - * routine that actually waits. See arch/i386/kernel/semaphore.c - */ -static inline void down(struct semaphore * sem) -{ - might_sleep(); - __asm__ __volatile__( - "# atomic down operation\n\t" - LOCK_PREFIX "decl %0\n\t" /* --sem->count */ - "jns 2f\n" - "\tlea %0,%%eax\n\t" - "call __down_failed\n" - "2:" - :"+m" (sem->count) - : - :"memory","ax"); -} - -/* - * Interruptible try to acquire a semaphore. If we obtained - * it, return zero. If we were interrupted, returns -EINTR - */ -static inline int down_interruptible(struct semaphore * sem) -{ - int result; - - might_sleep(); - __asm__ __volatile__( - "# atomic interruptible down operation\n\t" - "xorl %0,%0\n\t" - LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "jns 2f\n\t" - "lea %1,%%eax\n\t" - "call __down_failed_interruptible\n" - "2:" - :"=&a" (result), "+m" (sem->count) - : - :"memory"); - return result; -} - -/* - * Non-blockingly attempt to down() a semaphore. - * Returns zero if we acquired it - */ -static inline int down_trylock(struct semaphore * sem) -{ - int result; - - __asm__ __volatile__( - "# atomic interruptible down operation\n\t" - "xorl %0,%0\n\t" - LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "jns 2f\n\t" - "lea %1,%%eax\n\t" - "call __down_failed_trylock\n\t" - "2:\n" - :"=&a" (result), "+m" (sem->count) - : - :"memory"); - return result; -} - -/* - * Note! This is subtle. We jump to wake people up only if - * the semaphore was negative (== somebody was waiting on it). - */ -static inline void up(struct semaphore * sem) -{ - __asm__ __volatile__( - "# atomic up operation\n\t" - LOCK_PREFIX "incl %0\n\t" /* ++sem->count */ - "jg 1f\n\t" - "lea %0,%%eax\n\t" - "call __up_wakeup\n" - "1:" - :"+m" (sem->count) - : - :"memory","ax"); -} - -#endif -#endif diff --git a/include/asm-x86/semaphore_64.h b/include/asm-x86/semaphore_64.h deleted file mode 100644 index 79694306bf7..00000000000 --- a/include/asm-x86/semaphore_64.h +++ /dev/null @@ -1,180 +0,0 @@ -#ifndef _X86_64_SEMAPHORE_H -#define _X86_64_SEMAPHORE_H - -#include <linux/linkage.h> - -#ifdef __KERNEL__ - -/* - * SMP- and interrupt-safe semaphores.. - * - * (C) Copyright 1996 Linus Torvalds - * - * Modified 1996-12-23 by Dave Grothe <dave@gcom.com> to fix bugs in - * the original code and to make semaphore waits - * interruptible so that processes waiting on - * semaphores can be killed. - * Modified 1999-02-14 by Andrea Arcangeli, split the sched.c helper - * functions in asm/sempahore-helper.h while fixing a - * potential and subtle race discovered by Ulrich Schmid - * in down_interruptible(). Since I started to play here I - * also implemented the `trylock' semaphore operation. - * 1999-07-02 Artur Skawina <skawina@geocities.com> - * Optimized "0(ecx)" -> "(ecx)" (the assembler does not - * do this). Changed calling sequences from push/jmp to - * traditional call/ret. - * Modified 2001-01-01 Andreas Franck <afranck@gmx.de> - * Some hacks to ensure compatibility with recent - * GCC snapshots, to avoid stack corruption when compiling - * with -fomit-frame-pointer. It's not sure if this will - * be fixed in GCC, as our previous implementation was a - * bit dubious. - * - * If you would like to see an analysis of this implementation, please - * ftp to gcom.com and download the file - * /pub/linux/src/semaphore/semaphore-2.0.24.tar.gz. - * - */ - -#include <asm/system.h> -#include <asm/atomic.h> -#include <asm/rwlock.h> -#include <linux/wait.h> -#include <linux/rwsem.h> -#include <linux/stringify.h> - -struct semaphore { - atomic_t count; - int sleepers; - wait_queue_head_t wait; -}; - -#define __SEMAPHORE_INITIALIZER(name, n) \ -{ \ - .count = ATOMIC_INIT(n), \ - .sleepers = 0, \ - .wait = __WAIT_QUEUE_HEAD_INITIALIZER((name).wait) \ -} - -#define __DECLARE_SEMAPHORE_GENERIC(name,count) \ - struct semaphore name = __SEMAPHORE_INITIALIZER(name,count) - -#define DECLARE_MUTEX(name) __DECLARE_SEMAPHORE_GENERIC(name,1) - -static inline void sema_init (struct semaphore *sem, int val) -{ -/* - * *sem = (struct semaphore)__SEMAPHORE_INITIALIZER((*sem),val); - * - * i'd rather use the more flexible initialization above, but sadly - * GCC 2.7.2.3 emits a bogus warning. EGCS doesn't. Oh well. - */ - atomic_set(&sem->count, val); - sem->sleepers = 0; - init_waitqueue_head(&sem->wait); -} - -static inline void init_MUTEX (struct semaphore *sem) -{ - sema_init(sem, 1); -} - -static inline void init_MUTEX_LOCKED (struct semaphore *sem) -{ - sema_init(sem, 0); -} - -asmlinkage void __down_failed(void /* special register calling convention */); -asmlinkage int __down_failed_interruptible(void /* params in registers */); -asmlinkage int __down_failed_trylock(void /* params in registers */); -asmlinkage void __up_wakeup(void /* special register calling convention */); - -asmlinkage void __down(struct semaphore * sem); -asmlinkage int __down_interruptible(struct semaphore * sem); -asmlinkage int __down_trylock(struct semaphore * sem); -asmlinkage void __up(struct semaphore * sem); - -/* - * This is ugly, but we want the default case to fall through. - * "__down_failed" is a special asm handler that calls the C - * routine that actually waits. See arch/x86_64/kernel/semaphore.c - */ -static inline void down(struct semaphore * sem) -{ - might_sleep(); - - __asm__ __volatile__( - "# atomic down operation\n\t" - LOCK_PREFIX "decl %0\n\t" /* --sem->count */ - "jns 1f\n\t" - "call __down_failed\n" - "1:" - :"=m" (sem->count) - :"D" (sem) - :"memory"); -} - -/* - * Interruptible try to acquire a semaphore. If we obtained - * it, return zero. If we were interrupted, returns -EINTR - */ -static inline int down_interruptible(struct semaphore * sem) -{ - int result; - - might_sleep(); - - __asm__ __volatile__( - "# atomic interruptible down operation\n\t" - "xorl %0,%0\n\t" - LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "jns 2f\n\t" - "call __down_failed_interruptible\n" - "2:\n" - :"=&a" (result), "=m" (sem->count) - :"D" (sem) - :"memory"); - return result; -} - -/* - * Non-blockingly attempt to down() a semaphore. - * Returns zero if we acquired it - */ -static inline int down_trylock(struct semaphore * sem) -{ - int result; - - __asm__ __volatile__( - "# atomic interruptible down operation\n\t" - "xorl %0,%0\n\t" - LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "jns 2f\n\t" - "call __down_failed_trylock\n\t" - "2:\n" - :"=&a" (result), "=m" (sem->count) - :"D" (sem) - :"memory","cc"); - return result; -} - -/* - * Note! This is subtle. We jump to wake people up only if - * the semaphore was negative (== somebody was waiting on it). - * The default case (no contention) will result in NO - * jumps for both down() and up(). - */ -static inline void up(struct semaphore * sem) -{ - __asm__ __volatile__( - "# atomic up operation\n\t" - LOCK_PREFIX "incl %0\n\t" /* ++sem->count */ - "jg 1f\n\t" - "call __up_wakeup\n" - "1:" - :"=m" (sem->count) - :"D" (sem) - :"memory"); -} -#endif /* __KERNEL__ */ -#endif diff --git a/include/asm-x86/setup.h b/include/asm-x86/setup.h index 071e054abd8..fa6763af8d2 100644 --- a/include/asm-x86/setup.h +++ b/include/asm-x86/setup.h @@ -4,6 +4,10 @@ #define COMMAND_LINE_SIZE 2048 #ifndef __ASSEMBLY__ + +/* Interrupt control for vSMPowered x86_64 systems */ +void vsmp_init(void); + char *machine_specific_memory_setup(void); #ifndef CONFIG_PARAVIRT #define paravirt_post_allocator_init() do {} while (0) @@ -51,8 +55,8 @@ struct e820entry; char * __init machine_specific_memory_setup(void); char *memory_setup(void); -int __init copy_e820_map(struct e820entry * biosmap, int nr_map); -int __init sanitize_e820_map(struct e820entry * biosmap, char * pnr_map); +int __init copy_e820_map(struct e820entry *biosmap, int nr_map); +int __init sanitize_e820_map(struct e820entry *biosmap, char *pnr_map); void __init add_memory_region(unsigned long long start, unsigned long long size, int type); diff --git a/include/asm-x86/sigcontext.h b/include/asm-x86/sigcontext.h index d743947f4c7..2f9c884d2c0 100644 --- a/include/asm-x86/sigcontext.h +++ b/include/asm-x86/sigcontext.h @@ -79,7 +79,7 @@ struct sigcontext { unsigned long flags; unsigned long sp_at_signal; unsigned short ss, __ssh; - struct _fpstate __user * fpstate; + struct _fpstate __user *fpstate; unsigned long oldmask; unsigned long cr2; }; @@ -107,7 +107,7 @@ struct sigcontext { unsigned long eflags; unsigned long esp_at_signal; unsigned short ss, __ssh; - struct _fpstate __user * fpstate; + struct _fpstate __user *fpstate; unsigned long oldmask; unsigned long cr2; }; @@ -121,7 +121,8 @@ struct sigcontext { struct _fpstate { __u16 cwd; __u16 swd; - __u16 twd; /* Note this is not the same as the 32bit/x87/FSAVE twd */ + __u16 twd; /* Note this is not the same as the + 32bit/x87/FSAVE twd */ __u16 fop; __u64 rip; __u64 rdp; diff --git a/include/asm-x86/sigcontext32.h b/include/asm-x86/sigcontext32.h index 6ffab4fd593..57a9686fb49 100644 --- a/include/asm-x86/sigcontext32.h +++ b/include/asm-x86/sigcontext32.h @@ -26,7 +26,7 @@ struct _fpstate_ia32 { __u32 cw; __u32 sw; __u32 tag; /* not compatible to 64bit twd */ - __u32 ipoff; + __u32 ipoff; __u32 cssel; __u32 dataoff; __u32 datasel; @@ -39,7 +39,7 @@ struct _fpstate_ia32 { __u32 mxcsr; __u32 reserved; struct _fpxreg _fxsr_st[8]; - struct _xmmreg _xmm[8]; /* It's actually 16 */ + struct _xmmreg _xmm[8]; /* It's actually 16 */ __u32 padding[56]; }; diff --git a/include/asm-x86/signal.h b/include/asm-x86/signal.h index aee7eca585a..f15186d39c6 100644 --- a/include/asm-x86/signal.h +++ b/include/asm-x86/signal.h @@ -185,61 +185,61 @@ typedef struct sigaltstack { #define __HAVE_ARCH_SIG_BITOPS -#define sigaddset(set,sig) \ - (__builtin_constantp(sig) ? \ - __const_sigaddset((set),(sig)) : \ - __gen_sigaddset((set),(sig))) +#define sigaddset(set,sig) \ + (__builtin_constantp(sig) \ + ? __const_sigaddset((set), (sig)) \ + : __gen_sigaddset((set), (sig))) -static __inline__ void __gen_sigaddset(sigset_t *set, int _sig) +static inline void __gen_sigaddset(sigset_t *set, int _sig) { - __asm__("btsl %1,%0" : "+m"(*set) : "Ir"(_sig - 1) : "cc"); + asm("btsl %1,%0" : "+m"(*set) : "Ir"(_sig - 1) : "cc"); } -static __inline__ void __const_sigaddset(sigset_t *set, int _sig) +static inline void __const_sigaddset(sigset_t *set, int _sig) { unsigned long sig = _sig - 1; set->sig[sig / _NSIG_BPW] |= 1 << (sig % _NSIG_BPW); } -#define sigdelset(set,sig) \ - (__builtin_constant_p(sig) ? \ - __const_sigdelset((set),(sig)) : \ - __gen_sigdelset((set),(sig))) +#define sigdelset(set, sig) \ + (__builtin_constant_p(sig) \ + ? __const_sigdelset((set), (sig)) \ + : __gen_sigdelset((set), (sig))) -static __inline__ void __gen_sigdelset(sigset_t *set, int _sig) +static inline void __gen_sigdelset(sigset_t *set, int _sig) { - __asm__("btrl %1,%0" : "+m"(*set) : "Ir"(_sig - 1) : "cc"); + asm("btrl %1,%0" : "+m"(*set) : "Ir"(_sig - 1) : "cc"); } -static __inline__ void __const_sigdelset(sigset_t *set, int _sig) +static inline void __const_sigdelset(sigset_t *set, int _sig) { unsigned long sig = _sig - 1; set->sig[sig / _NSIG_BPW] &= ~(1 << (sig % _NSIG_BPW)); } -static __inline__ int __const_sigismember(sigset_t *set, int _sig) +static inline int __const_sigismember(sigset_t *set, int _sig) { unsigned long sig = _sig - 1; return 1 & (set->sig[sig / _NSIG_BPW] >> (sig % _NSIG_BPW)); } -static __inline__ int __gen_sigismember(sigset_t *set, int _sig) +static inline int __gen_sigismember(sigset_t *set, int _sig) { int ret; - __asm__("btl %2,%1\n\tsbbl %0,%0" - : "=r"(ret) : "m"(*set), "Ir"(_sig-1) : "cc"); + asm("btl %2,%1\n\tsbbl %0,%0" + : "=r"(ret) : "m"(*set), "Ir"(_sig-1) : "cc"); return ret; } -#define sigismember(set,sig) \ - (__builtin_constant_p(sig) ? \ - __const_sigismember((set),(sig)) : \ - __gen_sigismember((set),(sig))) +#define sigismember(set, sig) \ + (__builtin_constant_p(sig) \ + ? __const_sigismember((set), (sig)) \ + : __gen_sigismember((set), (sig))) -static __inline__ int sigfindinword(unsigned long word) +static inline int sigfindinword(unsigned long word) { - __asm__("bsfl %1,%0" : "=r"(word) : "rm"(word) : "cc"); + asm("bsfl %1,%0" : "=r"(word) : "rm"(word) : "cc"); return word; } diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h index f2e8319a6b0..1ebaa5cd311 100644 --- a/include/asm-x86/smp.h +++ b/include/asm-x86/smp.h @@ -1,5 +1,208 @@ -#ifdef CONFIG_X86_32 -# include "smp_32.h" +#ifndef _ASM_X86_SMP_H_ +#define _ASM_X86_SMP_H_ +#ifndef __ASSEMBLY__ +#include <linux/cpumask.h> +#include <linux/init.h> +#include <asm/percpu.h> + +/* + * We need the APIC definitions automatically as part of 'smp.h' + */ +#ifdef CONFIG_X86_LOCAL_APIC +# include <asm/mpspec.h> +# include <asm/apic.h> +# ifdef CONFIG_X86_IO_APIC +# include <asm/io_apic.h> +# endif +#endif +#include <asm/pda.h> +#include <asm/thread_info.h> + +extern cpumask_t cpu_callout_map; +extern cpumask_t cpu_initialized; +extern cpumask_t cpu_callin_map; + +extern void (*mtrr_hook)(void); +extern void zap_low_mappings(void); + +extern int smp_num_siblings; +extern unsigned int num_processors; +extern cpumask_t cpu_initialized; + +#ifdef CONFIG_SMP +extern u16 x86_cpu_to_apicid_init[]; +extern u16 x86_bios_cpu_apicid_init[]; +extern void *x86_cpu_to_apicid_early_ptr; +extern void *x86_bios_cpu_apicid_early_ptr; #else -# include "smp_64.h" +#define x86_cpu_to_apicid_early_ptr NULL +#define x86_bios_cpu_apicid_early_ptr NULL +#endif + +DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); +DECLARE_PER_CPU(cpumask_t, cpu_core_map); +DECLARE_PER_CPU(u16, cpu_llc_id); +DECLARE_PER_CPU(u16, x86_cpu_to_apicid); +DECLARE_PER_CPU(u16, x86_bios_cpu_apicid); + +/* Static state in head.S used to set up a CPU */ +extern struct { + void *sp; + unsigned short ss; +} stack_start; + +struct smp_ops { + void (*smp_prepare_boot_cpu)(void); + void (*smp_prepare_cpus)(unsigned max_cpus); + int (*cpu_up)(unsigned cpu); + void (*smp_cpus_done)(unsigned max_cpus); + + void (*smp_send_stop)(void); + void (*smp_send_reschedule)(int cpu); + int (*smp_call_function_mask)(cpumask_t mask, + void (*func)(void *info), void *info, + int wait); +}; + +/* Globals due to paravirt */ +extern void set_cpu_sibling_map(int cpu); + +#ifdef CONFIG_SMP +#ifndef CONFIG_PARAVIRT +#define startup_ipi_hook(phys_apicid, start_eip, start_esp) do { } while (0) +#endif +extern struct smp_ops smp_ops; + +static inline void smp_send_stop(void) +{ + smp_ops.smp_send_stop(); +} + +static inline void smp_prepare_boot_cpu(void) +{ + smp_ops.smp_prepare_boot_cpu(); +} + +static inline void smp_prepare_cpus(unsigned int max_cpus) +{ + smp_ops.smp_prepare_cpus(max_cpus); +} + +static inline void smp_cpus_done(unsigned int max_cpus) +{ + smp_ops.smp_cpus_done(max_cpus); +} + +static inline int __cpu_up(unsigned int cpu) +{ + return smp_ops.cpu_up(cpu); +} + +static inline void smp_send_reschedule(int cpu) +{ + smp_ops.smp_send_reschedule(cpu); +} + +static inline int smp_call_function_mask(cpumask_t mask, + void (*func) (void *info), void *info, + int wait) +{ + return smp_ops.smp_call_function_mask(mask, func, info, wait); +} + +void native_smp_prepare_boot_cpu(void); +void native_smp_prepare_cpus(unsigned int max_cpus); +void native_smp_cpus_done(unsigned int max_cpus); +int native_cpu_up(unsigned int cpunum); + +extern int __cpu_disable(void); +extern void __cpu_die(unsigned int cpu); + +extern void prefill_possible_map(void); + +void smp_store_cpu_info(int id); +#define cpu_physical_id(cpu) per_cpu(x86_cpu_to_apicid, cpu) + +/* We don't mark CPUs online until __cpu_up(), so we need another measure */ +static inline int num_booting_cpus(void) +{ + return cpus_weight(cpu_callout_map); +} +#endif /* CONFIG_SMP */ + +extern unsigned disabled_cpus __cpuinitdata; + +#ifdef CONFIG_X86_32_SMP +/* + * This function is needed by all SMP systems. It must _always_ be valid + * from the initial startup. We map APIC_BASE very early in page_setup(), + * so this is correct in the x86 case. + */ +DECLARE_PER_CPU(int, cpu_number); +#define raw_smp_processor_id() (x86_read_percpu(cpu_number)) +extern int safe_smp_processor_id(void); + +#elif defined(CONFIG_X86_64_SMP) +#define raw_smp_processor_id() read_pda(cpunumber) + +#define stack_smp_processor_id() \ +({ \ + struct thread_info *ti; \ + __asm__("andq %%rsp,%0; ":"=r" (ti) : "0" (CURRENT_MASK)); \ + ti->cpu; \ +}) +#define safe_smp_processor_id() smp_processor_id() + +#else /* !CONFIG_X86_32_SMP && !CONFIG_X86_64_SMP */ +#define cpu_physical_id(cpu) boot_cpu_physical_apicid +#define safe_smp_processor_id() 0 +#define stack_smp_processor_id() 0 +#endif + +#ifdef CONFIG_X86_LOCAL_APIC + +static inline int logical_smp_processor_id(void) +{ + /* we don't want to mark this access volatile - bad code generation */ + return GET_APIC_LOGICAL_ID(*(u32 *)(APIC_BASE + APIC_LDR)); +} + +#ifndef CONFIG_X86_64 +static inline unsigned int read_apic_id(void) +{ + return *(u32 *)(APIC_BASE + APIC_ID); +} +#else +extern unsigned int read_apic_id(void); +#endif + + +# ifdef APIC_DEFINITION +extern int hard_smp_processor_id(void); +# else +# include <mach_apicdef.h> +static inline int hard_smp_processor_id(void) +{ + /* we don't want to mark this access volatile - bad code generation */ + return GET_APIC_ID(read_apic_id()); +} +# endif /* APIC_DEFINITION */ + +#else /* CONFIG_X86_LOCAL_APIC */ + +# ifndef CONFIG_SMP +# define hard_smp_processor_id() 0 +# endif + +#endif /* CONFIG_X86_LOCAL_APIC */ + +#ifdef CONFIG_HOTPLUG_CPU +extern void cpu_exit_clear(void); +extern void cpu_uninit(void); +#endif + +extern void smp_alloc_memory(void); +extern void lock_ipi_call_lock(void); +extern void unlock_ipi_call_lock(void); +#endif /* __ASSEMBLY__ */ #endif diff --git a/include/asm-x86/smp_32.h b/include/asm-x86/smp_32.h deleted file mode 100644 index 56152e31228..00000000000 --- a/include/asm-x86/smp_32.h +++ /dev/null @@ -1,165 +0,0 @@ -#ifndef __ASM_SMP_H -#define __ASM_SMP_H - -#ifndef __ASSEMBLY__ -#include <linux/cpumask.h> -#include <linux/init.h> - -/* - * We need the APIC definitions automatically as part of 'smp.h' - */ -#ifdef CONFIG_X86_LOCAL_APIC -# include <asm/mpspec.h> -# include <asm/apic.h> -# ifdef CONFIG_X86_IO_APIC -# include <asm/io_apic.h> -# endif -#endif - -extern cpumask_t cpu_callout_map; -extern cpumask_t cpu_callin_map; - -extern int smp_num_siblings; -extern unsigned int num_processors; - -extern void smp_alloc_memory(void); -extern void lock_ipi_call_lock(void); -extern void unlock_ipi_call_lock(void); - -extern void (*mtrr_hook) (void); -extern void zap_low_mappings (void); - -extern u8 __initdata x86_cpu_to_apicid_init[]; -extern void *x86_cpu_to_apicid_early_ptr; - -DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); -DECLARE_PER_CPU(cpumask_t, cpu_core_map); -DECLARE_PER_CPU(u8, cpu_llc_id); -DECLARE_PER_CPU(u8, x86_cpu_to_apicid); - -#ifdef CONFIG_HOTPLUG_CPU -extern void cpu_exit_clear(void); -extern void cpu_uninit(void); -extern void remove_siblinginfo(int cpu); -#endif - -/* Globals due to paravirt */ -extern void set_cpu_sibling_map(int cpu); - -struct smp_ops -{ - void (*smp_prepare_boot_cpu)(void); - void (*smp_prepare_cpus)(unsigned max_cpus); - int (*cpu_up)(unsigned cpu); - void (*smp_cpus_done)(unsigned max_cpus); - - void (*smp_send_stop)(void); - void (*smp_send_reschedule)(int cpu); - int (*smp_call_function_mask)(cpumask_t mask, - void (*func)(void *info), void *info, - int wait); -}; - -#ifdef CONFIG_SMP -extern struct smp_ops smp_ops; - -static inline void smp_prepare_boot_cpu(void) -{ - smp_ops.smp_prepare_boot_cpu(); -} -static inline void smp_prepare_cpus(unsigned int max_cpus) -{ - smp_ops.smp_prepare_cpus(max_cpus); -} -static inline int __cpu_up(unsigned int cpu) -{ - return smp_ops.cpu_up(cpu); -} -static inline void smp_cpus_done(unsigned int max_cpus) -{ - smp_ops.smp_cpus_done(max_cpus); -} - -static inline void smp_send_stop(void) -{ - smp_ops.smp_send_stop(); -} -static inline void smp_send_reschedule(int cpu) -{ - smp_ops.smp_send_reschedule(cpu); -} -static inline int smp_call_function_mask(cpumask_t mask, - void (*func) (void *info), void *info, - int wait) -{ - return smp_ops.smp_call_function_mask(mask, func, info, wait); -} - -void native_smp_prepare_boot_cpu(void); -void native_smp_prepare_cpus(unsigned int max_cpus); -int native_cpu_up(unsigned int cpunum); -void native_smp_cpus_done(unsigned int max_cpus); - -#ifndef CONFIG_PARAVIRT -#define startup_ipi_hook(phys_apicid, start_eip, start_esp) do { } while (0) -#endif - -extern int __cpu_disable(void); -extern void __cpu_die(unsigned int cpu); - -/* - * This function is needed by all SMP systems. It must _always_ be valid - * from the initial startup. We map APIC_BASE very early in page_setup(), - * so this is correct in the x86 case. - */ -DECLARE_PER_CPU(int, cpu_number); -#define raw_smp_processor_id() (x86_read_percpu(cpu_number)) - -#define cpu_physical_id(cpu) per_cpu(x86_cpu_to_apicid, cpu) - -extern int safe_smp_processor_id(void); - -void __cpuinit smp_store_cpu_info(int id); - -/* We don't mark CPUs online until __cpu_up(), so we need another measure */ -static inline int num_booting_cpus(void) -{ - return cpus_weight(cpu_callout_map); -} - -#else /* CONFIG_SMP */ - -#define safe_smp_processor_id() 0 -#define cpu_physical_id(cpu) boot_cpu_physical_apicid - -#endif /* !CONFIG_SMP */ - -#ifdef CONFIG_X86_LOCAL_APIC - -static __inline int logical_smp_processor_id(void) -{ - /* we don't want to mark this access volatile - bad code generation */ - return GET_APIC_LOGICAL_ID(*(u32 *)(APIC_BASE + APIC_LDR)); -} - -# ifdef APIC_DEFINITION -extern int hard_smp_processor_id(void); -# else -# include <mach_apicdef.h> -static inline int hard_smp_processor_id(void) -{ - /* we don't want to mark this access volatile - bad code generation */ - return GET_APIC_ID(*(u32 *)(APIC_BASE + APIC_ID)); -} -# endif /* APIC_DEFINITION */ - -#else /* CONFIG_X86_LOCAL_APIC */ - -# ifndef CONFIG_SMP -# define hard_smp_processor_id() 0 -# endif - -#endif /* CONFIG_X86_LOCAL_APIC */ - -#endif /* !ASSEMBLY */ -#endif diff --git a/include/asm-x86/smp_64.h b/include/asm-x86/smp_64.h deleted file mode 100644 index e0a75519ad2..00000000000 --- a/include/asm-x86/smp_64.h +++ /dev/null @@ -1,101 +0,0 @@ -#ifndef __ASM_SMP_H -#define __ASM_SMP_H - -#include <linux/cpumask.h> -#include <linux/init.h> - -/* - * We need the APIC definitions automatically as part of 'smp.h' - */ -#include <asm/apic.h> -#include <asm/io_apic.h> -#include <asm/mpspec.h> -#include <asm/pda.h> -#include <asm/thread_info.h> - -extern cpumask_t cpu_callout_map; -extern cpumask_t cpu_initialized; - -extern int smp_num_siblings; -extern unsigned int num_processors; - -extern void smp_alloc_memory(void); -extern void lock_ipi_call_lock(void); -extern void unlock_ipi_call_lock(void); - -extern int smp_call_function_mask(cpumask_t mask, void (*func)(void *), - void *info, int wait); - -extern u16 __initdata x86_cpu_to_apicid_init[]; -extern u16 __initdata x86_bios_cpu_apicid_init[]; -extern void *x86_cpu_to_apicid_early_ptr; -extern void *x86_bios_cpu_apicid_early_ptr; - -DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); -DECLARE_PER_CPU(cpumask_t, cpu_core_map); -DECLARE_PER_CPU(u16, cpu_llc_id); -DECLARE_PER_CPU(u16, x86_cpu_to_apicid); -DECLARE_PER_CPU(u16, x86_bios_cpu_apicid); - -static inline int cpu_present_to_apicid(int mps_cpu) -{ - if (cpu_present(mps_cpu)) - return (int)per_cpu(x86_bios_cpu_apicid, mps_cpu); - else - return BAD_APICID; -} - -#ifdef CONFIG_SMP - -#define SMP_TRAMPOLINE_BASE 0x6000 - -extern int __cpu_disable(void); -extern void __cpu_die(unsigned int cpu); -extern void prefill_possible_map(void); -extern unsigned __cpuinitdata disabled_cpus; - -#define raw_smp_processor_id() read_pda(cpunumber) -#define cpu_physical_id(cpu) per_cpu(x86_cpu_to_apicid, cpu) - -#define stack_smp_processor_id() \ - ({ \ - struct thread_info *ti; \ - __asm__("andq %%rsp,%0; ":"=r" (ti) : "0" (CURRENT_MASK)); \ - ti->cpu; \ -}) - -/* - * On x86 all CPUs are mapped 1:1 to the APIC space. This simplifies - * scheduling and IPI sending and compresses data structures. - */ -static inline int num_booting_cpus(void) -{ - return cpus_weight(cpu_callout_map); -} - -extern void smp_send_reschedule(int cpu); - -#else /* CONFIG_SMP */ - -extern unsigned int boot_cpu_id; -#define cpu_physical_id(cpu) boot_cpu_id -#define stack_smp_processor_id() 0 - -#endif /* !CONFIG_SMP */ - -#define safe_smp_processor_id() smp_processor_id() - -static __inline int logical_smp_processor_id(void) -{ - /* we don't want to mark this access volatile - bad code generation */ - return GET_APIC_LOGICAL_ID(*(u32 *)(APIC_BASE + APIC_LDR)); -} - -static inline int hard_smp_processor_id(void) -{ - /* we don't want to mark this access volatile - bad code generation */ - return GET_APIC_ID(*(u32 *)(APIC_BASE + APIC_ID)); -} - -#endif - diff --git a/include/asm-x86/sparsemem.h b/include/asm-x86/sparsemem.h index fa58cd55411..9bd48b0a534 100644 --- a/include/asm-x86/sparsemem.h +++ b/include/asm-x86/sparsemem.h @@ -16,7 +16,7 @@ #ifdef CONFIG_X86_32 # ifdef CONFIG_X86_PAE -# define SECTION_SIZE_BITS 30 +# define SECTION_SIZE_BITS 29 # define MAX_PHYSADDR_BITS 36 # define MAX_PHYSMEM_BITS 36 # else @@ -26,8 +26,8 @@ # endif #else /* CONFIG_X86_32 */ # define SECTION_SIZE_BITS 27 /* matt - 128 is convenient right now */ -# define MAX_PHYSADDR_BITS 40 -# define MAX_PHYSMEM_BITS 40 +# define MAX_PHYSADDR_BITS 44 +# define MAX_PHYSMEM_BITS 44 #endif #endif /* CONFIG_SPARSEMEM */ diff --git a/include/asm-x86/spinlock.h b/include/asm-x86/spinlock.h index 23804c1890f..bc6376f1bc5 100644 --- a/include/asm-x86/spinlock.h +++ b/include/asm-x86/spinlock.h @@ -78,11 +78,11 @@ static inline int __raw_spin_is_contended(raw_spinlock_t *lock) return (((tmp >> 8) & 0xff) - (tmp & 0xff)) > 1; } -static inline void __raw_spin_lock(raw_spinlock_t *lock) +static __always_inline void __raw_spin_lock(raw_spinlock_t *lock) { short inc = 0x0100; - __asm__ __volatile__ ( + asm volatile ( LOCK_PREFIX "xaddw %w0, %1\n" "1:\t" "cmpb %h0, %b0\n\t" @@ -92,42 +92,40 @@ static inline void __raw_spin_lock(raw_spinlock_t *lock) /* don't need lfence here, because loads are in-order */ "jmp 1b\n" "2:" - :"+Q" (inc), "+m" (lock->slock) + : "+Q" (inc), "+m" (lock->slock) : - :"memory", "cc"); + : "memory", "cc"); } #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock) -static inline int __raw_spin_trylock(raw_spinlock_t *lock) +static __always_inline int __raw_spin_trylock(raw_spinlock_t *lock) { int tmp; short new; - asm volatile( - "movw %2,%w0\n\t" - "cmpb %h0,%b0\n\t" - "jne 1f\n\t" - "movw %w0,%w1\n\t" - "incb %h1\n\t" - "lock ; cmpxchgw %w1,%2\n\t" - "1:" - "sete %b1\n\t" - "movzbl %b1,%0\n\t" - :"=&a" (tmp), "=Q" (new), "+m" (lock->slock) - : - : "memory", "cc"); + asm volatile("movw %2,%w0\n\t" + "cmpb %h0,%b0\n\t" + "jne 1f\n\t" + "movw %w0,%w1\n\t" + "incb %h1\n\t" + "lock ; cmpxchgw %w1,%2\n\t" + "1:" + "sete %b1\n\t" + "movzbl %b1,%0\n\t" + : "=&a" (tmp), "=Q" (new), "+m" (lock->slock) + : + : "memory", "cc"); return tmp; } -static inline void __raw_spin_unlock(raw_spinlock_t *lock) +static __always_inline void __raw_spin_unlock(raw_spinlock_t *lock) { - __asm__ __volatile__( - UNLOCK_LOCK_PREFIX "incb %0" - :"+m" (lock->slock) - : - :"memory", "cc"); + asm volatile(UNLOCK_LOCK_PREFIX "incb %0" + : "+m" (lock->slock) + : + : "memory", "cc"); } #else static inline int __raw_spin_is_locked(raw_spinlock_t *lock) @@ -144,60 +142,57 @@ static inline int __raw_spin_is_contended(raw_spinlock_t *lock) return (((tmp >> 16) & 0xffff) - (tmp & 0xffff)) > 1; } -static inline void __raw_spin_lock(raw_spinlock_t *lock) +static __always_inline void __raw_spin_lock(raw_spinlock_t *lock) { int inc = 0x00010000; int tmp; - __asm__ __volatile__ ( - "lock ; xaddl %0, %1\n" - "movzwl %w0, %2\n\t" - "shrl $16, %0\n\t" - "1:\t" - "cmpl %0, %2\n\t" - "je 2f\n\t" - "rep ; nop\n\t" - "movzwl %1, %2\n\t" - /* don't need lfence here, because loads are in-order */ - "jmp 1b\n" - "2:" - :"+Q" (inc), "+m" (lock->slock), "=r" (tmp) - : - :"memory", "cc"); + asm volatile("lock ; xaddl %0, %1\n" + "movzwl %w0, %2\n\t" + "shrl $16, %0\n\t" + "1:\t" + "cmpl %0, %2\n\t" + "je 2f\n\t" + "rep ; nop\n\t" + "movzwl %1, %2\n\t" + /* don't need lfence here, because loads are in-order */ + "jmp 1b\n" + "2:" + : "+Q" (inc), "+m" (lock->slock), "=r" (tmp) + : + : "memory", "cc"); } #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock) -static inline int __raw_spin_trylock(raw_spinlock_t *lock) +static __always_inline int __raw_spin_trylock(raw_spinlock_t *lock) { int tmp; int new; - asm volatile( - "movl %2,%0\n\t" - "movl %0,%1\n\t" - "roll $16, %0\n\t" - "cmpl %0,%1\n\t" - "jne 1f\n\t" - "addl $0x00010000, %1\n\t" - "lock ; cmpxchgl %1,%2\n\t" - "1:" - "sete %b1\n\t" - "movzbl %b1,%0\n\t" - :"=&a" (tmp), "=r" (new), "+m" (lock->slock) - : - : "memory", "cc"); + asm volatile("movl %2,%0\n\t" + "movl %0,%1\n\t" + "roll $16, %0\n\t" + "cmpl %0,%1\n\t" + "jne 1f\n\t" + "addl $0x00010000, %1\n\t" + "lock ; cmpxchgl %1,%2\n\t" + "1:" + "sete %b1\n\t" + "movzbl %b1,%0\n\t" + : "=&a" (tmp), "=r" (new), "+m" (lock->slock) + : + : "memory", "cc"); return tmp; } -static inline void __raw_spin_unlock(raw_spinlock_t *lock) +static __always_inline void __raw_spin_unlock(raw_spinlock_t *lock) { - __asm__ __volatile__( - UNLOCK_LOCK_PREFIX "incw %0" - :"+m" (lock->slock) - : - :"memory", "cc"); + asm volatile(UNLOCK_LOCK_PREFIX "incw %0" + : "+m" (lock->slock) + : + : "memory", "cc"); } #endif diff --git a/include/asm-x86/srat.h b/include/asm-x86/srat.h index 165ab4bdc02..f4bba131d06 100644 --- a/include/asm-x86/srat.h +++ b/include/asm-x86/srat.h @@ -1,5 +1,5 @@ /* - * Some of the code in this file has been gleaned from the 64 bit + * Some of the code in this file has been gleaned from the 64 bit * discontigmem support code base. * * Copyright (C) 2002, IBM Corp. diff --git a/include/asm-x86/string_32.h b/include/asm-x86/string_32.h index c5d13a86dea..b49369ad9a6 100644 --- a/include/asm-x86/string_32.h +++ b/include/asm-x86/string_32.h @@ -3,7 +3,7 @@ #ifdef __KERNEL__ -/* Let gcc decide wether to inline or use the out of line functions */ +/* Let gcc decide whether to inline or use the out of line functions */ #define __HAVE_ARCH_STRCPY extern char *strcpy(char *dest, const char *src); diff --git a/include/asm-x86/string_64.h b/include/asm-x86/string_64.h index e583da7918f..52b5ab38339 100644 --- a/include/asm-x86/string_64.h +++ b/include/asm-x86/string_64.h @@ -3,26 +3,24 @@ #ifdef __KERNEL__ -/* Written 2002 by Andi Kleen */ +/* Written 2002 by Andi Kleen */ -/* Only used for special circumstances. Stolen from i386/string.h */ -static __always_inline void * -__inline_memcpy(void * to, const void * from, size_t n) +/* Only used for special circumstances. Stolen from i386/string.h */ +static __always_inline void *__inline_memcpy(void *to, const void *from, size_t n) { -unsigned long d0, d1, d2; -__asm__ __volatile__( - "rep ; movsl\n\t" - "testb $2,%b4\n\t" - "je 1f\n\t" - "movsw\n" - "1:\ttestb $1,%b4\n\t" - "je 2f\n\t" - "movsb\n" - "2:" - : "=&c" (d0), "=&D" (d1), "=&S" (d2) - :"0" (n/4), "q" (n),"1" ((long) to),"2" ((long) from) - : "memory"); -return (to); + unsigned long d0, d1, d2; + asm volatile("rep ; movsl\n\t" + "testb $2,%b4\n\t" + "je 1f\n\t" + "movsw\n" + "1:\ttestb $1,%b4\n\t" + "je 2f\n\t" + "movsb\n" + "2:" + : "=&c" (d0), "=&D" (d1), "=&S" (d2) + : "0" (n / 4), "q" (n), "1" ((long)to), "2" ((long)from) + : "memory"); + return to; } /* Even with __builtin_ the compiler may decide to use the out of line @@ -32,28 +30,30 @@ return (to); #if (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) || __GNUC__ > 4 extern void *memcpy(void *to, const void *from, size_t len); #else -extern void *__memcpy(void *to, const void *from, size_t len); -#define memcpy(dst,src,len) \ - ({ size_t __len = (len); \ - void *__ret; \ - if (__builtin_constant_p(len) && __len >= 64) \ - __ret = __memcpy((dst),(src),__len); \ - else \ - __ret = __builtin_memcpy((dst),(src),__len); \ - __ret; }) +extern void *__memcpy(void *to, const void *from, size_t len); +#define memcpy(dst, src, len) \ +({ \ + size_t __len = (len); \ + void *__ret; \ + if (__builtin_constant_p(len) && __len >= 64) \ + __ret = __memcpy((dst), (src), __len); \ + else \ + __ret = __builtin_memcpy((dst), (src), __len); \ + __ret; \ +}) #endif #define __HAVE_ARCH_MEMSET void *memset(void *s, int c, size_t n); #define __HAVE_ARCH_MEMMOVE -void * memmove(void * dest,const void *src,size_t count); +void *memmove(void *dest, const void *src, size_t count); -int memcmp(const void * cs,const void * ct,size_t count); -size_t strlen(const char * s); -char *strcpy(char * dest,const char *src); -char *strcat(char * dest, const char * src); -int strcmp(const char * cs,const char * ct); +int memcmp(const void *cs, const void *ct, size_t count); +size_t strlen(const char *s); +char *strcpy(char *dest, const char *src); +char *strcat(char *dest, const char *src); +int strcmp(const char *cs, const char *ct); #endif /* __KERNEL__ */ diff --git a/include/asm-x86/suspend_32.h b/include/asm-x86/suspend_32.h index 1bbda3ad779..24e1c080aa8 100644 --- a/include/asm-x86/suspend_32.h +++ b/include/asm-x86/suspend_32.h @@ -10,7 +10,7 @@ static inline int arch_prepare_suspend(void) { return 0; } /* image of the saved processor state */ struct saved_context { - u16 es, fs, gs, ss; + u16 es, fs, gs, ss; unsigned long cr0, cr2, cr3, cr4; struct desc_ptr gdt; struct desc_ptr idt; @@ -32,11 +32,11 @@ extern unsigned long saved_edi; static inline void acpi_save_register_state(unsigned long return_point) { saved_eip = return_point; - asm volatile ("movl %%esp,%0" : "=m" (saved_esp)); - asm volatile ("movl %%ebp,%0" : "=m" (saved_ebp)); - asm volatile ("movl %%ebx,%0" : "=m" (saved_ebx)); - asm volatile ("movl %%edi,%0" : "=m" (saved_edi)); - asm volatile ("movl %%esi,%0" : "=m" (saved_esi)); + asm volatile("movl %%esp,%0" : "=m" (saved_esp)); + asm volatile("movl %%ebp,%0" : "=m" (saved_ebp)); + asm volatile("movl %%ebx,%0" : "=m" (saved_ebx)); + asm volatile("movl %%edi,%0" : "=m" (saved_edi)); + asm volatile("movl %%esi,%0" : "=m" (saved_esi)); } #define acpi_restore_register_state() do {} while (0) diff --git a/include/asm-x86/suspend_64.h b/include/asm-x86/suspend_64.h index 2eb92cb81a0..dc3262b4307 100644 --- a/include/asm-x86/suspend_64.h +++ b/include/asm-x86/suspend_64.h @@ -9,8 +9,7 @@ #include <asm/desc.h> #include <asm/i387.h> -static inline int -arch_prepare_suspend(void) +static inline int arch_prepare_suspend(void) { return 0; } @@ -25,7 +24,7 @@ arch_prepare_suspend(void) */ struct saved_context { struct pt_regs regs; - u16 ds, es, fs, gs, ss; + u16 ds, es, fs, gs, ss; unsigned long gs_base, gs_kernel_base, fs_base; unsigned long cr0, cr2, cr3, cr4, cr8; unsigned long efer; diff --git a/include/asm-x86/swiotlb.h b/include/asm-x86/swiotlb.h index f9c589539a8..f5d9e74b1e4 100644 --- a/include/asm-x86/swiotlb.h +++ b/include/asm-x86/swiotlb.h @@ -8,15 +8,15 @@ extern dma_addr_t swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir); extern void *swiotlb_alloc_coherent(struct device *hwdev, size_t size, - dma_addr_t *dma_handle, gfp_t flags); + dma_addr_t *dma_handle, gfp_t flags); extern void swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, - size_t size, int dir); + size_t size, int dir); extern void swiotlb_sync_single_for_cpu(struct device *hwdev, - dma_addr_t dev_addr, - size_t size, int dir); + dma_addr_t dev_addr, + size_t size, int dir); extern void swiotlb_sync_single_for_device(struct device *hwdev, - dma_addr_t dev_addr, - size_t size, int dir); + dma_addr_t dev_addr, + size_t size, int dir); extern void swiotlb_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dev_addr, unsigned long offset, @@ -26,18 +26,18 @@ extern void swiotlb_sync_single_range_for_device(struct device *hwdev, unsigned long offset, size_t size, int dir); extern void swiotlb_sync_sg_for_cpu(struct device *hwdev, - struct scatterlist *sg, int nelems, - int dir); + struct scatterlist *sg, int nelems, + int dir); extern void swiotlb_sync_sg_for_device(struct device *hwdev, - struct scatterlist *sg, int nelems, - int dir); + struct scatterlist *sg, int nelems, + int dir); extern int swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, - int nents, int direction); + int nents, int direction); extern void swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, - int nents, int direction); + int nents, int direction); extern int swiotlb_dma_mapping_error(dma_addr_t dma_addr); -extern void swiotlb_free_coherent (struct device *hwdev, size_t size, - void *vaddr, dma_addr_t dma_handle); +extern void swiotlb_free_coherent(struct device *hwdev, size_t size, + void *vaddr, dma_addr_t dma_handle); extern int swiotlb_dma_supported(struct device *hwdev, u64 mask); extern void swiotlb_init(void); diff --git a/include/asm-x86/sync_bitops.h b/include/asm-x86/sync_bitops.h index 6b775c90566..b47a1d0b8a8 100644 --- a/include/asm-x86/sync_bitops.h +++ b/include/asm-x86/sync_bitops.h @@ -13,7 +13,7 @@ * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1). */ -#define ADDR (*(volatile long *) addr) +#define ADDR (*(volatile long *)addr) /** * sync_set_bit - Atomically set a bit in memory @@ -26,12 +26,12 @@ * Note that @nr may be almost arbitrarily large; this function is not * restricted to acting on a single-word quantity. */ -static inline void sync_set_bit(int nr, volatile unsigned long * addr) +static inline void sync_set_bit(int nr, volatile unsigned long *addr) { - __asm__ __volatile__("lock; btsl %1,%0" - :"+m" (ADDR) - :"Ir" (nr) - : "memory"); + asm volatile("lock; btsl %1,%0" + : "+m" (ADDR) + : "Ir" (nr) + : "memory"); } /** @@ -44,12 +44,12 @@ static inline void sync_set_bit(int nr, volatile unsigned long * addr) * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() * in order to ensure changes are visible on other processors. */ -static inline void sync_clear_bit(int nr, volatile unsigned long * addr) +static inline void sync_clear_bit(int nr, volatile unsigned long *addr) { - __asm__ __volatile__("lock; btrl %1,%0" - :"+m" (ADDR) - :"Ir" (nr) - : "memory"); + asm volatile("lock; btrl %1,%0" + : "+m" (ADDR) + : "Ir" (nr) + : "memory"); } /** @@ -61,12 +61,12 @@ static inline void sync_clear_bit(int nr, volatile unsigned long * addr) * Note that @nr may be almost arbitrarily large; this function is not * restricted to acting on a single-word quantity. */ -static inline void sync_change_bit(int nr, volatile unsigned long * addr) +static inline void sync_change_bit(int nr, volatile unsigned long *addr) { - __asm__ __volatile__("lock; btcl %1,%0" - :"+m" (ADDR) - :"Ir" (nr) - : "memory"); + asm volatile("lock; btcl %1,%0" + : "+m" (ADDR) + : "Ir" (nr) + : "memory"); } /** @@ -77,13 +77,13 @@ static inline void sync_change_bit(int nr, volatile unsigned long * addr) * This operation is atomic and cannot be reordered. * It also implies a memory barrier. */ -static inline int sync_test_and_set_bit(int nr, volatile unsigned long * addr) +static inline int sync_test_and_set_bit(int nr, volatile unsigned long *addr) { int oldbit; - __asm__ __volatile__("lock; btsl %2,%1\n\tsbbl %0,%0" - :"=r" (oldbit),"+m" (ADDR) - :"Ir" (nr) : "memory"); + asm volatile("lock; btsl %2,%1\n\tsbbl %0,%0" + : "=r" (oldbit), "+m" (ADDR) + : "Ir" (nr) : "memory"); return oldbit; } @@ -95,13 +95,13 @@ static inline int sync_test_and_set_bit(int nr, volatile unsigned long * addr) * This operation is atomic and cannot be reordered. * It also implies a memory barrier. */ -static inline int sync_test_and_clear_bit(int nr, volatile unsigned long * addr) +static inline int sync_test_and_clear_bit(int nr, volatile unsigned long *addr) { int oldbit; - __asm__ __volatile__("lock; btrl %2,%1\n\tsbbl %0,%0" - :"=r" (oldbit),"+m" (ADDR) - :"Ir" (nr) : "memory"); + asm volatile("lock; btrl %2,%1\n\tsbbl %0,%0" + : "=r" (oldbit), "+m" (ADDR) + : "Ir" (nr) : "memory"); return oldbit; } @@ -113,36 +113,17 @@ static inline int sync_test_and_clear_bit(int nr, volatile unsigned long * addr) * This operation is atomic and cannot be reordered. * It also implies a memory barrier. */ -static inline int sync_test_and_change_bit(int nr, volatile unsigned long* addr) +static inline int sync_test_and_change_bit(int nr, volatile unsigned long *addr) { int oldbit; - __asm__ __volatile__("lock; btcl %2,%1\n\tsbbl %0,%0" - :"=r" (oldbit),"+m" (ADDR) - :"Ir" (nr) : "memory"); + asm volatile("lock; btcl %2,%1\n\tsbbl %0,%0" + : "=r" (oldbit), "+m" (ADDR) + : "Ir" (nr) : "memory"); return oldbit; } -static __always_inline int sync_constant_test_bit(int nr, const volatile unsigned long *addr) -{ - return ((1UL << (nr & 31)) & - (((const volatile unsigned int *)addr)[nr >> 5])) != 0; -} - -static inline int sync_var_test_bit(int nr, const volatile unsigned long * addr) -{ - int oldbit; - - __asm__ __volatile__("btl %2,%1\n\tsbbl %0,%0" - :"=r" (oldbit) - :"m" (ADDR),"Ir" (nr)); - return oldbit; -} - -#define sync_test_bit(nr,addr) \ - (__builtin_constant_p(nr) ? \ - sync_constant_test_bit((nr),(addr)) : \ - sync_var_test_bit((nr),(addr))) +#define sync_test_bit(nr, addr) test_bit(nr, addr) #undef ADDR diff --git a/include/asm-x86/system.h b/include/asm-x86/system.h index 9cff02ffe6c..a2f04cd79b2 100644 --- a/include/asm-x86/system.h +++ b/include/asm-x86/system.h @@ -27,22 +27,44 @@ struct task_struct *__switch_to(struct task_struct *prev, * Saving eflags is important. It switches not only IOPL between tasks, * it also protects other tasks from NT leaking through sysenter etc. */ -#define switch_to(prev, next, last) do { \ - unsigned long esi, edi; \ - asm volatile("pushfl\n\t" /* Save flags */ \ - "pushl %%ebp\n\t" \ - "movl %%esp,%0\n\t" /* save ESP */ \ - "movl %5,%%esp\n\t" /* restore ESP */ \ - "movl $1f,%1\n\t" /* save EIP */ \ - "pushl %6\n\t" /* restore EIP */ \ - "jmp __switch_to\n" \ +#define switch_to(prev, next, last) \ +do { \ + /* \ + * Context-switching clobbers all registers, so we clobber \ + * them explicitly, via unused output variables. \ + * (EAX and EBP is not listed because EBP is saved/restored \ + * explicitly for wchan access and EAX is the return value of \ + * __switch_to()) \ + */ \ + unsigned long ebx, ecx, edx, esi, edi; \ + \ + asm volatile("pushfl\n\t" /* save flags */ \ + "pushl %%ebp\n\t" /* save EBP */ \ + "movl %%esp,%[prev_sp]\n\t" /* save ESP */ \ + "movl %[next_sp],%%esp\n\t" /* restore ESP */ \ + "movl $1f,%[prev_ip]\n\t" /* save EIP */ \ + "pushl %[next_ip]\n\t" /* restore EIP */ \ + "jmp __switch_to\n" /* regparm call */ \ "1:\t" \ - "popl %%ebp\n\t" \ - "popfl" \ - :"=m" (prev->thread.sp), "=m" (prev->thread.ip), \ - "=a" (last), "=S" (esi), "=D" (edi) \ - :"m" (next->thread.sp), "m" (next->thread.ip), \ - "2" (prev), "d" (next)); \ + "popl %%ebp\n\t" /* restore EBP */ \ + "popfl\n" /* restore flags */ \ + \ + /* output parameters */ \ + : [prev_sp] "=m" (prev->thread.sp), \ + [prev_ip] "=m" (prev->thread.ip), \ + "=a" (last), \ + \ + /* clobbered output registers: */ \ + "=b" (ebx), "=c" (ecx), "=d" (edx), \ + "=S" (esi), "=D" (edi) \ + \ + /* input parameters: */ \ + : [next_sp] "m" (next->thread.sp), \ + [next_ip] "m" (next->thread.ip), \ + \ + /* regparm parameters for __switch_to(): */ \ + [prev] "a" (prev), \ + [next] "d" (next)); \ } while (0) /* @@ -122,35 +144,34 @@ extern void load_gs_index(unsigned); */ #define loadsegment(seg, value) \ asm volatile("\n" \ - "1:\t" \ - "movl %k0,%%" #seg "\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3:\t" \ - "movl %k1, %%" #seg "\n\t" \ - "jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b,3b) \ - : :"r" (value), "r" (0)) + "1:\t" \ + "movl %k0,%%" #seg "\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3:\t" \ + "movl %k1, %%" #seg "\n\t" \ + "jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b,3b) \ + : :"r" (value), "r" (0)) /* * Save a segment register away */ -#define savesegment(seg, value) \ +#define savesegment(seg, value) \ asm volatile("mov %%" #seg ",%0":"=rm" (value)) static inline unsigned long get_limit(unsigned long segment) { unsigned long __limit; - __asm__("lsll %1,%0" - :"=r" (__limit):"r" (segment)); - return __limit+1; + asm("lsll %1,%0" : "=r" (__limit) : "r" (segment)); + return __limit + 1; } static inline void native_clts(void) { - asm volatile ("clts"); + asm volatile("clts"); } /* @@ -165,43 +186,43 @@ static unsigned long __force_order; static inline unsigned long native_read_cr0(void) { unsigned long val; - asm volatile("mov %%cr0,%0\n\t" :"=r" (val), "=m" (__force_order)); + asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order)); return val; } static inline void native_write_cr0(unsigned long val) { - asm volatile("mov %0,%%cr0": :"r" (val), "m" (__force_order)); + asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order)); } static inline unsigned long native_read_cr2(void) { unsigned long val; - asm volatile("mov %%cr2,%0\n\t" :"=r" (val), "=m" (__force_order)); + asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); return val; } static inline void native_write_cr2(unsigned long val) { - asm volatile("mov %0,%%cr2": :"r" (val), "m" (__force_order)); + asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); } static inline unsigned long native_read_cr3(void) { unsigned long val; - asm volatile("mov %%cr3,%0\n\t" :"=r" (val), "=m" (__force_order)); + asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order)); return val; } static inline void native_write_cr3(unsigned long val) { - asm volatile("mov %0,%%cr3": :"r" (val), "m" (__force_order)); + asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order)); } static inline unsigned long native_read_cr4(void) { unsigned long val; - asm volatile("mov %%cr4,%0\n\t" :"=r" (val), "=m" (__force_order)); + asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order)); return val; } @@ -213,7 +234,7 @@ static inline unsigned long native_read_cr4_safe(void) #ifdef CONFIG_X86_32 asm volatile("1: mov %%cr4, %0\n" "2:\n" - _ASM_EXTABLE(1b,2b) + _ASM_EXTABLE(1b, 2b) : "=r" (val), "=m" (__force_order) : "0" (0)); #else val = native_read_cr4(); @@ -223,7 +244,7 @@ static inline unsigned long native_read_cr4_safe(void) static inline void native_write_cr4(unsigned long val) { - asm volatile("mov %0,%%cr4": :"r" (val), "m" (__force_order)); + asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order)); } #ifdef CONFIG_X86_64 @@ -244,6 +265,7 @@ static inline void native_wbinvd(void) { asm volatile("wbinvd": : :"memory"); } + #ifdef CONFIG_PARAVIRT #include <asm/paravirt.h> #else @@ -276,7 +298,7 @@ static inline void clflush(volatile void *__p) asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p)); } -#define nop() __asm__ __volatile__ ("nop") +#define nop() asm volatile ("nop") void disable_hlt(void); void enable_hlt(void); @@ -296,16 +318,7 @@ void default_idle(void); */ #ifdef CONFIG_X86_32 /* - * For now, "wmb()" doesn't actually do anything, as all - * Intel CPU's follow what Intel calls a *Processor Order*, - * in which all writes are seen in the program order even - * outside the CPU. - * - * I expect future Intel CPU's to have a weaker ordering, - * but I'd also expect them to finally get their act together - * and add some real memory barriers if so. - * - * Some non intel clones support out of order store. wmb() ceases to be a + * Some non-Intel clones support out of order store. wmb() ceases to be a * nop for these. */ #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2) @@ -384,7 +397,7 @@ void default_idle(void); # define smp_wmb() barrier() #endif #define smp_read_barrier_depends() read_barrier_depends() -#define set_mb(var, value) do { (void) xchg(&var, value); } while (0) +#define set_mb(var, value) do { (void)xchg(&var, value); } while (0) #else #define smp_mb() barrier() #define smp_rmb() barrier() diff --git a/include/asm-x86/tce.h b/include/asm-x86/tce.h index cd955d3d112..b1a4ea00df7 100644 --- a/include/asm-x86/tce.h +++ b/include/asm-x86/tce.h @@ -39,7 +39,7 @@ struct iommu_table; #define TCE_RPN_MASK 0x0000fffffffff000ULL extern void tce_build(struct iommu_table *tbl, unsigned long index, - unsigned int npages, unsigned long uaddr, int direction); + unsigned int npages, unsigned long uaddr, int direction); extern void tce_free(struct iommu_table *tbl, long index, unsigned int npages); extern void * __init alloc_tce_table(void); extern void __init free_tce_table(void *tbl); diff --git a/include/asm-x86/thread_info.h b/include/asm-x86/thread_info.h index d5fd12f2abd..77244f17993 100644 --- a/include/asm-x86/thread_info.h +++ b/include/asm-x86/thread_info.h @@ -1,5 +1,14 @@ +#ifndef _ASM_X86_THREAD_INFO_H #ifdef CONFIG_X86_32 # include "thread_info_32.h" #else # include "thread_info_64.h" #endif + +#ifndef __ASSEMBLY__ +extern void arch_task_cache_init(void); +extern void free_thread_info(struct thread_info *ti); +extern int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); +#define arch_task_cache_init arch_task_cache_init +#endif +#endif /* _ASM_X86_THREAD_INFO_H */ diff --git a/include/asm-x86/thread_info_32.h b/include/asm-x86/thread_info_32.h index 5bd508260ff..b6338829d1a 100644 --- a/include/asm-x86/thread_info_32.h +++ b/include/asm-x86/thread_info_32.h @@ -20,7 +20,8 @@ * low level task data that entry.S needs immediate access to * - this struct should fit entirely inside of one cache line * - this struct shares the supervisor stack pages - * - if the contents of this structure are changed, the assembly constants must also be changed + * - if the contents of this structure are changed, + * the assembly constants must also be changed */ #ifndef __ASSEMBLY__ @@ -30,18 +31,16 @@ struct thread_info { unsigned long flags; /* low level flags */ unsigned long status; /* thread-synchronous flags */ __u32 cpu; /* current CPU */ - int preempt_count; /* 0 => preemptable, <0 => BUG */ - - + int preempt_count; /* 0 => preemptable, + <0 => BUG */ mm_segment_t addr_limit; /* thread address space: - 0-0xBFFFFFFF for user-thead - 0-0xFFFFFFFF for kernel-thread + 0-0xBFFFFFFF user-thread + 0-0xFFFFFFFF kernel-thread */ void *sysenter_return; struct restart_block restart_block; - - unsigned long previous_esp; /* ESP of the previous stack in case - of nested (IRQ) stacks + unsigned long previous_esp; /* ESP of the previous stack in + case of nested (IRQ) stacks */ __u8 supervisor_stack[0]; }; @@ -90,24 +89,23 @@ register unsigned long current_stack_pointer asm("esp") __used; /* how to get the thread information struct from C */ static inline struct thread_info *current_thread_info(void) { - return (struct thread_info *)(current_stack_pointer & ~(THREAD_SIZE - 1)); + return (struct thread_info *) + (current_stack_pointer & ~(THREAD_SIZE - 1)); } /* thread information allocation */ #ifdef CONFIG_DEBUG_STACK_USAGE -#define alloc_thread_info(tsk) ((struct thread_info *) \ - __get_free_pages(GFP_KERNEL| __GFP_ZERO, get_order(THREAD_SIZE))) +#define alloc_thread_info(tsk) ((struct thread_info *) \ + __get_free_pages(GFP_KERNEL | __GFP_ZERO, get_order(THREAD_SIZE))) #else -#define alloc_thread_info(tsk) ((struct thread_info *) \ +#define alloc_thread_info(tsk) ((struct thread_info *) \ __get_free_pages(GFP_KERNEL, get_order(THREAD_SIZE))) #endif -#define free_thread_info(info) free_pages((unsigned long)(info), get_order(THREAD_SIZE)) - #else /* !__ASSEMBLY__ */ /* how to get the thread information struct from ASM */ -#define GET_THREAD_INFO(reg) \ +#define GET_THREAD_INFO(reg) \ movl $-THREAD_SIZE, reg; \ andl %esp, reg @@ -119,19 +117,20 @@ static inline struct thread_info *current_thread_info(void) /* * thread information flags - * - these are process state flags that various assembly files may need to access + * - these are process state flags that various + * assembly files may need to access * - pending work-to-be-done flags are in LSW * - other flags in MSW */ #define TIF_SYSCALL_TRACE 0 /* syscall trace active */ #define TIF_SIGPENDING 1 /* signal pending */ #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ -#define TIF_SINGLESTEP 3 /* restore singlestep on return to user mode */ +#define TIF_SINGLESTEP 3 /* restore singlestep on return to + user mode */ #define TIF_IRET 4 /* return with iret */ #define TIF_SYSCALL_EMU 5 /* syscall emulation active */ #define TIF_SYSCALL_AUDIT 6 /* syscall auditing active */ #define TIF_SECCOMP 7 /* secure computing */ -#define TIF_RESTORE_SIGMASK 8 /* restore signal mask in do_signal() */ #define TIF_HRTICK_RESCHED 9 /* reprogram hrtick timer */ #define TIF_MEMDIE 16 #define TIF_DEBUG 17 /* uses debug registers */ @@ -143,36 +142,35 @@ static inline struct thread_info *current_thread_info(void) #define TIF_DS_AREA_MSR 23 /* uses thread_struct.ds_area_msr */ #define TIF_BTS_TRACE_TS 24 /* record scheduling event timestamps */ -#define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE) -#define _TIF_SIGPENDING (1<<TIF_SIGPENDING) -#define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) -#define _TIF_SINGLESTEP (1<<TIF_SINGLESTEP) -#define _TIF_IRET (1<<TIF_IRET) -#define _TIF_SYSCALL_EMU (1<<TIF_SYSCALL_EMU) -#define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) -#define _TIF_SECCOMP (1<<TIF_SECCOMP) -#define _TIF_RESTORE_SIGMASK (1<<TIF_RESTORE_SIGMASK) -#define _TIF_HRTICK_RESCHED (1<<TIF_HRTICK_RESCHED) -#define _TIF_DEBUG (1<<TIF_DEBUG) -#define _TIF_IO_BITMAP (1<<TIF_IO_BITMAP) -#define _TIF_FREEZE (1<<TIF_FREEZE) -#define _TIF_NOTSC (1<<TIF_NOTSC) -#define _TIF_FORCED_TF (1<<TIF_FORCED_TF) -#define _TIF_DEBUGCTLMSR (1<<TIF_DEBUGCTLMSR) -#define _TIF_DS_AREA_MSR (1<<TIF_DS_AREA_MSR) -#define _TIF_BTS_TRACE_TS (1<<TIF_BTS_TRACE_TS) +#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) +#define _TIF_SIGPENDING (1 << TIF_SIGPENDING) +#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) +#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) +#define _TIF_IRET (1 << TIF_IRET) +#define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU) +#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) +#define _TIF_SECCOMP (1 << TIF_SECCOMP) +#define _TIF_HRTICK_RESCHED (1 << TIF_HRTICK_RESCHED) +#define _TIF_DEBUG (1 << TIF_DEBUG) +#define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) +#define _TIF_FREEZE (1 << TIF_FREEZE) +#define _TIF_NOTSC (1 << TIF_NOTSC) +#define _TIF_FORCED_TF (1 << TIF_FORCED_TF) +#define _TIF_DEBUGCTLMSR (1 << TIF_DEBUGCTLMSR) +#define _TIF_DS_AREA_MSR (1 << TIF_DS_AREA_MSR) +#define _TIF_BTS_TRACE_TS (1 << TIF_BTS_TRACE_TS) /* work to do on interrupt/exception return */ -#define _TIF_WORK_MASK \ - (0x0000FFFF & ~(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ - _TIF_SECCOMP | _TIF_SYSCALL_EMU)) +#define _TIF_WORK_MASK \ + (0x0000FFFF & ~(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ + _TIF_SECCOMP | _TIF_SYSCALL_EMU)) /* work to do on any return to u-space */ #define _TIF_ALLWORK_MASK (0x0000FFFF & ~_TIF_SECCOMP) /* flags to check in __switch_to() */ -#define _TIF_WORK_CTXSW \ - (_TIF_IO_BITMAP | _TIF_NOTSC | _TIF_DEBUGCTLMSR | \ - _TIF_DS_AREA_MSR | _TIF_BTS_TRACE_TS) +#define _TIF_WORK_CTXSW \ + (_TIF_IO_BITMAP | _TIF_NOTSC | _TIF_DEBUGCTLMSR | \ + _TIF_DS_AREA_MSR | _TIF_BTS_TRACE_TS) #define _TIF_WORK_CTXSW_PREV _TIF_WORK_CTXSW #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW | _TIF_DEBUG) @@ -184,11 +182,24 @@ static inline struct thread_info *current_thread_info(void) * ever touches our thread-synchronous status, so we don't * have to worry about atomic accesses. */ -#define TS_USEDFPU 0x0001 /* FPU was used by this task this quantum (SMP) */ -#define TS_POLLING 0x0002 /* True if in idle loop and not sleeping */ +#define TS_USEDFPU 0x0001 /* FPU was used by this task + this quantum (SMP) */ +#define TS_POLLING 0x0002 /* True if in idle loop + and not sleeping */ +#define TS_RESTORE_SIGMASK 0x0004 /* restore signal mask in do_signal() */ #define tsk_is_polling(t) (task_thread_info(t)->status & TS_POLLING) +#ifndef __ASSEMBLY__ +#define HAVE_SET_RESTORE_SIGMASK 1 +static inline void set_restore_sigmask(void) +{ + struct thread_info *ti = current_thread_info(); + ti->status |= TS_RESTORE_SIGMASK; + set_bit(TIF_SIGPENDING, &ti->flags); +} +#endif /* !__ASSEMBLY__ */ + #endif /* __KERNEL__ */ #endif /* _ASM_THREAD_INFO_H */ diff --git a/include/asm-x86/thread_info_64.h b/include/asm-x86/thread_info_64.h index 6c9b214b8fc..cb69f70abba 100644 --- a/include/asm-x86/thread_info_64.h +++ b/include/asm-x86/thread_info_64.h @@ -29,9 +29,9 @@ struct thread_info { __u32 flags; /* low level flags */ __u32 status; /* thread synchronous flags */ __u32 cpu; /* current CPU */ - int preempt_count; /* 0 => preemptable, <0 => BUG */ - - mm_segment_t addr_limit; + int preempt_count; /* 0 => preemptable, + <0 => BUG */ + mm_segment_t addr_limit; struct restart_block restart_block; #ifdef CONFIG_IA32_EMULATION void __user *sysenter_return; @@ -61,17 +61,17 @@ struct thread_info { #define init_stack (init_thread_union.stack) static inline struct thread_info *current_thread_info(void) -{ +{ struct thread_info *ti; ti = (void *)(read_pda(kernelstack) + PDA_STACKOFFSET - THREAD_SIZE); - return ti; + return ti; } /* do not use in interrupt context */ static inline struct thread_info *stack_thread_info(void) { struct thread_info *ti; - __asm__("andq %%rsp,%0; ":"=r" (ti) : "0" (~(THREAD_SIZE - 1))); + asm("andq %%rsp,%0; " : "=r" (ti) : "0" (~(THREAD_SIZE - 1))); return ti; } @@ -82,10 +82,8 @@ static inline struct thread_info *stack_thread_info(void) #define THREAD_FLAGS GFP_KERNEL #endif -#define alloc_thread_info(tsk) \ - ((struct thread_info *) __get_free_pages(THREAD_FLAGS, THREAD_ORDER)) - -#define free_thread_info(ti) free_pages((unsigned long) (ti), THREAD_ORDER) +#define alloc_thread_info(tsk) \ + ((struct thread_info *)__get_free_pages(THREAD_FLAGS, THREAD_ORDER)) #else /* !__ASSEMBLY__ */ @@ -98,7 +96,8 @@ static inline struct thread_info *stack_thread_info(void) /* * thread information flags - * - these are process state flags that various assembly files may need to access + * - these are process state flags that various assembly files + * may need to access * - pending work-to-be-done flags are in LSW * - other flags in MSW * Warning: layout of LSW is hardcoded in entry.S @@ -110,11 +109,10 @@ static inline struct thread_info *stack_thread_info(void) #define TIF_IRET 5 /* force IRET */ #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ #define TIF_SECCOMP 8 /* secure computing */ -#define TIF_RESTORE_SIGMASK 9 /* restore signal mask in do_signal */ #define TIF_MCE_NOTIFY 10 /* notify userspace of an MCE */ #define TIF_HRTICK_RESCHED 11 /* reprogram hrtick timer */ /* 16 free */ -#define TIF_IA32 17 /* 32bit process */ +#define TIF_IA32 17 /* 32bit process */ #define TIF_FORK 18 /* ret_from_fork */ #define TIF_ABI_PENDING 19 #define TIF_MEMDIE 20 @@ -125,40 +123,42 @@ static inline struct thread_info *stack_thread_info(void) #define TIF_DEBUGCTLMSR 25 /* uses thread_struct.debugctlmsr */ #define TIF_DS_AREA_MSR 26 /* uses thread_struct.ds_area_msr */ #define TIF_BTS_TRACE_TS 27 /* record scheduling event timestamps */ - -#define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE) -#define _TIF_SIGPENDING (1<<TIF_SIGPENDING) -#define _TIF_SINGLESTEP (1<<TIF_SINGLESTEP) -#define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) -#define _TIF_IRET (1<<TIF_IRET) -#define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) -#define _TIF_SECCOMP (1<<TIF_SECCOMP) -#define _TIF_RESTORE_SIGMASK (1<<TIF_RESTORE_SIGMASK) -#define _TIF_MCE_NOTIFY (1<<TIF_MCE_NOTIFY) -#define _TIF_HRTICK_RESCHED (1<<TIF_HRTICK_RESCHED) -#define _TIF_IA32 (1<<TIF_IA32) -#define _TIF_FORK (1<<TIF_FORK) -#define _TIF_ABI_PENDING (1<<TIF_ABI_PENDING) -#define _TIF_DEBUG (1<<TIF_DEBUG) -#define _TIF_IO_BITMAP (1<<TIF_IO_BITMAP) -#define _TIF_FREEZE (1<<TIF_FREEZE) -#define _TIF_FORCED_TF (1<<TIF_FORCED_TF) -#define _TIF_DEBUGCTLMSR (1<<TIF_DEBUGCTLMSR) -#define _TIF_DS_AREA_MSR (1<<TIF_DS_AREA_MSR) -#define _TIF_BTS_TRACE_TS (1<<TIF_BTS_TRACE_TS) +#define TIF_NOTSC 28 /* TSC is not accessible in userland */ + +#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) +#define _TIF_SIGPENDING (1 << TIF_SIGPENDING) +#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) +#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) +#define _TIF_IRET (1 << TIF_IRET) +#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) +#define _TIF_SECCOMP (1 << TIF_SECCOMP) +#define _TIF_MCE_NOTIFY (1 << TIF_MCE_NOTIFY) +#define _TIF_HRTICK_RESCHED (1 << TIF_HRTICK_RESCHED) +#define _TIF_IA32 (1 << TIF_IA32) +#define _TIF_FORK (1 << TIF_FORK) +#define _TIF_ABI_PENDING (1 << TIF_ABI_PENDING) +#define _TIF_DEBUG (1 << TIF_DEBUG) +#define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) +#define _TIF_FREEZE (1 << TIF_FREEZE) +#define _TIF_FORCED_TF (1 << TIF_FORCED_TF) +#define _TIF_DEBUGCTLMSR (1 << TIF_DEBUGCTLMSR) +#define _TIF_DS_AREA_MSR (1 << TIF_DS_AREA_MSR) +#define _TIF_BTS_TRACE_TS (1 << TIF_BTS_TRACE_TS) +#define _TIF_NOTSC (1 << TIF_NOTSC) /* work to do on interrupt/exception return */ -#define _TIF_WORK_MASK \ - (0x0000FFFF & ~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP|_TIF_SECCOMP)) +#define _TIF_WORK_MASK \ + (0x0000FFFF & \ + ~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP|_TIF_SECCOMP)) /* work to do on any return to user space */ #define _TIF_ALLWORK_MASK (0x0000FFFF & ~_TIF_SECCOMP) -#define _TIF_DO_NOTIFY_MASK \ +#define _TIF_DO_NOTIFY_MASK \ (_TIF_SIGPENDING|_TIF_SINGLESTEP|_TIF_MCE_NOTIFY|_TIF_HRTICK_RESCHED) /* flags to check in __switch_to() */ -#define _TIF_WORK_CTXSW \ - (_TIF_IO_BITMAP|_TIF_DEBUGCTLMSR|_TIF_DS_AREA_MSR|_TIF_BTS_TRACE_TS) +#define _TIF_WORK_CTXSW \ + (_TIF_IO_BITMAP|_TIF_DEBUGCTLMSR|_TIF_DS_AREA_MSR|_TIF_BTS_TRACE_TS|_TIF_NOTSC) #define _TIF_WORK_CTXSW_PREV _TIF_WORK_CTXSW #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW|_TIF_DEBUG) @@ -171,12 +171,25 @@ static inline struct thread_info *stack_thread_info(void) * ever touches our thread-synchronous status, so we don't * have to worry about atomic accesses. */ -#define TS_USEDFPU 0x0001 /* FPU was used by this task this quantum (SMP) */ +#define TS_USEDFPU 0x0001 /* FPU was used by this task + this quantum (SMP) */ #define TS_COMPAT 0x0002 /* 32bit syscall active */ -#define TS_POLLING 0x0004 /* true if in idle loop and not sleeping */ +#define TS_POLLING 0x0004 /* true if in idle loop + and not sleeping */ +#define TS_RESTORE_SIGMASK 0x0008 /* restore signal mask in do_signal() */ #define tsk_is_polling(t) (task_thread_info(t)->status & TS_POLLING) +#ifndef __ASSEMBLY__ +#define HAVE_SET_RESTORE_SIGMASK 1 +static inline void set_restore_sigmask(void) +{ + struct thread_info *ti = current_thread_info(); + ti->status |= TS_RESTORE_SIGMASK; + set_bit(TIF_SIGPENDING, &ti->flags); +} +#endif /* !__ASSEMBLY__ */ + #endif /* __KERNEL__ */ #endif /* _ASM_THREAD_INFO_H */ diff --git a/include/asm-x86/time.h b/include/asm-x86/time.h index 68779b048a3..bce72d7a958 100644 --- a/include/asm-x86/time.h +++ b/include/asm-x86/time.h @@ -1,7 +1,6 @@ #ifndef _ASMX86_TIME_H #define _ASMX86_TIME_H -extern void (*late_time_init)(void); extern void hpet_time_init(void); #include <asm/mc146818rtc.h> diff --git a/include/asm-x86/tlbflush.h b/include/asm-x86/tlbflush.h index 3998709ed63..0c0674d9425 100644 --- a/include/asm-x86/tlbflush.h +++ b/include/asm-x86/tlbflush.h @@ -32,7 +32,7 @@ static inline void __native_flush_tlb_global(void) static inline void __native_flush_tlb_single(unsigned long addr) { - __asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory"); + asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); } static inline void __flush_tlb_all(void) @@ -134,8 +134,7 @@ void native_flush_tlb_others(const cpumask_t *cpumask, struct mm_struct *mm, #define TLBSTATE_LAZY 2 #ifdef CONFIG_X86_32 -struct tlb_state -{ +struct tlb_state { struct mm_struct *active_mm; int state; char __cacheline_padding[L1_CACHE_BYTES-8]; diff --git a/include/asm-x86/topology.h b/include/asm-x86/topology.h index 8af05a93f09..4f35a0fb4f2 100644 --- a/include/asm-x86/topology.h +++ b/include/asm-x86/topology.h @@ -32,13 +32,18 @@ /* Mappings between logical cpu number and node number */ #ifdef CONFIG_X86_32 extern int cpu_to_node_map[]; - #else +/* Returns the number of the current Node. */ +#define numa_node_id() (early_cpu_to_node(raw_smp_processor_id())) +#endif + DECLARE_PER_CPU(int, x86_cpu_to_node_map); + +#ifdef CONFIG_SMP extern int x86_cpu_to_node_map_init[]; extern void *x86_cpu_to_node_map_early_ptr; -/* Returns the number of the current Node. */ -#define numa_node_id() (early_cpu_to_node(raw_smp_processor_id())) +#else +#define x86_cpu_to_node_map_early_ptr NULL #endif extern cpumask_t node_to_cpumask_map[]; @@ -54,6 +59,8 @@ static inline int cpu_to_node(int cpu) } #else /* CONFIG_X86_64 */ + +#ifdef CONFIG_SMP static inline int early_cpu_to_node(int cpu) { int *cpu_to_node_map = x86_cpu_to_node_map_early_ptr; @@ -65,22 +72,33 @@ static inline int early_cpu_to_node(int cpu) else return NUMA_NO_NODE; } +#else +#define early_cpu_to_node(cpu) cpu_to_node(cpu) +#endif static inline int cpu_to_node(int cpu) { #ifdef CONFIG_DEBUG_PER_CPU_MAPS if (x86_cpu_to_node_map_early_ptr) { printk("KERN_NOTICE cpu_to_node(%d): usage too early!\n", - (int)cpu); + (int)cpu); dump_stack(); return ((int *)x86_cpu_to_node_map_early_ptr)[cpu]; } #endif - if (per_cpu_offset(cpu)) - return per_cpu(x86_cpu_to_node_map, cpu); - else - return NUMA_NO_NODE; + return per_cpu(x86_cpu_to_node_map, cpu); } + +#ifdef CONFIG_NUMA + +/* Returns a pointer to the cpumask of CPUs on Node 'node'. */ +#define node_to_cpumask_ptr(v, node) \ + cpumask_t *v = &(node_to_cpumask_map[node]) + +#define node_to_cpumask_ptr_next(v, node) \ + v = &(node_to_cpumask_map[node]) +#endif + #endif /* CONFIG_X86_64 */ /* @@ -129,17 +147,13 @@ extern unsigned long node_remap_size[]; # define SD_CACHE_NICE_TRIES 2 # define SD_IDLE_IDX 2 -# define SD_NEWIDLE_IDX 0 +# define SD_NEWIDLE_IDX 2 # define SD_FORKEXEC_IDX 1 #endif /* sched_domains SD_NODE_INIT for NUMAQ machines */ #define SD_NODE_INIT (struct sched_domain) { \ - .span = CPU_MASK_NONE, \ - .parent = NULL, \ - .child = NULL, \ - .groups = NULL, \ .min_interval = 8, \ .max_interval = 32, \ .busy_factor = 32, \ @@ -157,7 +171,6 @@ extern unsigned long node_remap_size[]; | SD_WAKE_BALANCE, \ .last_balance = jiffies, \ .balance_interval = 1, \ - .nr_balance_failed = 0, \ } #ifdef CONFIG_X86_64_ACPI_NUMA @@ -167,10 +180,10 @@ extern int __node_distance(int, int); #else /* CONFIG_NUMA */ -#include <asm-generic/topology.h> - #endif +#include <asm-generic/topology.h> + extern cpumask_t cpu_coregroup_map(int cpu); #ifdef ENABLE_TOPO_DEFINES @@ -180,9 +193,29 @@ extern cpumask_t cpu_coregroup_map(int cpu); #define topology_thread_siblings(cpu) (per_cpu(cpu_sibling_map, cpu)) #endif +static inline void arch_fix_phys_package_id(int num, u32 slot) +{ +} + +struct pci_bus; +void set_pci_bus_resources_arch_default(struct pci_bus *b); + #ifdef CONFIG_SMP #define mc_capable() (boot_cpu_data.x86_max_cores > 1) #define smt_capable() (smp_num_siblings > 1) #endif +#ifdef CONFIG_NUMA +extern int get_mp_bus_to_node(int busnum); +extern void set_mp_bus_to_node(int busnum, int node); +#else +static inline int get_mp_bus_to_node(int busnum) +{ + return 0; +} +static inline void set_mp_bus_to_node(int busnum, int node) +{ +} +#endif + #endif diff --git a/include/asm-x86/trampoline.h b/include/asm-x86/trampoline.h new file mode 100644 index 00000000000..b156b08d013 --- /dev/null +++ b/include/asm-x86/trampoline.h @@ -0,0 +1,21 @@ +#ifndef __TRAMPOLINE_HEADER +#define __TRAMPOLINE_HEADER + +#ifndef __ASSEMBLY__ + +/* + * Trampoline 80x86 program as an array. + */ +extern const unsigned char trampoline_data []; +extern const unsigned char trampoline_end []; +extern unsigned char *trampoline_base; + +extern unsigned long init_rsp; +extern unsigned long initial_code; + +#define TRAMPOLINE_BASE 0x6000 +extern unsigned long setup_trampoline(void); + +#endif /* __ASSEMBLY__ */ + +#endif /* __TRAMPOLINE_HEADER */ diff --git a/include/asm-x86/tsc.h b/include/asm-x86/tsc.h index 7d3e27f7d48..548873ab5fc 100644 --- a/include/asm-x86/tsc.h +++ b/include/asm-x86/tsc.h @@ -32,7 +32,7 @@ static inline cycles_t get_cycles(void) return ret; } -static inline cycles_t vget_cycles(void) +static __always_inline cycles_t vget_cycles(void) { /* * We only do VDSOs on TSC capable CPUs, so this shouldnt @@ -42,7 +42,7 @@ static inline cycles_t vget_cycles(void) if (!cpu_has_tsc) return 0; #endif - return (cycles_t) __native_read_tsc(); + return (cycles_t)__native_read_tsc(); } extern void tsc_init(void); diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index fcc570ec4fe..8e7595c1f34 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -32,7 +32,7 @@ #define get_fs() (current_thread_info()->addr_limit) #define set_fs(x) (current_thread_info()->addr_limit = (x)) -#define segment_eq(a,b) ((a).seg == (b).seg) +#define segment_eq(a, b) ((a).seg == (b).seg) /* * movsl can be slow when source and dest are not both 8-byte aligned @@ -43,7 +43,9 @@ extern struct movsl_mask { } ____cacheline_aligned_in_smp movsl_mask; #endif -#define __addr_ok(addr) ((unsigned long __force)(addr) < (current_thread_info()->addr_limit.seg)) +#define __addr_ok(addr) \ + ((unsigned long __force)(addr) < \ + (current_thread_info()->addr_limit.seg)) /* * Test whether a block of memory is a valid user space address. @@ -54,13 +56,16 @@ extern struct movsl_mask { * * This needs 33-bit arithmetic. We have a carry... */ -#define __range_ok(addr,size) ({ \ - unsigned long flag,roksum; \ - __chk_user_ptr(addr); \ - asm("addl %3,%1 ; sbbl %0,%0; cmpl %1,%4; sbbl $0,%0" \ - :"=&r" (flag), "=r" (roksum) \ - :"1" (addr),"g" ((int)(size)),"rm" (current_thread_info()->addr_limit.seg)); \ - flag; }) +#define __range_ok(addr, size) \ +({ \ + unsigned long flag, roksum; \ + __chk_user_ptr(addr); \ + asm("addl %3,%1 ; sbbl %0,%0; cmpl %1,%4; sbbl $0,%0" \ + :"=&r" (flag), "=r" (roksum) \ + :"1" (addr), "g" ((int)(size)), \ + "rm" (current_thread_info()->addr_limit.seg)); \ + flag; \ +}) /** * access_ok: - Checks if a user space pointer is valid @@ -81,7 +86,7 @@ extern struct movsl_mask { * checks that the pointer is in the user space range - after calling * this function, memory access functions may still return -EFAULT. */ -#define access_ok(type,addr,size) (likely(__range_ok(addr,size) == 0)) +#define access_ok(type, addr, size) (likely(__range_ok(addr, size) == 0)) /* * The exception table consists of pairs of addresses: the first is the @@ -96,8 +101,7 @@ extern struct movsl_mask { * on our cache or tlb entries. */ -struct exception_table_entry -{ +struct exception_table_entry { unsigned long insn, fixup; }; @@ -122,13 +126,15 @@ extern void __get_user_1(void); extern void __get_user_2(void); extern void __get_user_4(void); -#define __get_user_x(size,ret,x,ptr) \ - __asm__ __volatile__("call __get_user_" #size \ - :"=a" (ret),"=d" (x) \ - :"0" (ptr)) +#define __get_user_x(size, ret, x, ptr) \ + asm volatile("call __get_user_" #size \ + :"=a" (ret),"=d" (x) \ + :"0" (ptr)) + +/* Careful: we have to cast the result to the type of the pointer + * for sign reasons */ -/* Careful: we have to cast the result to the type of the pointer for sign reasons */ /** * get_user: - Get a simple variable from user space. * @x: Variable to store result. @@ -146,15 +152,24 @@ extern void __get_user_4(void); * Returns zero on success, or -EFAULT on error. * On error, the variable @x is set to zero. */ -#define get_user(x,ptr) \ -({ int __ret_gu; \ +#define get_user(x, ptr) \ +({ \ + int __ret_gu; \ unsigned long __val_gu; \ __chk_user_ptr(ptr); \ - switch(sizeof (*(ptr))) { \ - case 1: __get_user_x(1,__ret_gu,__val_gu,ptr); break; \ - case 2: __get_user_x(2,__ret_gu,__val_gu,ptr); break; \ - case 4: __get_user_x(4,__ret_gu,__val_gu,ptr); break; \ - default: __get_user_x(X,__ret_gu,__val_gu,ptr); break; \ + switch (sizeof(*(ptr))) { \ + case 1: \ + __get_user_x(1, __ret_gu, __val_gu, ptr); \ + break; \ + case 2: \ + __get_user_x(2, __ret_gu, __val_gu, ptr); \ + break; \ + case 4: \ + __get_user_x(4, __ret_gu, __val_gu, ptr); \ + break; \ + default: \ + __get_user_x(X, __ret_gu, __val_gu, ptr); \ + break; \ } \ (x) = (__typeof__(*(ptr)))__val_gu; \ __ret_gu; \ @@ -171,11 +186,25 @@ extern void __put_user_2(void); extern void __put_user_4(void); extern void __put_user_8(void); -#define __put_user_1(x, ptr) __asm__ __volatile__("call __put_user_1":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr)) -#define __put_user_2(x, ptr) __asm__ __volatile__("call __put_user_2":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr)) -#define __put_user_4(x, ptr) __asm__ __volatile__("call __put_user_4":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr)) -#define __put_user_8(x, ptr) __asm__ __volatile__("call __put_user_8":"=a" (__ret_pu):"A" ((typeof(*(ptr)))(x)), "c" (ptr)) -#define __put_user_X(x, ptr) __asm__ __volatile__("call __put_user_X":"=a" (__ret_pu):"c" (ptr)) +#define __put_user_1(x, ptr) \ + asm volatile("call __put_user_1" : "=a" (__ret_pu) \ + : "0" ((typeof(*(ptr)))(x)), "c" (ptr)) + +#define __put_user_2(x, ptr) \ + asm volatile("call __put_user_2" : "=a" (__ret_pu) \ + : "0" ((typeof(*(ptr)))(x)), "c" (ptr)) + +#define __put_user_4(x, ptr) \ + asm volatile("call __put_user_4" : "=a" (__ret_pu) \ + : "0" ((typeof(*(ptr)))(x)), "c" (ptr)) + +#define __put_user_8(x, ptr) \ + asm volatile("call __put_user_8" : "=a" (__ret_pu) \ + : "A" ((typeof(*(ptr)))(x)), "c" (ptr)) + +#define __put_user_X(x, ptr) \ + asm volatile("call __put_user_X" : "=a" (__ret_pu) \ + : "c" (ptr)) /** * put_user: - Write a simple value into user space. @@ -195,32 +224,43 @@ extern void __put_user_8(void); */ #ifdef CONFIG_X86_WP_WORKS_OK -#define put_user(x,ptr) \ -({ int __ret_pu; \ +#define put_user(x, ptr) \ +({ \ + int __ret_pu; \ __typeof__(*(ptr)) __pu_val; \ __chk_user_ptr(ptr); \ __pu_val = x; \ - switch(sizeof(*(ptr))) { \ - case 1: __put_user_1(__pu_val, ptr); break; \ - case 2: __put_user_2(__pu_val, ptr); break; \ - case 4: __put_user_4(__pu_val, ptr); break; \ - case 8: __put_user_8(__pu_val, ptr); break; \ - default:__put_user_X(__pu_val, ptr); break; \ + switch (sizeof(*(ptr))) { \ + case 1: \ + __put_user_1(__pu_val, ptr); \ + break; \ + case 2: \ + __put_user_2(__pu_val, ptr); \ + break; \ + case 4: \ + __put_user_4(__pu_val, ptr); \ + break; \ + case 8: \ + __put_user_8(__pu_val, ptr); \ + break; \ + default: \ + __put_user_X(__pu_val, ptr); \ + break; \ } \ __ret_pu; \ }) #else -#define put_user(x,ptr) \ +#define put_user(x, ptr) \ ({ \ - int __ret_pu; \ - __typeof__(*(ptr)) __pus_tmp = x; \ - __ret_pu=0; \ - if(unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \ - sizeof(*(ptr))) != 0)) \ - __ret_pu=-EFAULT; \ - __ret_pu; \ - }) + int __ret_pu; \ + __typeof__(*(ptr))__pus_tmp = x; \ + __ret_pu = 0; \ + if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \ + sizeof(*(ptr))) != 0)) \ + __ret_pu = -EFAULT; \ + __ret_pu; \ +}) #endif @@ -245,8 +285,8 @@ extern void __put_user_8(void); * Returns zero on success, or -EFAULT on error. * On error, the variable @x is set to zero. */ -#define __get_user(x,ptr) \ - __get_user_nocheck((x),(ptr),sizeof(*(ptr))) +#define __get_user(x, ptr) \ + __get_user_nocheck((x), (ptr), sizeof(*(ptr))) /** @@ -268,54 +308,62 @@ extern void __put_user_8(void); * * Returns zero on success, or -EFAULT on error. */ -#define __put_user(x,ptr) \ - __put_user_nocheck((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr))) +#define __put_user(x, ptr) \ + __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) -#define __put_user_nocheck(x,ptr,size) \ +#define __put_user_nocheck(x, ptr, size) \ ({ \ long __pu_err; \ - __put_user_size((x),(ptr),(size),__pu_err,-EFAULT); \ + __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ __pu_err; \ }) -#define __put_user_u64(x, addr, err) \ - __asm__ __volatile__( \ - "1: movl %%eax,0(%2)\n" \ - "2: movl %%edx,4(%2)\n" \ - "3:\n" \ - ".section .fixup,\"ax\"\n" \ - "4: movl %3,%0\n" \ - " jmp 3b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b,4b) \ - _ASM_EXTABLE(2b,4b) \ - : "=r"(err) \ - : "A" (x), "r" (addr), "i"(-EFAULT), "0"(err)) +#define __put_user_u64(x, addr, err) \ + asm volatile("1: movl %%eax,0(%2)\n" \ + "2: movl %%edx,4(%2)\n" \ + "3:\n" \ + ".section .fixup,\"ax\"\n" \ + "4: movl %3,%0\n" \ + " jmp 3b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 4b) \ + _ASM_EXTABLE(2b, 4b) \ + : "=r" (err) \ + : "A" (x), "r" (addr), "i" (-EFAULT), "0" (err)) #ifdef CONFIG_X86_WP_WORKS_OK -#define __put_user_size(x,ptr,size,retval,errret) \ +#define __put_user_size(x, ptr, size, retval, errret) \ do { \ retval = 0; \ __chk_user_ptr(ptr); \ switch (size) { \ - case 1: __put_user_asm(x,ptr,retval,"b","b","iq",errret);break; \ - case 2: __put_user_asm(x,ptr,retval,"w","w","ir",errret);break; \ - case 4: __put_user_asm(x,ptr,retval,"l","","ir",errret); break; \ - case 8: __put_user_u64((__typeof__(*ptr))(x),ptr,retval); break;\ - default: __put_user_bad(); \ + case 1: \ + __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \ + break; \ + case 2: \ + __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \ + break; \ + case 4: \ + __put_user_asm(x, ptr, retval, "l", "", "ir", errret); \ + break; \ + case 8: \ + __put_user_u64((__typeof__(*ptr))(x), ptr, retval); \ + break; \ + default: \ + __put_user_bad(); \ } \ } while (0) #else -#define __put_user_size(x,ptr,size,retval,errret) \ +#define __put_user_size(x, ptr, size, retval, errret) \ do { \ - __typeof__(*(ptr)) __pus_tmp = x; \ + __typeof__(*(ptr))__pus_tmp = x; \ retval = 0; \ \ - if(unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \ + if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \ retval = errret; \ } while (0) @@ -329,65 +377,70 @@ struct __large_struct { unsigned long buf[100]; }; * aliasing issues. */ #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ - __asm__ __volatile__( \ - "1: mov"itype" %"rtype"1,%2\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movl %3,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b,3b) \ - : "=r"(err) \ - : ltype (x), "m"(__m(addr)), "i"(errret), "0"(err)) - - -#define __get_user_nocheck(x,ptr,size) \ -({ \ - long __gu_err; \ - unsigned long __gu_val; \ - __get_user_size(__gu_val,(ptr),(size),__gu_err,-EFAULT);\ - (x) = (__typeof__(*(ptr)))__gu_val; \ - __gu_err; \ + asm volatile("1: mov"itype" %"rtype"1,%2\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: movl %3,%0\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r"(err) \ + : ltype (x), "m" (__m(addr)), "i" (errret), "0" (err)) + + +#define __get_user_nocheck(x, ptr, size) \ +({ \ + long __gu_err; \ + unsigned long __gu_val; \ + __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \ + (x) = (__typeof__(*(ptr)))__gu_val; \ + __gu_err; \ }) extern long __get_user_bad(void); -#define __get_user_size(x,ptr,size,retval,errret) \ +#define __get_user_size(x, ptr, size, retval, errret) \ do { \ retval = 0; \ __chk_user_ptr(ptr); \ switch (size) { \ - case 1: __get_user_asm(x,ptr,retval,"b","b","=q",errret);break; \ - case 2: __get_user_asm(x,ptr,retval,"w","w","=r",errret);break; \ - case 4: __get_user_asm(x,ptr,retval,"l","","=r",errret);break; \ - default: (x) = __get_user_bad(); \ + case 1: \ + __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \ + break; \ + case 2: \ + __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \ + break; \ + case 4: \ + __get_user_asm(x, ptr, retval, "l", "", "=r", errret); \ + break; \ + default: \ + (x) = __get_user_bad(); \ } \ } while (0) #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ - __asm__ __volatile__( \ - "1: mov"itype" %2,%"rtype"1\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movl %3,%0\n" \ - " xor"itype" %"rtype"1,%"rtype"1\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b,3b) \ - : "=r"(err), ltype (x) \ - : "m"(__m(addr)), "i"(errret), "0"(err)) - - -unsigned long __must_check __copy_to_user_ll(void __user *to, - const void *from, unsigned long n); -unsigned long __must_check __copy_from_user_ll(void *to, - const void __user *from, unsigned long n); -unsigned long __must_check __copy_from_user_ll_nozero(void *to, - const void __user *from, unsigned long n); -unsigned long __must_check __copy_from_user_ll_nocache(void *to, - const void __user *from, unsigned long n); -unsigned long __must_check __copy_from_user_ll_nocache_nozero(void *to, - const void __user *from, unsigned long n); + asm volatile("1: mov"itype" %2,%"rtype"1\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: movl %3,%0\n" \ + " xor"itype" %"rtype"1,%"rtype"1\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r" (err), ltype (x) \ + : "m" (__m(addr)), "i" (errret), "0" (err)) + + +unsigned long __must_check __copy_to_user_ll + (void __user *to, const void *from, unsigned long n); +unsigned long __must_check __copy_from_user_ll + (void *to, const void __user *from, unsigned long n); +unsigned long __must_check __copy_from_user_ll_nozero + (void *to, const void __user *from, unsigned long n); +unsigned long __must_check __copy_from_user_ll_nocache + (void *to, const void __user *from, unsigned long n); +unsigned long __must_check __copy_from_user_ll_nocache_nozero + (void *to, const void __user *from, unsigned long n); /** * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking. @@ -416,13 +469,16 @@ __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n) switch (n) { case 1: - __put_user_size(*(u8 *)from, (u8 __user *)to, 1, ret, 1); + __put_user_size(*(u8 *)from, (u8 __user *)to, + 1, ret, 1); return ret; case 2: - __put_user_size(*(u16 *)from, (u16 __user *)to, 2, ret, 2); + __put_user_size(*(u16 *)from, (u16 __user *)to, + 2, ret, 2); return ret; case 4: - __put_user_size(*(u32 *)from, (u32 __user *)to, 4, ret, 4); + __put_user_size(*(u32 *)from, (u32 __user *)to, + 4, ret, 4); return ret; } } @@ -545,19 +601,21 @@ static __always_inline unsigned long __copy_from_user_nocache(void *to, } static __always_inline unsigned long -__copy_from_user_inatomic_nocache(void *to, const void __user *from, unsigned long n) +__copy_from_user_inatomic_nocache(void *to, const void __user *from, + unsigned long n) { return __copy_from_user_ll_nocache_nozero(to, from, n); } unsigned long __must_check copy_to_user(void __user *to, - const void *from, unsigned long n); + const void *from, unsigned long n); unsigned long __must_check copy_from_user(void *to, - const void __user *from, unsigned long n); + const void __user *from, + unsigned long n); long __must_check strncpy_from_user(char *dst, const char __user *src, - long count); + long count); long __must_check __strncpy_from_user(char *dst, - const char __user *src, long count); + const char __user *src, long count); /** * strlen_user: - Get the size of a string in user space. diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index b87eb4ba8f9..b8a2f433990 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -29,23 +29,27 @@ #define get_fs() (current_thread_info()->addr_limit) #define set_fs(x) (current_thread_info()->addr_limit = (x)) -#define segment_eq(a,b) ((a).seg == (b).seg) +#define segment_eq(a, b) ((a).seg == (b).seg) -#define __addr_ok(addr) (!((unsigned long)(addr) & (current_thread_info()->addr_limit.seg))) +#define __addr_ok(addr) (!((unsigned long)(addr) & \ + (current_thread_info()->addr_limit.seg))) /* * Uhhuh, this needs 65-bit arithmetic. We have a carry.. */ -#define __range_not_ok(addr,size) ({ \ - unsigned long flag,roksum; \ - __chk_user_ptr(addr); \ - asm("# range_ok\n\r" \ - "addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0" \ - :"=&r" (flag), "=r" (roksum) \ - :"1" (addr),"g" ((long)(size)),"g" (current_thread_info()->addr_limit.seg)); \ - flag; }) +#define __range_not_ok(addr, size) \ +({ \ + unsigned long flag, roksum; \ + __chk_user_ptr(addr); \ + asm("# range_ok\n\r" \ + "addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0" \ + : "=&r" (flag), "=r" (roksum) \ + : "1" (addr), "g" ((long)(size)), \ + "g" (current_thread_info()->addr_limit.seg)); \ + flag; \ +}) -#define access_ok(type, addr, size) (__range_not_ok(addr,size) == 0) +#define access_ok(type, addr, size) (__range_not_ok(addr, size) == 0) /* * The exception table consists of pairs of addresses: the first is the @@ -60,8 +64,7 @@ * on our cache or tlb entries. */ -struct exception_table_entry -{ +struct exception_table_entry { unsigned long insn, fixup; }; @@ -84,23 +87,36 @@ extern int fixup_exception(struct pt_regs *regs); * accesses to the same area of user memory). */ -#define __get_user_x(size,ret,x,ptr) \ - asm volatile("call __get_user_" #size \ - :"=a" (ret),"=d" (x) \ - :"c" (ptr) \ - :"r8") +#define __get_user_x(size, ret, x, ptr) \ + asm volatile("call __get_user_" #size \ + : "=a" (ret),"=d" (x) \ + : "c" (ptr) \ + : "r8") + +/* Careful: we have to cast the result to the type of the pointer + * for sign reasons */ -/* Careful: we have to cast the result to the type of the pointer for sign reasons */ -#define get_user(x,ptr) \ -({ unsigned long __val_gu; \ - int __ret_gu; \ +#define get_user(x, ptr) \ +({ \ + unsigned long __val_gu; \ + int __ret_gu; \ __chk_user_ptr(ptr); \ - switch(sizeof (*(ptr))) { \ - case 1: __get_user_x(1,__ret_gu,__val_gu,ptr); break; \ - case 2: __get_user_x(2,__ret_gu,__val_gu,ptr); break; \ - case 4: __get_user_x(4,__ret_gu,__val_gu,ptr); break; \ - case 8: __get_user_x(8,__ret_gu,__val_gu,ptr); break; \ - default: __get_user_bad(); break; \ + switch (sizeof(*(ptr))) { \ + case 1: \ + __get_user_x(1, __ret_gu, __val_gu, ptr); \ + break; \ + case 2: \ + __get_user_x(2, __ret_gu, __val_gu, ptr); \ + break; \ + case 4: \ + __get_user_x(4, __ret_gu, __val_gu, ptr); \ + break; \ + case 8: \ + __get_user_x(8, __ret_gu, __val_gu, ptr); \ + break; \ + default: \ + __get_user_bad(); \ + break; \ } \ (x) = (__force typeof(*(ptr)))__val_gu; \ __ret_gu; \ @@ -112,55 +128,73 @@ extern void __put_user_4(void); extern void __put_user_8(void); extern void __put_user_bad(void); -#define __put_user_x(size,ret,x,ptr) \ - asm volatile("call __put_user_" #size \ - :"=a" (ret) \ - :"c" (ptr),"d" (x) \ - :"r8") +#define __put_user_x(size, ret, x, ptr) \ + asm volatile("call __put_user_" #size \ + :"=a" (ret) \ + :"c" (ptr),"d" (x) \ + :"r8") -#define put_user(x,ptr) \ - __put_user_check((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr))) +#define put_user(x, ptr) \ + __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) -#define __get_user(x,ptr) \ - __get_user_nocheck((x),(ptr),sizeof(*(ptr))) -#define __put_user(x,ptr) \ - __put_user_nocheck((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr))) +#define __get_user(x, ptr) \ + __get_user_nocheck((x), (ptr), sizeof(*(ptr))) +#define __put_user(x, ptr) \ + __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) #define __get_user_unaligned __get_user #define __put_user_unaligned __put_user -#define __put_user_nocheck(x,ptr,size) \ +#define __put_user_nocheck(x, ptr, size) \ ({ \ int __pu_err; \ - __put_user_size((x),(ptr),(size),__pu_err); \ + __put_user_size((x), (ptr), (size), __pu_err); \ __pu_err; \ }) -#define __put_user_check(x,ptr,size) \ -({ \ - int __pu_err; \ - typeof(*(ptr)) __user *__pu_addr = (ptr); \ - switch (size) { \ - case 1: __put_user_x(1,__pu_err,x,__pu_addr); break; \ - case 2: __put_user_x(2,__pu_err,x,__pu_addr); break; \ - case 4: __put_user_x(4,__pu_err,x,__pu_addr); break; \ - case 8: __put_user_x(8,__pu_err,x,__pu_addr); break; \ - default: __put_user_bad(); \ - } \ - __pu_err; \ +#define __put_user_check(x, ptr, size) \ +({ \ + int __pu_err; \ + typeof(*(ptr)) __user *__pu_addr = (ptr); \ + switch (size) { \ + case 1: \ + __put_user_x(1, __pu_err, x, __pu_addr); \ + break; \ + case 2: \ + __put_user_x(2, __pu_err, x, __pu_addr); \ + break; \ + case 4: \ + __put_user_x(4, __pu_err, x, __pu_addr); \ + break; \ + case 8: \ + __put_user_x(8, __pu_err, x, __pu_addr); \ + break; \ + default: \ + __put_user_bad(); \ + } \ + __pu_err; \ }) -#define __put_user_size(x,ptr,size,retval) \ +#define __put_user_size(x, ptr, size, retval) \ do { \ retval = 0; \ __chk_user_ptr(ptr); \ switch (size) { \ - case 1: __put_user_asm(x,ptr,retval,"b","b","iq",-EFAULT); break;\ - case 2: __put_user_asm(x,ptr,retval,"w","w","ir",-EFAULT); break;\ - case 4: __put_user_asm(x,ptr,retval,"l","k","ir",-EFAULT); break;\ - case 8: __put_user_asm(x,ptr,retval,"q","","Zr",-EFAULT); break;\ - default: __put_user_bad(); \ + case 1: \ + __put_user_asm(x, ptr, retval, "b", "b", "iq", -EFAULT);\ + break; \ + case 2: \ + __put_user_asm(x, ptr, retval, "w", "w", "ir", -EFAULT);\ + break; \ + case 4: \ + __put_user_asm(x, ptr, retval, "l", "k", "ir", -EFAULT);\ + break; \ + case 8: \ + __put_user_asm(x, ptr, retval, "q", "", "Zr", -EFAULT); \ + break; \ + default: \ + __put_user_bad(); \ } \ } while (0) @@ -174,23 +208,22 @@ struct __large_struct { unsigned long buf[100]; }; * aliasing issues. */ #define __put_user_asm(x, addr, err, itype, rtype, ltype, errno) \ - asm volatile( \ - "1: mov"itype" %"rtype"1,%2\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: mov %3,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b,3b) \ - : "=r"(err) \ - : ltype (x), "m"(__m(addr)), "i"(errno), "0"(err)) - - -#define __get_user_nocheck(x,ptr,size) \ + asm volatile("1: mov"itype" %"rtype"1,%2\n" \ + "2:\n" \ + ".section .fixup, \"ax\"\n" \ + "3: mov %3,%0\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r"(err) \ + : ltype (x), "m" (__m(addr)), "i" (errno), "0" (err)) + + +#define __get_user_nocheck(x, ptr, size) \ ({ \ int __gu_err; \ unsigned long __gu_val; \ - __get_user_size(__gu_val,(ptr),(size),__gu_err); \ + __get_user_size(__gu_val, (ptr), (size), __gu_err); \ (x) = (__force typeof(*(ptr)))__gu_val; \ __gu_err; \ }) @@ -201,31 +234,39 @@ extern int __get_user_4(void); extern int __get_user_8(void); extern int __get_user_bad(void); -#define __get_user_size(x,ptr,size,retval) \ +#define __get_user_size(x, ptr, size, retval) \ do { \ retval = 0; \ __chk_user_ptr(ptr); \ switch (size) { \ - case 1: __get_user_asm(x,ptr,retval,"b","b","=q",-EFAULT); break;\ - case 2: __get_user_asm(x,ptr,retval,"w","w","=r",-EFAULT); break;\ - case 4: __get_user_asm(x,ptr,retval,"l","k","=r",-EFAULT); break;\ - case 8: __get_user_asm(x,ptr,retval,"q","","=r",-EFAULT); break;\ - default: (x) = __get_user_bad(); \ + case 1: \ + __get_user_asm(x, ptr, retval, "b", "b", "=q", -EFAULT);\ + break; \ + case 2: \ + __get_user_asm(x, ptr, retval, "w", "w", "=r", -EFAULT);\ + break; \ + case 4: \ + __get_user_asm(x, ptr, retval, "l", "k", "=r", -EFAULT);\ + break; \ + case 8: \ + __get_user_asm(x, ptr, retval, "q", "", "=r", -EFAULT); \ + break; \ + default: \ + (x) = __get_user_bad(); \ } \ } while (0) #define __get_user_asm(x, addr, err, itype, rtype, ltype, errno) \ - asm volatile( \ - "1: mov"itype" %2,%"rtype"1\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: mov %3,%0\n" \ - " xor"itype" %"rtype"1,%"rtype"1\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b,3b) \ - : "=r"(err), ltype (x) \ - : "m"(__m(addr)), "i"(errno), "0"(err)) + asm volatile("1: mov"itype" %2,%"rtype"1\n" \ + "2:\n" \ + ".section .fixup, \"ax\"\n" \ + "3: mov %3,%0\n" \ + " xor"itype" %"rtype"1,%"rtype"1\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r" (err), ltype (x) \ + : "m" (__m(addr)), "i"(errno), "0"(err)) /* * Copy To/From Userspace @@ -244,110 +285,142 @@ copy_in_user(void __user *to, const void __user *from, unsigned len); static __always_inline __must_check int __copy_from_user(void *dst, const void __user *src, unsigned size) -{ +{ int ret = 0; if (!__builtin_constant_p(size)) - return copy_user_generic(dst,(__force void *)src,size); - switch (size) { - case 1:__get_user_asm(*(u8*)dst,(u8 __user *)src,ret,"b","b","=q",1); + return copy_user_generic(dst, (__force void *)src, size); + switch (size) { + case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src, + ret, "b", "b", "=q", 1); return ret; - case 2:__get_user_asm(*(u16*)dst,(u16 __user *)src,ret,"w","w","=r",2); + case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src, + ret, "w", "w", "=r", 2); return ret; - case 4:__get_user_asm(*(u32*)dst,(u32 __user *)src,ret,"l","k","=r",4); + case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src, + ret, "l", "k", "=r", 4); + return ret; + case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src, + ret, "q", "", "=r", 8); return ret; - case 8:__get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",8); - return ret; case 10: - __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); - if (unlikely(ret)) return ret; - __get_user_asm(*(u16*)(8+(char*)dst),(u16 __user *)(8+(char __user *)src),ret,"w","w","=r",2); - return ret; + __get_user_asm(*(u64 *)dst, (u64 __user *)src, + ret, "q", "", "=r", 16); + if (unlikely(ret)) + return ret; + __get_user_asm(*(u16 *)(8 + (char *)dst), + (u16 __user *)(8 + (char __user *)src), + ret, "w", "w", "=r", 2); + return ret; case 16: - __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); - if (unlikely(ret)) return ret; - __get_user_asm(*(u64*)(8+(char*)dst),(u64 __user *)(8+(char __user *)src),ret,"q","","=r",8); - return ret; + __get_user_asm(*(u64 *)dst, (u64 __user *)src, + ret, "q", "", "=r", 16); + if (unlikely(ret)) + return ret; + __get_user_asm(*(u64 *)(8 + (char *)dst), + (u64 __user *)(8 + (char __user *)src), + ret, "q", "", "=r", 8); + return ret; default: - return copy_user_generic(dst,(__force void *)src,size); + return copy_user_generic(dst, (__force void *)src, size); } -} +} static __always_inline __must_check int __copy_to_user(void __user *dst, const void *src, unsigned size) -{ +{ int ret = 0; if (!__builtin_constant_p(size)) - return copy_user_generic((__force void *)dst,src,size); - switch (size) { - case 1:__put_user_asm(*(u8*)src,(u8 __user *)dst,ret,"b","b","iq",1); + return copy_user_generic((__force void *)dst, src, size); + switch (size) { + case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst, + ret, "b", "b", "iq", 1); return ret; - case 2:__put_user_asm(*(u16*)src,(u16 __user *)dst,ret,"w","w","ir",2); + case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst, + ret, "w", "w", "ir", 2); return ret; - case 4:__put_user_asm(*(u32*)src,(u32 __user *)dst,ret,"l","k","ir",4); + case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst, + ret, "l", "k", "ir", 4); + return ret; + case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst, + ret, "q", "", "ir", 8); return ret; - case 8:__put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",8); - return ret; case 10: - __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",10); - if (unlikely(ret)) return ret; + __put_user_asm(*(u64 *)src, (u64 __user *)dst, + ret, "q", "", "ir", 10); + if (unlikely(ret)) + return ret; asm("":::"memory"); - __put_user_asm(4[(u16*)src],4+(u16 __user *)dst,ret,"w","w","ir",2); - return ret; + __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, + ret, "w", "w", "ir", 2); + return ret; case 16: - __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",16); - if (unlikely(ret)) return ret; + __put_user_asm(*(u64 *)src, (u64 __user *)dst, + ret, "q", "", "ir", 16); + if (unlikely(ret)) + return ret; asm("":::"memory"); - __put_user_asm(1[(u64*)src],1+(u64 __user *)dst,ret,"q","","ir",8); - return ret; + __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, + ret, "q", "", "ir", 8); + return ret; default: - return copy_user_generic((__force void *)dst,src,size); + return copy_user_generic((__force void *)dst, src, size); } -} +} static __always_inline __must_check int __copy_in_user(void __user *dst, const void __user *src, unsigned size) -{ +{ int ret = 0; if (!__builtin_constant_p(size)) - return copy_user_generic((__force void *)dst,(__force void *)src,size); - switch (size) { - case 1: { + return copy_user_generic((__force void *)dst, + (__force void *)src, size); + switch (size) { + case 1: { u8 tmp; - __get_user_asm(tmp,(u8 __user *)src,ret,"b","b","=q",1); + __get_user_asm(tmp, (u8 __user *)src, + ret, "b", "b", "=q", 1); if (likely(!ret)) - __put_user_asm(tmp,(u8 __user *)dst,ret,"b","b","iq",1); + __put_user_asm(tmp, (u8 __user *)dst, + ret, "b", "b", "iq", 1); return ret; } - case 2: { + case 2: { u16 tmp; - __get_user_asm(tmp,(u16 __user *)src,ret,"w","w","=r",2); + __get_user_asm(tmp, (u16 __user *)src, + ret, "w", "w", "=r", 2); if (likely(!ret)) - __put_user_asm(tmp,(u16 __user *)dst,ret,"w","w","ir",2); + __put_user_asm(tmp, (u16 __user *)dst, + ret, "w", "w", "ir", 2); return ret; } - case 4: { + case 4: { u32 tmp; - __get_user_asm(tmp,(u32 __user *)src,ret,"l","k","=r",4); + __get_user_asm(tmp, (u32 __user *)src, + ret, "l", "k", "=r", 4); if (likely(!ret)) - __put_user_asm(tmp,(u32 __user *)dst,ret,"l","k","ir",4); + __put_user_asm(tmp, (u32 __user *)dst, + ret, "l", "k", "ir", 4); return ret; } - case 8: { + case 8: { u64 tmp; - __get_user_asm(tmp,(u64 __user *)src,ret,"q","","=r",8); + __get_user_asm(tmp, (u64 __user *)src, + ret, "q", "", "=r", 8); if (likely(!ret)) - __put_user_asm(tmp,(u64 __user *)dst,ret,"q","","ir",8); + __put_user_asm(tmp, (u64 __user *)dst, + ret, "q", "", "ir", 8); return ret; } default: - return copy_user_generic((__force void *)dst,(__force void *)src,size); + return copy_user_generic((__force void *)dst, + (__force void *)src, size); } -} +} -__must_check long +__must_check long strncpy_from_user(char *dst, const char __user *src, long count); -__must_check long +__must_check long __strncpy_from_user(char *dst, const char __user *src, long count); __must_check long strnlen_user(const char __user *str, long n); __must_check long __strnlen_user(const char __user *str, long n); @@ -355,7 +428,8 @@ __must_check long strlen_user(const char __user *str); __must_check unsigned long clear_user(void __user *mem, unsigned long len); __must_check unsigned long __clear_user(void __user *mem, unsigned long len); -__must_check long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size); +__must_check long __copy_from_user_inatomic(void *dst, const void __user *src, + unsigned size); static __must_check __always_inline int __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) @@ -364,15 +438,19 @@ __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) } #define ARCH_HAS_NOCACHE_UACCESS 1 -extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size, int zerorest); +extern long __copy_user_nocache(void *dst, const void __user *src, + unsigned size, int zerorest); -static inline int __copy_from_user_nocache(void *dst, const void __user *src, unsigned size) +static inline int __copy_from_user_nocache(void *dst, const void __user *src, + unsigned size) { might_sleep(); return __copy_user_nocache(dst, src, size, 1); } -static inline int __copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned size) +static inline int __copy_from_user_inatomic_nocache(void *dst, + const void __user *src, + unsigned size) { return __copy_user_nocache(dst, src, size, 0); } diff --git a/include/asm-x86/unaligned.h b/include/asm-x86/unaligned.h index 913598d4f76..a7bd416b476 100644 --- a/include/asm-x86/unaligned.h +++ b/include/asm-x86/unaligned.h @@ -3,35 +3,12 @@ /* * The x86 can do unaligned accesses itself. - * - * The strange macros are there to make sure these can't - * be misused in a way that makes them not work on other - * architectures where unaligned accesses aren't as simple. */ -/** - * get_unaligned - get value from possibly mis-aligned location - * @ptr: pointer to value - * - * This macro should be used for accessing values larger in size than - * single bytes at locations that are expected to be improperly aligned, - * e.g. retrieving a u16 value from a location not u16-aligned. - * - * Note that unaligned accesses can be very expensive on some architectures. - */ -#define get_unaligned(ptr) (*(ptr)) +#include <linux/unaligned/access_ok.h> +#include <linux/unaligned/generic.h> -/** - * put_unaligned - put value to a possibly mis-aligned location - * @val: value to place - * @ptr: pointer to location - * - * This macro should be used for placing values larger in size than - * single bytes at locations that are expected to be improperly aligned, - * e.g. writing a u16 value to a location not u16-aligned. - * - * Note that unaligned accesses can be very expensive on some architectures. - */ -#define put_unaligned(val, ptr) ((void)( *(ptr) = (val) )) +#define get_unaligned __get_unaligned_le +#define put_unaligned __put_unaligned_le #endif /* _ASM_X86_UNALIGNED_H */ diff --git a/include/asm-x86/unistd_32.h b/include/asm-x86/unistd_32.h index 984123a68f7..8317d94771d 100644 --- a/include/asm-x86/unistd_32.h +++ b/include/asm-x86/unistd_32.h @@ -81,7 +81,7 @@ #define __NR_sigpending 73 #define __NR_sethostname 74 #define __NR_setrlimit 75 -#define __NR_getrlimit 76 /* Back compatible 2Gig limited rlimit */ +#define __NR_getrlimit 76 /* Back compatible 2Gig limited rlimit */ #define __NR_getrusage 77 #define __NR_gettimeofday 78 #define __NR_settimeofday 79 diff --git a/include/asm-x86/unistd_64.h b/include/asm-x86/unistd_64.h index 3883ceb54ef..fe26e36d0f5 100644 --- a/include/asm-x86/unistd_64.h +++ b/include/asm-x86/unistd_64.h @@ -2,7 +2,7 @@ #define _ASM_X86_64_UNISTD_H_ #ifndef __SYSCALL -#define __SYSCALL(a,b) +#define __SYSCALL(a, b) #endif /* diff --git a/include/asm-x86/user32.h b/include/asm-x86/user32.h index f769872debe..a3d91004787 100644 --- a/include/asm-x86/user32.h +++ b/include/asm-x86/user32.h @@ -1,7 +1,8 @@ #ifndef USER32_H #define USER32_H 1 -/* IA32 compatible user structures for ptrace. These should be used for 32bit coredumps too. */ +/* IA32 compatible user structures for ptrace. + * These should be used for 32bit coredumps too. */ struct user_i387_ia32_struct { u32 cwd; @@ -42,9 +43,9 @@ struct user_regs_struct32 { }; struct user32 { - struct user_regs_struct32 regs; /* Where the registers are actually stored */ + struct user_regs_struct32 regs; /* Where the registers are actually stored */ int u_fpvalid; /* True if math co-processor being used. */ - /* for this mess. Not yet used. */ + /* for this mess. Not yet used. */ struct user_i387_ia32_struct i387; /* Math Co-processor registers. */ /* The rest of this junk is to help gdb figure out what goes where */ __u32 u_tsize; /* Text segment size (pages). */ diff --git a/include/asm-x86/user_32.h b/include/asm-x86/user_32.h index 6157da6f882..d6e51edc259 100644 --- a/include/asm-x86/user_32.h +++ b/include/asm-x86/user_32.h @@ -100,10 +100,10 @@ struct user_regs_struct { struct user{ /* We start with the registers, to mimic the way that "memory" is returned from the ptrace(3,...) function. */ - struct user_regs_struct regs; /* Where the registers are actually stored */ + struct user_regs_struct regs; /* Where the registers are actually stored */ /* ptrace does not yet supply these. Someday.... */ int u_fpvalid; /* True if math co-processor being used. */ - /* for this mess. Not yet used. */ + /* for this mess. Not yet used. */ struct user_i387_struct i387; /* Math Co-processor registers. */ /* The rest of this junk is to help gdb figure out what goes where */ unsigned long int u_tsize; /* Text segment size (pages). */ @@ -118,7 +118,7 @@ struct user{ int reserved; /* No longer used */ unsigned long u_ar0; /* Used by gdb to help find the values for */ /* the registers. */ - struct user_i387_struct* u_fpstate; /* Math Co-processor pointer. */ + struct user_i387_struct *u_fpstate; /* Math Co-processor pointer. */ unsigned long magic; /* To uniquely identify a core file */ char u_comm[32]; /* User command that was responsible */ int u_debugreg[8]; diff --git a/include/asm-x86/user_64.h b/include/asm-x86/user_64.h index 96361645560..6037b634c77 100644 --- a/include/asm-x86/user_64.h +++ b/include/asm-x86/user_64.h @@ -45,12 +45,13 @@ */ /* This matches the 64bit FXSAVE format as defined by AMD. It is the same - as the 32bit format defined by Intel, except that the selector:offset pairs for - data and eip are replaced with flat 64bit pointers. */ + as the 32bit format defined by Intel, except that the selector:offset pairs + for data and eip are replaced with flat 64bit pointers. */ struct user_i387_struct { unsigned short cwd; unsigned short swd; - unsigned short twd; /* Note this is not the same as the 32bit/x87/FSAVE twd */ + unsigned short twd; /* Note this is not the same as + the 32bit/x87/FSAVE twd */ unsigned short fop; __u64 rip; __u64 rdp; @@ -97,13 +98,14 @@ struct user_regs_struct { /* When the kernel dumps core, it starts by dumping the user struct - this will be used by gdb to figure out where the data and stack segments are within the file, and what virtual addresses to use. */ -struct user{ + +struct user { /* We start with the registers, to mimic the way that "memory" is returned from the ptrace(3,...) function. */ - struct user_regs_struct regs; /* Where the registers are actually stored */ + struct user_regs_struct regs; /* Where the registers are actually stored */ /* ptrace does not yet supply these. Someday.... */ int u_fpvalid; /* True if math co-processor being used. */ - /* for this mess. Not yet used. */ + /* for this mess. Not yet used. */ int pad0; struct user_i387_struct i387; /* Math Co-processor registers. */ /* The rest of this junk is to help gdb figure out what goes where */ @@ -120,7 +122,7 @@ struct user{ int pad1; unsigned long u_ar0; /* Used by gdb to help find the values for */ /* the registers. */ - struct user_i387_struct* u_fpstate; /* Math Co-processor pointer. */ + struct user_i387_struct *u_fpstate; /* Math Co-processor pointer. */ unsigned long magic; /* To uniquely identify a core file */ char u_comm[32]; /* User command that was responsible */ unsigned long u_debugreg[8]; diff --git a/include/asm-x86/uv/uv_hub.h b/include/asm-x86/uv/uv_hub.h new file mode 100644 index 00000000000..26b9240d1e2 --- /dev/null +++ b/include/asm-x86/uv/uv_hub.h @@ -0,0 +1,284 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * SGI UV architectural definitions + * + * Copyright (C) 2007 Silicon Graphics, Inc. All rights reserved. + */ + +#ifndef __ASM_X86_UV_HUB_H__ +#define __ASM_X86_UV_HUB_H__ + +#include <linux/numa.h> +#include <linux/percpu.h> +#include <asm/types.h> +#include <asm/percpu.h> + + +/* + * Addressing Terminology + * + * NASID - network ID of a router, Mbrick or Cbrick. Nasid values of + * routers always have low bit of 1, C/MBricks have low bit + * equal to 0. Most addressing macros that target UV hub chips + * right shift the NASID by 1 to exclude the always-zero bit. + * + * SNASID - NASID right shifted by 1 bit. + * + * + * Memory/UV-HUB Processor Socket Address Format: + * +--------+---------------+---------------------+ + * |00..0000| SNASID | NodeOffset | + * +--------+---------------+---------------------+ + * <--- N bits --->|<--------M bits -----> + * + * M number of node offset bits (35 .. 40) + * N number of SNASID bits (0 .. 10) + * + * Note: M + N cannot currently exceed 44 (x86_64) or 46 (IA64). + * The actual values are configuration dependent and are set at + * boot time + * + * APICID format + * NOTE!!!!!! This is the current format of the APICID. However, code + * should assume that this will change in the future. Use functions + * in this file for all APICID bit manipulations and conversion. + * + * 1111110000000000 + * 5432109876543210 + * nnnnnnnnnnlc0cch + * sssssssssss + * + * n = snasid bits + * l = socket number on board + * c = core + * h = hyperthread + * s = bits that are in the socket CSR + * + * Note: Processor only supports 12 bits in the APICID register. The ACPI + * tables hold all 16 bits. Software needs to be aware of this. + * + * Unless otherwise specified, all references to APICID refer to + * the FULL value contained in ACPI tables, not the subset in the + * processor APICID register. + */ + + +/* + * Maximum number of bricks in all partitions and in all coherency domains. + * This is the total number of bricks accessible in the numalink fabric. It + * includes all C & M bricks. Routers are NOT included. + * + * This value is also the value of the maximum number of non-router NASIDs + * in the numalink fabric. + * + * NOTE: a brick may be 1 or 2 OS nodes. Don't get these confused. + */ +#define UV_MAX_NUMALINK_BLADES 16384 + +/* + * Maximum number of C/Mbricks within a software SSI (hardware may support + * more). + */ +#define UV_MAX_SSI_BLADES 256 + +/* + * The largest possible NASID of a C or M brick (+ 2) + */ +#define UV_MAX_NASID_VALUE (UV_MAX_NUMALINK_NODES * 2) + +/* + * The following defines attributes of the HUB chip. These attributes are + * frequently referenced and are kept in the per-cpu data areas of each cpu. + * They are kept together in a struct to minimize cache misses. + */ +struct uv_hub_info_s { + unsigned long global_mmr_base; + unsigned short local_nasid; + unsigned short gnode_upper; + unsigned short coherency_domain_number; + unsigned short numa_blade_id; + unsigned char blade_processor_id; + unsigned char m_val; + unsigned char n_val; +}; +DECLARE_PER_CPU(struct uv_hub_info_s, __uv_hub_info); +#define uv_hub_info (&__get_cpu_var(__uv_hub_info)) +#define uv_cpu_hub_info(cpu) (&per_cpu(__uv_hub_info, cpu)) + +/* + * Local & Global MMR space macros. + * Note: macros are intended to be used ONLY by inline functions + * in this file - not by other kernel code. + */ +#define UV_SNASID(n) ((n) >> 1) +#define UV_NASID(n) ((n) << 1) + +#define UV_LOCAL_MMR_BASE 0xf4000000UL +#define UV_GLOBAL_MMR32_BASE 0xf8000000UL +#define UV_GLOBAL_MMR64_BASE (uv_hub_info->global_mmr_base) + +#define UV_GLOBAL_MMR32_SNASID_MASK 0x3ff +#define UV_GLOBAL_MMR32_SNASID_SHIFT 15 +#define UV_GLOBAL_MMR64_SNASID_SHIFT 26 + +#define UV_GLOBAL_MMR32_NASID_BITS(n) \ + (((UV_SNASID(n) & UV_GLOBAL_MMR32_SNASID_MASK)) << \ + (UV_GLOBAL_MMR32_SNASID_SHIFT)) + +#define UV_GLOBAL_MMR64_NASID_BITS(n) \ + ((unsigned long)UV_SNASID(n) << UV_GLOBAL_MMR64_SNASID_SHIFT) + +#define UV_APIC_NASID_SHIFT 6 + +/* + * Extract a NASID from an APICID (full apicid, not processor subset) + */ +static inline int uv_apicid_to_nasid(int apicid) +{ + return (UV_NASID(apicid >> UV_APIC_NASID_SHIFT)); +} + +/* + * Access global MMRs using the low memory MMR32 space. This region supports + * faster MMR access but not all MMRs are accessible in this space. + */ +static inline unsigned long *uv_global_mmr32_address(int nasid, + unsigned long offset) +{ + return __va(UV_GLOBAL_MMR32_BASE | + UV_GLOBAL_MMR32_NASID_BITS(nasid) | offset); +} + +static inline void uv_write_global_mmr32(int nasid, unsigned long offset, + unsigned long val) +{ + *uv_global_mmr32_address(nasid, offset) = val; +} + +static inline unsigned long uv_read_global_mmr32(int nasid, + unsigned long offset) +{ + return *uv_global_mmr32_address(nasid, offset); +} + +/* + * Access Global MMR space using the MMR space located at the top of physical + * memory. + */ +static inline unsigned long *uv_global_mmr64_address(int nasid, + unsigned long offset) +{ + return __va(UV_GLOBAL_MMR64_BASE | + UV_GLOBAL_MMR64_NASID_BITS(nasid) | offset); +} + +static inline void uv_write_global_mmr64(int nasid, unsigned long offset, + unsigned long val) +{ + *uv_global_mmr64_address(nasid, offset) = val; +} + +static inline unsigned long uv_read_global_mmr64(int nasid, + unsigned long offset) +{ + return *uv_global_mmr64_address(nasid, offset); +} + +/* + * Access node local MMRs. Faster than using global space but only local MMRs + * are accessible. + */ +static inline unsigned long *uv_local_mmr_address(unsigned long offset) +{ + return __va(UV_LOCAL_MMR_BASE | offset); +} + +static inline unsigned long uv_read_local_mmr(unsigned long offset) +{ + return *uv_local_mmr_address(offset); +} + +static inline void uv_write_local_mmr(unsigned long offset, unsigned long val) +{ + *uv_local_mmr_address(offset) = val; +} + +/* + * Structures and definitions for converting between cpu, node, and blade + * numbers. + */ +struct uv_blade_info { + unsigned short nr_posible_cpus; + unsigned short nr_online_cpus; + unsigned short nasid; +}; +struct uv_blade_info *uv_blade_info; +extern short *uv_node_to_blade; +extern short *uv_cpu_to_blade; +extern short uv_possible_blades; + +/* Blade-local cpu number of current cpu. Numbered 0 .. <# cpus on the blade> */ +static inline int uv_blade_processor_id(void) +{ + return uv_hub_info->blade_processor_id; +} + +/* Blade number of current cpu. Numnbered 0 .. <#blades -1> */ +static inline int uv_numa_blade_id(void) +{ + return uv_hub_info->numa_blade_id; +} + +/* Convert a cpu number to the the UV blade number */ +static inline int uv_cpu_to_blade_id(int cpu) +{ + return uv_cpu_to_blade[cpu]; +} + +/* Convert linux node number to the UV blade number */ +static inline int uv_node_to_blade_id(int nid) +{ + return uv_node_to_blade[nid]; +} + +/* Convert a blade id to the NASID of the blade */ +static inline int uv_blade_to_nasid(int bid) +{ + return uv_blade_info[bid].nasid; +} + +/* Determine the number of possible cpus on a blade */ +static inline int uv_blade_nr_possible_cpus(int bid) +{ + return uv_blade_info[bid].nr_posible_cpus; +} + +/* Determine the number of online cpus on a blade */ +static inline int uv_blade_nr_online_cpus(int bid) +{ + return uv_blade_info[bid].nr_online_cpus; +} + +/* Convert a cpu id to the NASID of the blade containing the cpu */ +static inline int uv_cpu_to_nasid(int cpu) +{ + return uv_blade_info[uv_cpu_to_blade_id(cpu)].nasid; +} + +/* Convert a node number to the NASID of the blade */ +static inline int uv_node_to_nasid(int nid) +{ + return uv_blade_info[uv_node_to_blade_id(nid)].nasid; +} + +/* Maximum possible number of blades */ +static inline int uv_num_possible_blades(void) +{ + return uv_possible_blades; +} + +#endif /* __ASM_X86_UV_HUB__ */ + diff --git a/include/asm-x86/uv/uv_mmrs.h b/include/asm-x86/uv/uv_mmrs.h new file mode 100644 index 00000000000..3b69fe6b637 --- /dev/null +++ b/include/asm-x86/uv/uv_mmrs.h @@ -0,0 +1,373 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * SGI UV MMR definitions + * + * Copyright (C) 2007-2008 Silicon Graphics, Inc. All rights reserved. + */ + +#ifndef __ASM_X86_UV_MMRS__ +#define __ASM_X86_UV_MMRS__ + +/* + * AUTO GENERATED - Do not edit + */ + + #define UV_MMR_ENABLE (1UL << 63) + +/* ========================================================================= */ +/* UVH_IPI_INT */ +/* ========================================================================= */ +#define UVH_IPI_INT 0x60500UL +#define UVH_IPI_INT_32 0x0360 + +#define UVH_IPI_INT_VECTOR_SHFT 0 +#define UVH_IPI_INT_VECTOR_MASK 0x00000000000000ffUL +#define UVH_IPI_INT_DELIVERY_MODE_SHFT 8 +#define UVH_IPI_INT_DELIVERY_MODE_MASK 0x0000000000000700UL +#define UVH_IPI_INT_DESTMODE_SHFT 11 +#define UVH_IPI_INT_DESTMODE_MASK 0x0000000000000800UL +#define UVH_IPI_INT_APIC_ID_SHFT 16 +#define UVH_IPI_INT_APIC_ID_MASK 0x0000ffffffff0000UL +#define UVH_IPI_INT_SEND_SHFT 63 +#define UVH_IPI_INT_SEND_MASK 0x8000000000000000UL + +union uvh_ipi_int_u { + unsigned long v; + struct uvh_ipi_int_s { + unsigned long vector_ : 8; /* RW */ + unsigned long delivery_mode : 3; /* RW */ + unsigned long destmode : 1; /* RW */ + unsigned long rsvd_12_15 : 4; /* */ + unsigned long apic_id : 32; /* RW */ + unsigned long rsvd_48_62 : 15; /* */ + unsigned long send : 1; /* WP */ + } s; +}; + +/* ========================================================================= */ +/* UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST */ +/* ========================================================================= */ +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST 0x320050UL +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_32 0x009f0 + +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_ADDRESS_SHFT 4 +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_ADDRESS_MASK 0x000007fffffffff0UL +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_NODE_ID_SHFT 49 +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_NODE_ID_MASK 0x7ffe000000000000UL + +union uvh_lb_bau_intd_payload_queue_first_u { + unsigned long v; + struct uvh_lb_bau_intd_payload_queue_first_s { + unsigned long rsvd_0_3: 4; /* */ + unsigned long address : 39; /* RW */ + unsigned long rsvd_43_48: 6; /* */ + unsigned long node_id : 14; /* RW */ + unsigned long rsvd_63 : 1; /* */ + } s; +}; + +/* ========================================================================= */ +/* UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST */ +/* ========================================================================= */ +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST 0x320060UL +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_32 0x009f8 + +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_ADDRESS_SHFT 4 +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_ADDRESS_MASK 0x000007fffffffff0UL + +union uvh_lb_bau_intd_payload_queue_last_u { + unsigned long v; + struct uvh_lb_bau_intd_payload_queue_last_s { + unsigned long rsvd_0_3: 4; /* */ + unsigned long address : 39; /* RW */ + unsigned long rsvd_43_63: 21; /* */ + } s; +}; + +/* ========================================================================= */ +/* UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL */ +/* ========================================================================= */ +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL 0x320070UL +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_32 0x00a00 + +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_ADDRESS_SHFT 4 +#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_ADDRESS_MASK 0x000007fffffffff0UL + +union uvh_lb_bau_intd_payload_queue_tail_u { + unsigned long v; + struct uvh_lb_bau_intd_payload_queue_tail_s { + unsigned long rsvd_0_3: 4; /* */ + unsigned long address : 39; /* RW */ + unsigned long rsvd_43_63: 21; /* */ + } s; +}; + +/* ========================================================================= */ +/* UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE */ +/* ========================================================================= */ +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE 0x320080UL + +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_0_SHFT 0 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_0_MASK 0x0000000000000001UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_1_SHFT 1 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_1_MASK 0x0000000000000002UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_2_SHFT 2 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_2_MASK 0x0000000000000004UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_3_SHFT 3 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_3_MASK 0x0000000000000008UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_4_SHFT 4 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_4_MASK 0x0000000000000010UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_5_SHFT 5 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_5_MASK 0x0000000000000020UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_6_SHFT 6 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_6_MASK 0x0000000000000040UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_7_SHFT 7 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_7_MASK 0x0000000000000080UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_0_SHFT 8 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_0_MASK 0x0000000000000100UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_1_SHFT 9 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_1_MASK 0x0000000000000200UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_2_SHFT 10 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_2_MASK 0x0000000000000400UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_3_SHFT 11 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_3_MASK 0x0000000000000800UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_4_SHFT 12 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_4_MASK 0x0000000000001000UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_5_SHFT 13 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_5_MASK 0x0000000000002000UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_6_SHFT 14 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_6_MASK 0x0000000000004000UL +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_7_SHFT 15 +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_7_MASK 0x0000000000008000UL +union uvh_lb_bau_intd_software_acknowledge_u { + unsigned long v; + struct uvh_lb_bau_intd_software_acknowledge_s { + unsigned long pending_0 : 1; /* RW, W1C */ + unsigned long pending_1 : 1; /* RW, W1C */ + unsigned long pending_2 : 1; /* RW, W1C */ + unsigned long pending_3 : 1; /* RW, W1C */ + unsigned long pending_4 : 1; /* RW, W1C */ + unsigned long pending_5 : 1; /* RW, W1C */ + unsigned long pending_6 : 1; /* RW, W1C */ + unsigned long pending_7 : 1; /* RW, W1C */ + unsigned long timeout_0 : 1; /* RW, W1C */ + unsigned long timeout_1 : 1; /* RW, W1C */ + unsigned long timeout_2 : 1; /* RW, W1C */ + unsigned long timeout_3 : 1; /* RW, W1C */ + unsigned long timeout_4 : 1; /* RW, W1C */ + unsigned long timeout_5 : 1; /* RW, W1C */ + unsigned long timeout_6 : 1; /* RW, W1C */ + unsigned long timeout_7 : 1; /* RW, W1C */ + unsigned long rsvd_16_63: 48; /* */ + } s; +}; + +/* ========================================================================= */ +/* UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS */ +/* ========================================================================= */ +#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS 0x0000000000320088UL + +/* ========================================================================= */ +/* UVH_LB_BAU_SB_ACTIVATION_CONTROL */ +/* ========================================================================= */ +#define UVH_LB_BAU_SB_ACTIVATION_CONTROL 0x320020UL +#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_32 0x009d8 + +#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_INDEX_SHFT 0 +#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_INDEX_MASK 0x000000000000003fUL +#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT 62 +#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_MASK 0x4000000000000000UL +#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_INIT_SHFT 63 +#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_INIT_MASK 0x8000000000000000UL + +union uvh_lb_bau_sb_activation_control_u { + unsigned long v; + struct uvh_lb_bau_sb_activation_control_s { + unsigned long index : 6; /* RW */ + unsigned long rsvd_6_61: 56; /* */ + unsigned long push : 1; /* WP */ + unsigned long init : 1; /* WP */ + } s; +}; + +/* ========================================================================= */ +/* UVH_LB_BAU_SB_ACTIVATION_STATUS_0 */ +/* ========================================================================= */ +#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0 0x320030UL +#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0_32 0x009e0 + +#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0_STATUS_SHFT 0 +#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0_STATUS_MASK 0xffffffffffffffffUL + +union uvh_lb_bau_sb_activation_status_0_u { + unsigned long v; + struct uvh_lb_bau_sb_activation_status_0_s { + unsigned long status : 64; /* RW */ + } s; +}; + +/* ========================================================================= */ +/* UVH_LB_BAU_SB_ACTIVATION_STATUS_1 */ +/* ========================================================================= */ +#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1 0x320040UL +#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1_32 0x009e8 + +#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1_STATUS_SHFT 0 +#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1_STATUS_MASK 0xffffffffffffffffUL + +union uvh_lb_bau_sb_activation_status_1_u { + unsigned long v; + struct uvh_lb_bau_sb_activation_status_1_s { + unsigned long status : 64; /* RW */ + } s; +}; + +/* ========================================================================= */ +/* UVH_LB_BAU_SB_DESCRIPTOR_BASE */ +/* ========================================================================= */ +#define UVH_LB_BAU_SB_DESCRIPTOR_BASE 0x320010UL +#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_32 0x009d0 + +#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_SHFT 12 +#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_MASK 0x000007fffffff000UL +#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT 49 +#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK 0x7ffe000000000000UL + +union uvh_lb_bau_sb_descriptor_base_u { + unsigned long v; + struct uvh_lb_bau_sb_descriptor_base_s { + unsigned long rsvd_0_11 : 12; /* */ + unsigned long page_address : 31; /* RW */ + unsigned long rsvd_43_48 : 6; /* */ + unsigned long node_id : 14; /* RW */ + unsigned long rsvd_63 : 1; /* */ + } s; +}; + +/* ========================================================================= */ +/* UVH_NODE_ID */ +/* ========================================================================= */ +#define UVH_NODE_ID 0x0UL + +#define UVH_NODE_ID_FORCE1_SHFT 0 +#define UVH_NODE_ID_FORCE1_MASK 0x0000000000000001UL +#define UVH_NODE_ID_MANUFACTURER_SHFT 1 +#define UVH_NODE_ID_MANUFACTURER_MASK 0x0000000000000ffeUL +#define UVH_NODE_ID_PART_NUMBER_SHFT 12 +#define UVH_NODE_ID_PART_NUMBER_MASK 0x000000000ffff000UL +#define UVH_NODE_ID_REVISION_SHFT 28 +#define UVH_NODE_ID_REVISION_MASK 0x00000000f0000000UL +#define UVH_NODE_ID_NODE_ID_SHFT 32 +#define UVH_NODE_ID_NODE_ID_MASK 0x00007fff00000000UL +#define UVH_NODE_ID_NODES_PER_BIT_SHFT 48 +#define UVH_NODE_ID_NODES_PER_BIT_MASK 0x007f000000000000UL +#define UVH_NODE_ID_NI_PORT_SHFT 56 +#define UVH_NODE_ID_NI_PORT_MASK 0x0f00000000000000UL + +union uvh_node_id_u { + unsigned long v; + struct uvh_node_id_s { + unsigned long force1 : 1; /* RO */ + unsigned long manufacturer : 11; /* RO */ + unsigned long part_number : 16; /* RO */ + unsigned long revision : 4; /* RO */ + unsigned long node_id : 15; /* RW */ + unsigned long rsvd_47 : 1; /* */ + unsigned long nodes_per_bit : 7; /* RW */ + unsigned long rsvd_55 : 1; /* */ + unsigned long ni_port : 4; /* RO */ + unsigned long rsvd_60_63 : 4; /* */ + } s; +}; + +/* ========================================================================= */ +/* UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR */ +/* ========================================================================= */ +#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR 0x1600010UL + +#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT 28 +#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffff0000000UL +#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_GR4_SHFT 46 +#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_GR4_MASK 0x0000400000000000UL +#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_SHFT 52 +#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_MASK 0x00f0000000000000UL +#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63 +#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL + +union uvh_rh_gam_gru_overlay_config_mmr_u { + unsigned long v; + struct uvh_rh_gam_gru_overlay_config_mmr_s { + unsigned long rsvd_0_27: 28; /* */ + unsigned long base : 18; /* RW */ + unsigned long gr4 : 1; /* RW */ + unsigned long rsvd_47_51: 5; /* */ + unsigned long n_gru : 4; /* RW */ + unsigned long rsvd_56_62: 7; /* */ + unsigned long enable : 1; /* RW */ + } s; +}; + +/* ========================================================================= */ +/* UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR */ +/* ========================================================================= */ +#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR 0x1600028UL + +#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT 26 +#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffffc000000UL +#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_DUAL_HUB_SHFT 46 +#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_DUAL_HUB_MASK 0x0000400000000000UL +#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63 +#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL + +union uvh_rh_gam_mmr_overlay_config_mmr_u { + unsigned long v; + struct uvh_rh_gam_mmr_overlay_config_mmr_s { + unsigned long rsvd_0_25: 26; /* */ + unsigned long base : 20; /* RW */ + unsigned long dual_hub : 1; /* RW */ + unsigned long rsvd_47_62: 16; /* */ + unsigned long enable : 1; /* RW */ + } s; +}; + +/* ========================================================================= */ +/* UVH_RTC */ +/* ========================================================================= */ +#define UVH_RTC 0x28000UL + +#define UVH_RTC_REAL_TIME_CLOCK_SHFT 0 +#define UVH_RTC_REAL_TIME_CLOCK_MASK 0x00ffffffffffffffUL + +union uvh_rtc_u { + unsigned long v; + struct uvh_rtc_s { + unsigned long real_time_clock : 56; /* RW */ + unsigned long rsvd_56_63 : 8; /* */ + } s; +}; + +/* ========================================================================= */ +/* UVH_SI_ADDR_MAP_CONFIG */ +/* ========================================================================= */ +#define UVH_SI_ADDR_MAP_CONFIG 0xc80000UL + +#define UVH_SI_ADDR_MAP_CONFIG_M_SKT_SHFT 0 +#define UVH_SI_ADDR_MAP_CONFIG_M_SKT_MASK 0x000000000000003fUL +#define UVH_SI_ADDR_MAP_CONFIG_N_SKT_SHFT 8 +#define UVH_SI_ADDR_MAP_CONFIG_N_SKT_MASK 0x0000000000000f00UL + +union uvh_si_addr_map_config_u { + unsigned long v; + struct uvh_si_addr_map_config_s { + unsigned long m_skt : 6; /* RW */ + unsigned long rsvd_6_7: 2; /* */ + unsigned long n_skt : 4; /* RW */ + unsigned long rsvd_12_63: 52; /* */ + } s; +}; + + +#endif /* __ASM_X86_UV_MMRS__ */ diff --git a/include/asm-x86/vdso.h b/include/asm-x86/vdso.h index 629bcb6e8e4..86e085e003d 100644 --- a/include/asm-x86/vdso.h +++ b/include/asm-x86/vdso.h @@ -8,9 +8,11 @@ extern const char VDSO64_PRELINK[]; * Given a pointer to the vDSO image, find the pointer to VDSO64_name * as that symbol is defined in the vDSO sources or linker script. */ -#define VDSO64_SYMBOL(base, name) ({ \ - extern const char VDSO64_##name[]; \ - (void *) (VDSO64_##name - VDSO64_PRELINK + (unsigned long) (base)); }) +#define VDSO64_SYMBOL(base, name) \ +({ \ + extern const char VDSO64_##name[]; \ + (void *)(VDSO64_##name - VDSO64_PRELINK + (unsigned long)(base)); \ +}) #endif #if defined CONFIG_X86_32 || defined CONFIG_COMPAT @@ -20,9 +22,18 @@ extern const char VDSO32_PRELINK[]; * Given a pointer to the vDSO image, find the pointer to VDSO32_name * as that symbol is defined in the vDSO sources or linker script. */ -#define VDSO32_SYMBOL(base, name) ({ \ - extern const char VDSO32_##name[]; \ - (void *) (VDSO32_##name - VDSO32_PRELINK + (unsigned long) (base)); }) +#define VDSO32_SYMBOL(base, name) \ +({ \ + extern const char VDSO32_##name[]; \ + (void *)(VDSO32_##name - VDSO32_PRELINK + (unsigned long)(base)); \ +}) #endif +/* + * These symbols are defined with the addresses in the vsyscall page. + * See vsyscall-sigreturn.S. + */ +extern void __user __kernel_sigreturn; +extern void __user __kernel_rt_sigreturn; + #endif /* asm-x86/vdso.h */ diff --git a/include/asm-x86/vga.h b/include/asm-x86/vga.h index 0ecf68ac03a..0ccf804377e 100644 --- a/include/asm-x86/vga.h +++ b/include/asm-x86/vga.h @@ -12,9 +12,9 @@ * access the videoram directly without any black magic. */ -#define VGA_MAP_MEM(x,s) (unsigned long)phys_to_virt(x) +#define VGA_MAP_MEM(x, s) (unsigned long)phys_to_virt(x) #define vga_readb(x) (*(x)) -#define vga_writeb(x,y) (*(y) = (x)) +#define vga_writeb(x, y) (*(y) = (x)) #endif diff --git a/include/asm-x86/vm86.h b/include/asm-x86/vm86.h index c92fe4af52e..074b357146d 100644 --- a/include/asm-x86/vm86.h +++ b/include/asm-x86/vm86.h @@ -12,19 +12,13 @@ * Linus */ -#define TF_MASK 0x00000100 -#define IF_MASK 0x00000200 -#define IOPL_MASK 0x00003000 -#define NT_MASK 0x00004000 +#include <asm/processor-flags.h> + #ifdef CONFIG_VM86 -#define VM_MASK 0x00020000 +#define X86_VM_MASK X86_EFLAGS_VM #else -#define VM_MASK 0 /* ignored */ +#define X86_VM_MASK 0 /* No VM86 support */ #endif -#define AC_MASK 0x00040000 -#define VIF_MASK 0x00080000 /* virtual interrupt flag */ -#define VIP_MASK 0x00100000 /* virtual interrupt pending */ -#define ID_MASK 0x00200000 #define BIOSSEG 0x0f000 @@ -42,9 +36,11 @@ #define VM86_ARG(retval) ((retval) >> 8) #define VM86_SIGNAL 0 /* return due to signal */ -#define VM86_UNKNOWN 1 /* unhandled GP fault - IO-instruction or similar */ +#define VM86_UNKNOWN 1 /* unhandled GP fault + - IO-instruction or similar */ #define VM86_INTx 2 /* int3/int x instruction (ARG = x) */ -#define VM86_STI 3 /* sti/popf/iret instruction enabled virtual interrupts */ +#define VM86_STI 3 /* sti/popf/iret instruction enabled + virtual interrupts */ /* * Additional return values when invoking new vm86() @@ -205,7 +201,8 @@ void release_vm86_irqs(struct task_struct *); #define handle_vm86_fault(a, b) #define release_vm86_irqs(a) -static inline int handle_vm86_trap(struct kernel_vm86_regs *a, long b, int c) { +static inline int handle_vm86_trap(struct kernel_vm86_regs *a, long b, int c) +{ return 0; } diff --git a/include/asm-x86/vmi.h b/include/asm-x86/vmi.h index eb8bd892c01..b7c0dea119f 100644 --- a/include/asm-x86/vmi.h +++ b/include/asm-x86/vmi.h @@ -155,9 +155,9 @@ #ifndef __ASSEMBLY__ struct vmi_relocation_info { - unsigned char *eip; - unsigned char type; - unsigned char reserved[3]; + unsigned char *eip; + unsigned char type; + unsigned char reserved[3]; }; #endif @@ -173,53 +173,53 @@ struct vmi_relocation_info { #ifndef __ASSEMBLY__ struct vrom_header { - u16 rom_signature; // option ROM signature - u8 rom_length; // ROM length in 512 byte chunks - u8 rom_entry[4]; // 16-bit code entry point - u8 rom_pad0; // 4-byte align pad - u32 vrom_signature; // VROM identification signature - u8 api_version_min;// Minor version of API - u8 api_version_maj;// Major version of API - u8 jump_slots; // Number of jump slots - u8 reserved1; // Reserved for expansion - u32 virtual_top; // Hypervisor virtual address start - u16 reserved2; // Reserved for expansion - u16 license_offs; // Offset to License string - u16 pci_header_offs;// Offset to PCI OPROM header - u16 pnp_header_offs;// Offset to PnP OPROM header - u32 rom_pad3; // PnP reserverd / VMI reserved - u8 reserved[96]; // Reserved for headers - char vmi_init[8]; // VMI_Init jump point - char get_reloc[8]; // VMI_GetRelocationInfo jump point + u16 rom_signature; /* option ROM signature */ + u8 rom_length; /* ROM length in 512 byte chunks */ + u8 rom_entry[4]; /* 16-bit code entry point */ + u8 rom_pad0; /* 4-byte align pad */ + u32 vrom_signature; /* VROM identification signature */ + u8 api_version_min;/* Minor version of API */ + u8 api_version_maj;/* Major version of API */ + u8 jump_slots; /* Number of jump slots */ + u8 reserved1; /* Reserved for expansion */ + u32 virtual_top; /* Hypervisor virtual address start */ + u16 reserved2; /* Reserved for expansion */ + u16 license_offs; /* Offset to License string */ + u16 pci_header_offs;/* Offset to PCI OPROM header */ + u16 pnp_header_offs;/* Offset to PnP OPROM header */ + u32 rom_pad3; /* PnP reserverd / VMI reserved */ + u8 reserved[96]; /* Reserved for headers */ + char vmi_init[8]; /* VMI_Init jump point */ + char get_reloc[8]; /* VMI_GetRelocationInfo jump point */ } __attribute__((packed)); struct pnp_header { - char sig[4]; - char rev; - char size; - short next; - short res; - long devID; - unsigned short manufacturer_offset; - unsigned short product_offset; + char sig[4]; + char rev; + char size; + short next; + short res; + long devID; + unsigned short manufacturer_offset; + unsigned short product_offset; } __attribute__((packed)); struct pci_header { - char sig[4]; - short vendorID; - short deviceID; - short vpdData; - short size; - char rev; - char class; - char subclass; - char interface; - short chunks; - char rom_version_min; - char rom_version_maj; - char codetype; - char lastRom; - short reserved; + char sig[4]; + short vendorID; + short deviceID; + short vpdData; + short size; + char rev; + char class; + char subclass; + char interface; + short chunks; + char rom_version_min; + char rom_version_maj; + char codetype; + char lastRom; + short reserved; } __attribute__((packed)); /* Function prototypes for bootstrapping */ diff --git a/include/asm-x86/voyager.h b/include/asm-x86/voyager.h index 91a9932937a..9c811d2e6f9 100644 --- a/include/asm-x86/voyager.h +++ b/include/asm-x86/voyager.h @@ -91,8 +91,7 @@ #define VOYAGER_WRITE_CONFIG 0x2 #define VOYAGER_BYPASS 0xff -typedef struct voyager_asic -{ +typedef struct voyager_asic { __u8 asic_addr; /* ASIC address; Level 4 */ __u8 asic_type; /* ASIC type */ __u8 asic_id; /* ASIC id */ @@ -113,7 +112,7 @@ typedef struct voyager_module { __u16 largest_reg; /* Largest register in the scan path */ __u16 smallest_reg; /* Smallest register in the scan path */ voyager_asic_t *asic; /* First ASIC in scan path (CAT_I) */ - struct voyager_module *submodule; /* Submodule pointer */ + struct voyager_module *submodule; /* Submodule pointer */ struct voyager_module *next; /* Next module in linked list */ } voyager_module_t; @@ -135,7 +134,7 @@ typedef struct voyager_eeprom_hdr { __u16 cct_offset; __u16 log_length; /* length of err log */ __u16 xsum_end; /* offset to end of - checksum */ + checksum */ __u8 reserved[4]; __u8 sflag; /* starting sentinal */ __u8 part_number[13]; /* prom part number */ @@ -148,7 +147,8 @@ typedef struct voyager_eeprom_hdr { -#define VOYAGER_EPROM_SIZE_OFFSET ((__u16)(&(((voyager_eprom_hdr_t *)0)->ee_size))) +#define VOYAGER_EPROM_SIZE_OFFSET \ + ((__u16)(&(((voyager_eprom_hdr_t *)0)->ee_size))) #define VOYAGER_XSUM_END_OFFSET 0x2a /* the following three definitions are for internal table layouts @@ -199,7 +199,7 @@ typedef struct voyager_asic_data_table { #define VOYAGER_WCBIC_TOM_L 0x4 #define VOYAGER_WCBIC_TOM_H 0x5 -/* register defines for Voyager Memory Contol (VMC) +/* register defines for Voyager Memory Contol (VMC) * these are present on L4 machines only */ #define VOYAGER_VMC1 0x81 #define VOYAGER_VMC2 0x91 @@ -334,7 +334,7 @@ typedef struct { struct QuadDescription { __u8 Type; /* for type 0 (DYADIC or MONADIC) all fields - * will be zero except for slot */ + * will be zero except for slot */ __u8 StructureVersion; __u32 CPI_BaseAddress; __u32 LARC_BankSize; @@ -342,7 +342,7 @@ struct QuadDescription { __u8 Slot; /* Processor slots 1 - 4 */ } __attribute__((packed)); -struct ProcBoardInfo { +struct ProcBoardInfo { __u8 Type; __u8 StructureVersion; __u8 NumberOfBoards; @@ -382,19 +382,30 @@ struct CPU_Info { * packed in it by our friend the compiler. */ typedef struct { - __u8 Mailbox_SUS; /* Written to by SUS to give commands/response to the OS */ - __u8 Mailbox_OS; /* Written to by the OS to give commands/response to SUS */ - __u8 SUS_MailboxVersion; /* Tells the OS which iteration of the interface SUS supports */ - __u8 OS_MailboxVersion; /* Tells SUS which iteration of the interface the OS supports */ - __u32 OS_Flags; /* Flags set by the OS as info for SUS */ - __u32 SUS_Flags; /* Flags set by SUS as info for the OS */ - __u32 WatchDogPeriod; /* Watchdog period (in seconds) which the DP uses to see if the OS is dead */ + __u8 Mailbox_SUS; /* Written to by SUS to give + commands/response to the OS */ + __u8 Mailbox_OS; /* Written to by the OS to give + commands/response to SUS */ + __u8 SUS_MailboxVersion; /* Tells the OS which iteration of the + interface SUS supports */ + __u8 OS_MailboxVersion; /* Tells SUS which iteration of the + interface the OS supports */ + __u32 OS_Flags; /* Flags set by the OS as info for + SUS */ + __u32 SUS_Flags; /* Flags set by SUS as info + for the OS */ + __u32 WatchDogPeriod; /* Watchdog period (in seconds) which + the DP uses to see if the OS + is dead */ __u32 WatchDogCount; /* Updated by the OS on every tic. */ - __u32 MemoryFor_SUS_ErrorLog; /* Flat 32 bit address which tells SUS where to stuff the SUS error log on a dump */ - MC_SlotInformation_t MC_SlotInfo[NUMBER_OF_MC_BUSSES*SLOTS_PER_MC_BUS]; /* Storage for MCA POS data */ + __u32 MemoryFor_SUS_ErrorLog; /* Flat 32 bit address which tells SUS + where to stuff the SUS error log + on a dump */ + MC_SlotInformation_t MC_SlotInfo[NUMBER_OF_MC_BUSSES*SLOTS_PER_MC_BUS]; + /* Storage for MCA POS data */ /* All new SECOND_PASS_INTERFACE fields added from this point */ - struct ProcBoardInfo *BoardData; - struct CPU_Info *CPU_Data; + struct ProcBoardInfo *BoardData; + struct CPU_Info *CPU_Data; /* All new fields must be added from this point */ } Voyager_KernelSUS_Mbox_t; @@ -478,7 +489,7 @@ struct voyager_SUS { __u32 SUS_errorlog; /* lots of system configuration stuff under here */ }; - + /* Variables exported by voyager_smp */ extern __u32 voyager_extended_vic_processors; extern __u32 voyager_allowed_boot_processors; diff --git a/include/asm-x86/xen/events.h b/include/asm-x86/xen/events.h new file mode 100644 index 00000000000..596312a7bfc --- /dev/null +++ b/include/asm-x86/xen/events.h @@ -0,0 +1,22 @@ +#ifndef __XEN_EVENTS_H +#define __XEN_EVENTS_H + +enum ipi_vector { + XEN_RESCHEDULE_VECTOR, + XEN_CALL_FUNCTION_VECTOR, + + XEN_NR_IPIS, +}; + +static inline int xen_irqs_disabled(struct pt_regs *regs) +{ + return raw_irqs_disabled_flags(regs->flags); +} + +static inline void xen_do_IRQ(int irq, struct pt_regs *regs) +{ + regs->orig_ax = ~irq; + do_IRQ(regs); +} + +#endif /* __XEN_EVENTS_H */ diff --git a/include/asm-x86/xen/grant_table.h b/include/asm-x86/xen/grant_table.h new file mode 100644 index 00000000000..2444d4593a3 --- /dev/null +++ b/include/asm-x86/xen/grant_table.h @@ -0,0 +1,7 @@ +#ifndef __XEN_GRANT_TABLE_H +#define __XEN_GRANT_TABLE_H + +#define xen_alloc_vm_area(size) alloc_vm_area(size) +#define xen_free_vm_area(area) free_vm_area(area) + +#endif /* __XEN_GRANT_TABLE_H */ diff --git a/include/asm-x86/xen/hypercall.h b/include/asm-x86/xen/hypercall.h index bc0ee7d961c..c2ccd997ed3 100644 --- a/include/asm-x86/xen/hypercall.h +++ b/include/asm-x86/xen/hypercall.h @@ -164,6 +164,12 @@ HYPERVISOR_set_callbacks(unsigned long event_selector, } static inline int +HYPERVISOR_callback_op(int cmd, void *arg) +{ + return _hypercall2(int, callback_op, cmd, arg); +} + +static inline int HYPERVISOR_fpu_taskswitch(int set) { return _hypercall1(int, fpu_taskswitch, set); diff --git a/include/asm-x86/xen/interface.h b/include/asm-x86/xen/interface.h index 165c3968e13..6227000a1e8 100644 --- a/include/asm-x86/xen/interface.h +++ b/include/asm-x86/xen/interface.h @@ -22,6 +22,30 @@ #define DEFINE_GUEST_HANDLE(name) __DEFINE_GUEST_HANDLE(name, name) #define GUEST_HANDLE(name) __guest_handle_ ## name +#ifdef __XEN__ +#if defined(__i386__) +#define set_xen_guest_handle(hnd, val) \ + do { \ + if (sizeof(hnd) == 8) \ + *(uint64_t *)&(hnd) = 0; \ + (hnd).p = val; \ + } while (0) +#elif defined(__x86_64__) +#define set_xen_guest_handle(hnd, val) do { (hnd).p = val; } while (0) +#endif +#else +#if defined(__i386__) +#define set_xen_guest_handle(hnd, val) \ + do { \ + if (sizeof(hnd) == 8) \ + *(uint64_t *)&(hnd) = 0; \ + (hnd) = val; \ + } while (0) +#elif defined(__x86_64__) +#define set_xen_guest_handle(hnd, val) do { (hnd) = val; } while (0) +#endif +#endif + #ifndef __ASSEMBLY__ /* Guest handles for primitive C types. */ __DEFINE_GUEST_HANDLE(uchar, unsigned char); @@ -171,6 +195,10 @@ struct arch_vcpu_info { unsigned long pad[5]; /* sizeof(struct vcpu_info) == 64 */ }; +struct xen_callback { + unsigned long cs; + unsigned long eip; +}; #endif /* !__ASSEMBLY__ */ /* diff --git a/include/asm-x86/xen/page.h b/include/asm-x86/xen/page.h new file mode 100644 index 00000000000..01799305f02 --- /dev/null +++ b/include/asm-x86/xen/page.h @@ -0,0 +1,168 @@ +#ifndef __XEN_PAGE_H +#define __XEN_PAGE_H + +#include <linux/pfn.h> + +#include <asm/uaccess.h> +#include <asm/pgtable.h> + +#include <xen/features.h> + +/* Xen machine address */ +typedef struct xmaddr { + phys_addr_t maddr; +} xmaddr_t; + +/* Xen pseudo-physical address */ +typedef struct xpaddr { + phys_addr_t paddr; +} xpaddr_t; + +#define XMADDR(x) ((xmaddr_t) { .maddr = (x) }) +#define XPADDR(x) ((xpaddr_t) { .paddr = (x) }) + +/**** MACHINE <-> PHYSICAL CONVERSION MACROS ****/ +#define INVALID_P2M_ENTRY (~0UL) +#define FOREIGN_FRAME_BIT (1UL<<31) +#define FOREIGN_FRAME(m) ((m) | FOREIGN_FRAME_BIT) + +extern unsigned long *phys_to_machine_mapping; + +static inline unsigned long pfn_to_mfn(unsigned long pfn) +{ + if (xen_feature(XENFEAT_auto_translated_physmap)) + return pfn; + + return phys_to_machine_mapping[(unsigned int)(pfn)] & + ~FOREIGN_FRAME_BIT; +} + +static inline int phys_to_machine_mapping_valid(unsigned long pfn) +{ + if (xen_feature(XENFEAT_auto_translated_physmap)) + return 1; + + return (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY); +} + +static inline unsigned long mfn_to_pfn(unsigned long mfn) +{ + unsigned long pfn; + + if (xen_feature(XENFEAT_auto_translated_physmap)) + return mfn; + +#if 0 + if (unlikely((mfn >> machine_to_phys_order) != 0)) + return max_mapnr; +#endif + + pfn = 0; + /* + * The array access can fail (e.g., device space beyond end of RAM). + * In such cases it doesn't matter what we return (we return garbage), + * but we must handle the fault without crashing! + */ + __get_user(pfn, &machine_to_phys_mapping[mfn]); + + return pfn; +} + +static inline xmaddr_t phys_to_machine(xpaddr_t phys) +{ + unsigned offset = phys.paddr & ~PAGE_MASK; + return XMADDR(PFN_PHYS((u64)pfn_to_mfn(PFN_DOWN(phys.paddr))) | offset); +} + +static inline xpaddr_t machine_to_phys(xmaddr_t machine) +{ + unsigned offset = machine.maddr & ~PAGE_MASK; + return XPADDR(PFN_PHYS((u64)mfn_to_pfn(PFN_DOWN(machine.maddr))) | offset); +} + +/* + * We detect special mappings in one of two ways: + * 1. If the MFN is an I/O page then Xen will set the m2p entry + * to be outside our maximum possible pseudophys range. + * 2. If the MFN belongs to a different domain then we will certainly + * not have MFN in our p2m table. Conversely, if the page is ours, + * then we'll have p2m(m2p(MFN))==MFN. + * If we detect a special mapping then it doesn't have a 'struct page'. + * We force !pfn_valid() by returning an out-of-range pointer. + * + * NB. These checks require that, for any MFN that is not in our reservation, + * there is no PFN such that p2m(PFN) == MFN. Otherwise we can get confused if + * we are foreign-mapping the MFN, and the other domain as m2p(MFN) == PFN. + * Yikes! Various places must poke in INVALID_P2M_ENTRY for safety. + * + * NB2. When deliberately mapping foreign pages into the p2m table, you *must* + * use FOREIGN_FRAME(). This will cause pte_pfn() to choke on it, as we + * require. In all the cases we care about, the FOREIGN_FRAME bit is + * masked (e.g., pfn_to_mfn()) so behaviour there is correct. + */ +static inline unsigned long mfn_to_local_pfn(unsigned long mfn) +{ + extern unsigned long max_mapnr; + unsigned long pfn = mfn_to_pfn(mfn); + if ((pfn < max_mapnr) + && !xen_feature(XENFEAT_auto_translated_physmap) + && (phys_to_machine_mapping[pfn] != mfn)) + return max_mapnr; /* force !pfn_valid() */ + return pfn; +} + +static inline void set_phys_to_machine(unsigned long pfn, unsigned long mfn) +{ + if (xen_feature(XENFEAT_auto_translated_physmap)) { + BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY); + return; + } + phys_to_machine_mapping[pfn] = mfn; +} + +/* VIRT <-> MACHINE conversion */ +#define virt_to_machine(v) (phys_to_machine(XPADDR(__pa(v)))) +#define virt_to_mfn(v) (pfn_to_mfn(PFN_DOWN(__pa(v)))) +#define mfn_to_virt(m) (__va(mfn_to_pfn(m) << PAGE_SHIFT)) + +static inline unsigned long pte_mfn(pte_t pte) +{ + return (pte.pte & ~_PAGE_NX) >> PAGE_SHIFT; +} + +static inline pte_t mfn_pte(unsigned long page_nr, pgprot_t pgprot) +{ + pte_t pte; + + pte.pte = ((phys_addr_t)page_nr << PAGE_SHIFT) | + (pgprot_val(pgprot) & __supported_pte_mask); + + return pte; +} + +static inline pteval_t pte_val_ma(pte_t pte) +{ + return pte.pte; +} + +static inline pte_t __pte_ma(pteval_t x) +{ + return (pte_t) { .pte = x }; +} + +#ifdef CONFIG_X86_PAE +#define pmd_val_ma(v) ((v).pmd) +#define pud_val_ma(v) ((v).pgd.pgd) +#define __pmd_ma(x) ((pmd_t) { (x) } ) +#else /* !X86_PAE */ +#define pmd_val_ma(v) ((v).pud.pgd.pgd) +#endif /* CONFIG_X86_PAE */ + +#define pgd_val_ma(x) ((x).pgd) + + +xmaddr_t arbitrary_virt_to_machine(unsigned long address); +void make_lowmem_page_readonly(void *vaddr); +void make_lowmem_page_readwrite(void *vaddr); + +#endif /* __XEN_PAGE_H */ diff --git a/include/asm-x86/xor_32.h b/include/asm-x86/xor_32.h index a41ef1bdd42..067b5c1835a 100644 --- a/include/asm-x86/xor_32.h +++ b/include/asm-x86/xor_32.h @@ -16,12 +16,12 @@ * Copyright (C) 1998 Ingo Molnar. */ -#define LD(x,y) " movq 8*("#x")(%1), %%mm"#y" ;\n" -#define ST(x,y) " movq %%mm"#y", 8*("#x")(%1) ;\n" -#define XO1(x,y) " pxor 8*("#x")(%2), %%mm"#y" ;\n" -#define XO2(x,y) " pxor 8*("#x")(%3), %%mm"#y" ;\n" -#define XO3(x,y) " pxor 8*("#x")(%4), %%mm"#y" ;\n" -#define XO4(x,y) " pxor 8*("#x")(%5), %%mm"#y" ;\n" +#define LD(x, y) " movq 8*("#x")(%1), %%mm"#y" ;\n" +#define ST(x, y) " movq %%mm"#y", 8*("#x")(%1) ;\n" +#define XO1(x, y) " pxor 8*("#x")(%2), %%mm"#y" ;\n" +#define XO2(x, y) " pxor 8*("#x")(%3), %%mm"#y" ;\n" +#define XO3(x, y) " pxor 8*("#x")(%4), %%mm"#y" ;\n" +#define XO4(x, y) " pxor 8*("#x")(%5), %%mm"#y" ;\n" #include <asm/i387.h> @@ -32,24 +32,24 @@ xor_pII_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) kernel_fpu_begin(); - __asm__ __volatile__ ( + asm volatile( #undef BLOCK -#define BLOCK(i) \ - LD(i,0) \ - LD(i+1,1) \ - LD(i+2,2) \ - LD(i+3,3) \ - XO1(i,0) \ - ST(i,0) \ - XO1(i+1,1) \ - ST(i+1,1) \ - XO1(i+2,2) \ - ST(i+2,2) \ - XO1(i+3,3) \ - ST(i+3,3) +#define BLOCK(i) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + XO1(i, 0) \ + ST(i, 0) \ + XO1(i+1, 1) \ + ST(i+1, 1) \ + XO1(i + 2, 2) \ + ST(i + 2, 2) \ + XO1(i + 3, 3) \ + ST(i + 3, 3) " .align 32 ;\n" - " 1: ;\n" + " 1: ;\n" BLOCK(0) BLOCK(4) @@ -76,25 +76,25 @@ xor_pII_mmx_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, kernel_fpu_begin(); - __asm__ __volatile__ ( + asm volatile( #undef BLOCK -#define BLOCK(i) \ - LD(i,0) \ - LD(i+1,1) \ - LD(i+2,2) \ - LD(i+3,3) \ - XO1(i,0) \ - XO1(i+1,1) \ - XO1(i+2,2) \ - XO1(i+3,3) \ - XO2(i,0) \ - ST(i,0) \ - XO2(i+1,1) \ - ST(i+1,1) \ - XO2(i+2,2) \ - ST(i+2,2) \ - XO2(i+3,3) \ - ST(i+3,3) +#define BLOCK(i) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + XO2(i, 0) \ + ST(i, 0) \ + XO2(i + 1, 1) \ + ST(i + 1, 1) \ + XO2(i + 2, 2) \ + ST(i + 2, 2) \ + XO2(i + 3, 3) \ + ST(i + 3, 3) " .align 32 ;\n" " 1: ;\n" @@ -125,29 +125,29 @@ xor_pII_mmx_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, kernel_fpu_begin(); - __asm__ __volatile__ ( + asm volatile( #undef BLOCK -#define BLOCK(i) \ - LD(i,0) \ - LD(i+1,1) \ - LD(i+2,2) \ - LD(i+3,3) \ - XO1(i,0) \ - XO1(i+1,1) \ - XO1(i+2,2) \ - XO1(i+3,3) \ - XO2(i,0) \ - XO2(i+1,1) \ - XO2(i+2,2) \ - XO2(i+3,3) \ - XO3(i,0) \ - ST(i,0) \ - XO3(i+1,1) \ - ST(i+1,1) \ - XO3(i+2,2) \ - ST(i+2,2) \ - XO3(i+3,3) \ - ST(i+3,3) +#define BLOCK(i) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + XO2(i, 0) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ + XO3(i, 0) \ + ST(i, 0) \ + XO3(i + 1, 1) \ + ST(i + 1, 1) \ + XO3(i + 2, 2) \ + ST(i + 2, 2) \ + XO3(i + 3, 3) \ + ST(i + 3, 3) " .align 32 ;\n" " 1: ;\n" @@ -186,35 +186,35 @@ xor_pII_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, because we modify p4 and p5 there, but we can't mark them as read/write, otherwise we'd overflow the 10-asm-operands limit of GCC < 3.1. */ - __asm__ ("" : "+r" (p4), "+r" (p5)); + asm("" : "+r" (p4), "+r" (p5)); - __asm__ __volatile__ ( + asm volatile( #undef BLOCK -#define BLOCK(i) \ - LD(i,0) \ - LD(i+1,1) \ - LD(i+2,2) \ - LD(i+3,3) \ - XO1(i,0) \ - XO1(i+1,1) \ - XO1(i+2,2) \ - XO1(i+3,3) \ - XO2(i,0) \ - XO2(i+1,1) \ - XO2(i+2,2) \ - XO2(i+3,3) \ - XO3(i,0) \ - XO3(i+1,1) \ - XO3(i+2,2) \ - XO3(i+3,3) \ - XO4(i,0) \ - ST(i,0) \ - XO4(i+1,1) \ - ST(i+1,1) \ - XO4(i+2,2) \ - ST(i+2,2) \ - XO4(i+3,3) \ - ST(i+3,3) +#define BLOCK(i) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + XO2(i, 0) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ + XO3(i, 0) \ + XO3(i + 1, 1) \ + XO3(i + 2, 2) \ + XO3(i + 3, 3) \ + XO4(i, 0) \ + ST(i, 0) \ + XO4(i + 1, 1) \ + ST(i + 1, 1) \ + XO4(i + 2, 2) \ + ST(i + 2, 2) \ + XO4(i + 3, 3) \ + ST(i + 3, 3) " .align 32 ;\n" " 1: ;\n" @@ -233,13 +233,13 @@ xor_pII_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, " jnz 1b ;\n" : "+r" (lines), "+r" (p1), "+r" (p2), "+r" (p3) - : "r" (p4), "r" (p5) + : "r" (p4), "r" (p5) : "memory"); /* p4 and p5 were modified, and now the variables are dead. Clobber them just to be sure nobody does something stupid like assuming they have some legal value. */ - __asm__ ("" : "=r" (p4), "=r" (p5)); + asm("" : "=r" (p4), "=r" (p5)); kernel_fpu_end(); } @@ -259,7 +259,7 @@ xor_p5_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) kernel_fpu_begin(); - __asm__ __volatile__ ( + asm volatile( " .align 32 ;\n" " 1: ;\n" " movq (%1), %%mm0 ;\n" @@ -286,7 +286,7 @@ xor_p5_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) " pxor 56(%2), %%mm7 ;\n" " movq %%mm6, 48(%1) ;\n" " movq %%mm7, 56(%1) ;\n" - + " addl $64, %1 ;\n" " addl $64, %2 ;\n" " decl %0 ;\n" @@ -307,7 +307,7 @@ xor_p5_mmx_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, kernel_fpu_begin(); - __asm__ __volatile__ ( + asm volatile( " .align 32,0x90 ;\n" " 1: ;\n" " movq (%1), %%mm0 ;\n" @@ -342,7 +342,7 @@ xor_p5_mmx_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, " pxor 56(%3), %%mm7 ;\n" " movq %%mm6, 48(%1) ;\n" " movq %%mm7, 56(%1) ;\n" - + " addl $64, %1 ;\n" " addl $64, %2 ;\n" " addl $64, %3 ;\n" @@ -364,7 +364,7 @@ xor_p5_mmx_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, kernel_fpu_begin(); - __asm__ __volatile__ ( + asm volatile( " .align 32,0x90 ;\n" " 1: ;\n" " movq (%1), %%mm0 ;\n" @@ -407,7 +407,7 @@ xor_p5_mmx_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, " pxor 56(%4), %%mm7 ;\n" " movq %%mm6, 48(%1) ;\n" " movq %%mm7, 56(%1) ;\n" - + " addl $64, %1 ;\n" " addl $64, %2 ;\n" " addl $64, %3 ;\n" @@ -436,9 +436,9 @@ xor_p5_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, because we modify p4 and p5 there, but we can't mark them as read/write, otherwise we'd overflow the 10-asm-operands limit of GCC < 3.1. */ - __asm__ ("" : "+r" (p4), "+r" (p5)); + asm("" : "+r" (p4), "+r" (p5)); - __asm__ __volatile__ ( + asm volatile( " .align 32,0x90 ;\n" " 1: ;\n" " movq (%1), %%mm0 ;\n" @@ -489,7 +489,7 @@ xor_p5_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, " pxor 56(%5), %%mm7 ;\n" " movq %%mm6, 48(%1) ;\n" " movq %%mm7, 56(%1) ;\n" - + " addl $64, %1 ;\n" " addl $64, %2 ;\n" " addl $64, %3 ;\n" @@ -505,7 +505,7 @@ xor_p5_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, /* p4 and p5 were modified, and now the variables are dead. Clobber them just to be sure nobody does something stupid like assuming they have some legal value. */ - __asm__ ("" : "=r" (p4), "=r" (p5)); + asm("" : "=r" (p4), "=r" (p5)); kernel_fpu_end(); } @@ -531,11 +531,12 @@ static struct xor_block_template xor_block_p5_mmx = { * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo) */ -#define XMMS_SAVE do { \ +#define XMMS_SAVE \ +do { \ preempt_disable(); \ cr0 = read_cr0(); \ clts(); \ - __asm__ __volatile__ ( \ + asm volatile( \ "movups %%xmm0,(%0) ;\n\t" \ "movups %%xmm1,0x10(%0) ;\n\t" \ "movups %%xmm2,0x20(%0) ;\n\t" \ @@ -543,10 +544,11 @@ static struct xor_block_template xor_block_p5_mmx = { : \ : "r" (xmm_save) \ : "memory"); \ -} while(0) +} while (0) -#define XMMS_RESTORE do { \ - __asm__ __volatile__ ( \ +#define XMMS_RESTORE \ +do { \ + asm volatile( \ "sfence ;\n\t" \ "movups (%0),%%xmm0 ;\n\t" \ "movups 0x10(%0),%%xmm1 ;\n\t" \ @@ -557,76 +559,76 @@ static struct xor_block_template xor_block_p5_mmx = { : "memory"); \ write_cr0(cr0); \ preempt_enable(); \ -} while(0) +} while (0) #define ALIGN16 __attribute__((aligned(16))) #define OFFS(x) "16*("#x")" #define PF_OFFS(x) "256+16*("#x")" #define PF0(x) " prefetchnta "PF_OFFS(x)"(%1) ;\n" -#define LD(x,y) " movaps "OFFS(x)"(%1), %%xmm"#y" ;\n" -#define ST(x,y) " movaps %%xmm"#y", "OFFS(x)"(%1) ;\n" +#define LD(x, y) " movaps "OFFS(x)"(%1), %%xmm"#y" ;\n" +#define ST(x, y) " movaps %%xmm"#y", "OFFS(x)"(%1) ;\n" #define PF1(x) " prefetchnta "PF_OFFS(x)"(%2) ;\n" #define PF2(x) " prefetchnta "PF_OFFS(x)"(%3) ;\n" #define PF3(x) " prefetchnta "PF_OFFS(x)"(%4) ;\n" #define PF4(x) " prefetchnta "PF_OFFS(x)"(%5) ;\n" #define PF5(x) " prefetchnta "PF_OFFS(x)"(%6) ;\n" -#define XO1(x,y) " xorps "OFFS(x)"(%2), %%xmm"#y" ;\n" -#define XO2(x,y) " xorps "OFFS(x)"(%3), %%xmm"#y" ;\n" -#define XO3(x,y) " xorps "OFFS(x)"(%4), %%xmm"#y" ;\n" -#define XO4(x,y) " xorps "OFFS(x)"(%5), %%xmm"#y" ;\n" -#define XO5(x,y) " xorps "OFFS(x)"(%6), %%xmm"#y" ;\n" +#define XO1(x, y) " xorps "OFFS(x)"(%2), %%xmm"#y" ;\n" +#define XO2(x, y) " xorps "OFFS(x)"(%3), %%xmm"#y" ;\n" +#define XO3(x, y) " xorps "OFFS(x)"(%4), %%xmm"#y" ;\n" +#define XO4(x, y) " xorps "OFFS(x)"(%5), %%xmm"#y" ;\n" +#define XO5(x, y) " xorps "OFFS(x)"(%6), %%xmm"#y" ;\n" static void xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) { - unsigned long lines = bytes >> 8; + unsigned long lines = bytes >> 8; char xmm_save[16*4] ALIGN16; int cr0; XMMS_SAVE; - __asm__ __volatile__ ( + asm volatile( #undef BLOCK -#define BLOCK(i) \ - LD(i,0) \ - LD(i+1,1) \ +#define BLOCK(i) \ + LD(i, 0) \ + LD(i + 1, 1) \ PF1(i) \ - PF1(i+2) \ - LD(i+2,2) \ - LD(i+3,3) \ - PF0(i+4) \ - PF0(i+6) \ - XO1(i,0) \ - XO1(i+1,1) \ - XO1(i+2,2) \ - XO1(i+3,3) \ - ST(i,0) \ - ST(i+1,1) \ - ST(i+2,2) \ - ST(i+3,3) \ + PF1(i + 2) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + PF0(i + 4) \ + PF0(i + 6) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + ST(i, 0) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ PF0(0) PF0(2) " .align 32 ;\n" - " 1: ;\n" + " 1: ;\n" BLOCK(0) BLOCK(4) BLOCK(8) BLOCK(12) - " addl $256, %1 ;\n" - " addl $256, %2 ;\n" - " decl %0 ;\n" - " jnz 1b ;\n" + " addl $256, %1 ;\n" + " addl $256, %2 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" : "+r" (lines), "+r" (p1), "+r" (p2) : - : "memory"); + : "memory"); XMMS_RESTORE; } @@ -635,59 +637,59 @@ static void xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, unsigned long *p3) { - unsigned long lines = bytes >> 8; + unsigned long lines = bytes >> 8; char xmm_save[16*4] ALIGN16; int cr0; XMMS_SAVE; - __asm__ __volatile__ ( + asm volatile( #undef BLOCK #define BLOCK(i) \ PF1(i) \ - PF1(i+2) \ + PF1(i + 2) \ LD(i,0) \ - LD(i+1,1) \ - LD(i+2,2) \ - LD(i+3,3) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ PF2(i) \ - PF2(i+2) \ - PF0(i+4) \ - PF0(i+6) \ + PF2(i + 2) \ + PF0(i + 4) \ + PF0(i + 6) \ XO1(i,0) \ - XO1(i+1,1) \ - XO1(i+2,2) \ - XO1(i+3,3) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ XO2(i,0) \ - XO2(i+1,1) \ - XO2(i+2,2) \ - XO2(i+3,3) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ ST(i,0) \ - ST(i+1,1) \ - ST(i+2,2) \ - ST(i+3,3) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ PF0(0) PF0(2) " .align 32 ;\n" - " 1: ;\n" + " 1: ;\n" BLOCK(0) BLOCK(4) BLOCK(8) BLOCK(12) - " addl $256, %1 ;\n" - " addl $256, %2 ;\n" - " addl $256, %3 ;\n" - " decl %0 ;\n" - " jnz 1b ;\n" + " addl $256, %1 ;\n" + " addl $256, %2 ;\n" + " addl $256, %3 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" : "+r" (lines), "+r" (p1), "+r"(p2), "+r"(p3) : - : "memory" ); + : "memory" ); XMMS_RESTORE; } @@ -696,66 +698,66 @@ static void xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, unsigned long *p3, unsigned long *p4) { - unsigned long lines = bytes >> 8; + unsigned long lines = bytes >> 8; char xmm_save[16*4] ALIGN16; int cr0; XMMS_SAVE; - __asm__ __volatile__ ( + asm volatile( #undef BLOCK #define BLOCK(i) \ PF1(i) \ - PF1(i+2) \ + PF1(i + 2) \ LD(i,0) \ - LD(i+1,1) \ - LD(i+2,2) \ - LD(i+3,3) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ PF2(i) \ - PF2(i+2) \ + PF2(i + 2) \ XO1(i,0) \ - XO1(i+1,1) \ - XO1(i+2,2) \ - XO1(i+3,3) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ PF3(i) \ - PF3(i+2) \ - PF0(i+4) \ - PF0(i+6) \ + PF3(i + 2) \ + PF0(i + 4) \ + PF0(i + 6) \ XO2(i,0) \ - XO2(i+1,1) \ - XO2(i+2,2) \ - XO2(i+3,3) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ XO3(i,0) \ - XO3(i+1,1) \ - XO3(i+2,2) \ - XO3(i+3,3) \ + XO3(i + 1, 1) \ + XO3(i + 2, 2) \ + XO3(i + 3, 3) \ ST(i,0) \ - ST(i+1,1) \ - ST(i+2,2) \ - ST(i+3,3) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ PF0(0) PF0(2) " .align 32 ;\n" - " 1: ;\n" + " 1: ;\n" BLOCK(0) BLOCK(4) BLOCK(8) BLOCK(12) - " addl $256, %1 ;\n" - " addl $256, %2 ;\n" - " addl $256, %3 ;\n" - " addl $256, %4 ;\n" - " decl %0 ;\n" - " jnz 1b ;\n" + " addl $256, %1 ;\n" + " addl $256, %2 ;\n" + " addl $256, %3 ;\n" + " addl $256, %4 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" : "+r" (lines), "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4) : - : "memory" ); + : "memory" ); XMMS_RESTORE; } @@ -764,7 +766,7 @@ static void xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, unsigned long *p3, unsigned long *p4, unsigned long *p5) { - unsigned long lines = bytes >> 8; + unsigned long lines = bytes >> 8; char xmm_save[16*4] ALIGN16; int cr0; @@ -776,65 +778,65 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, because we modify p4 and p5 there, but we can't mark them as read/write, otherwise we'd overflow the 10-asm-operands limit of GCC < 3.1. */ - __asm__ ("" : "+r" (p4), "+r" (p5)); + asm("" : "+r" (p4), "+r" (p5)); - __asm__ __volatile__ ( + asm volatile( #undef BLOCK #define BLOCK(i) \ PF1(i) \ - PF1(i+2) \ + PF1(i + 2) \ LD(i,0) \ - LD(i+1,1) \ - LD(i+2,2) \ - LD(i+3,3) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ PF2(i) \ - PF2(i+2) \ + PF2(i + 2) \ XO1(i,0) \ - XO1(i+1,1) \ - XO1(i+2,2) \ - XO1(i+3,3) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ PF3(i) \ - PF3(i+2) \ + PF3(i + 2) \ XO2(i,0) \ - XO2(i+1,1) \ - XO2(i+2,2) \ - XO2(i+3,3) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ PF4(i) \ - PF4(i+2) \ - PF0(i+4) \ - PF0(i+6) \ + PF4(i + 2) \ + PF0(i + 4) \ + PF0(i + 6) \ XO3(i,0) \ - XO3(i+1,1) \ - XO3(i+2,2) \ - XO3(i+3,3) \ + XO3(i + 1, 1) \ + XO3(i + 2, 2) \ + XO3(i + 3, 3) \ XO4(i,0) \ - XO4(i+1,1) \ - XO4(i+2,2) \ - XO4(i+3,3) \ + XO4(i + 1, 1) \ + XO4(i + 2, 2) \ + XO4(i + 3, 3) \ ST(i,0) \ - ST(i+1,1) \ - ST(i+2,2) \ - ST(i+3,3) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ PF0(0) PF0(2) " .align 32 ;\n" - " 1: ;\n" + " 1: ;\n" BLOCK(0) BLOCK(4) BLOCK(8) BLOCK(12) - " addl $256, %1 ;\n" - " addl $256, %2 ;\n" - " addl $256, %3 ;\n" - " addl $256, %4 ;\n" - " addl $256, %5 ;\n" - " decl %0 ;\n" - " jnz 1b ;\n" + " addl $256, %1 ;\n" + " addl $256, %2 ;\n" + " addl $256, %3 ;\n" + " addl $256, %4 ;\n" + " addl $256, %5 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" : "+r" (lines), "+r" (p1), "+r" (p2), "+r" (p3) : "r" (p4), "r" (p5) @@ -843,17 +845,17 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, /* p4 and p5 were modified, and now the variables are dead. Clobber them just to be sure nobody does something stupid like assuming they have some legal value. */ - __asm__ ("" : "=r" (p4), "=r" (p5)); + asm("" : "=r" (p4), "=r" (p5)); XMMS_RESTORE; } static struct xor_block_template xor_block_pIII_sse = { - .name = "pIII_sse", - .do_2 = xor_sse_2, - .do_3 = xor_sse_3, - .do_4 = xor_sse_4, - .do_5 = xor_sse_5, + .name = "pIII_sse", + .do_2 = xor_sse_2, + .do_3 = xor_sse_3, + .do_4 = xor_sse_4, + .do_5 = xor_sse_5, }; /* Also try the generic routines. */ @@ -861,21 +863,21 @@ static struct xor_block_template xor_block_pIII_sse = { #undef XOR_TRY_TEMPLATES #define XOR_TRY_TEMPLATES \ - do { \ - xor_speed(&xor_block_8regs); \ - xor_speed(&xor_block_8regs_p); \ - xor_speed(&xor_block_32regs); \ - xor_speed(&xor_block_32regs_p); \ - if (cpu_has_xmm) \ - xor_speed(&xor_block_pIII_sse); \ - if (cpu_has_mmx) { \ - xor_speed(&xor_block_pII_mmx); \ - xor_speed(&xor_block_p5_mmx); \ - } \ - } while (0) +do { \ + xor_speed(&xor_block_8regs); \ + xor_speed(&xor_block_8regs_p); \ + xor_speed(&xor_block_32regs); \ + xor_speed(&xor_block_32regs_p); \ + if (cpu_has_xmm) \ + xor_speed(&xor_block_pIII_sse); \ + if (cpu_has_mmx) { \ + xor_speed(&xor_block_pII_mmx); \ + xor_speed(&xor_block_p5_mmx); \ + } \ +} while (0) /* We force the use of the SSE xor block because it can write around L2. We may also be able to load into the L1 only depending on how the cpu deals with a load to a line that is being prefetched. */ -#define XOR_SELECT_TEMPLATE(FASTEST) \ +#define XOR_SELECT_TEMPLATE(FASTEST) \ (cpu_has_xmm ? &xor_block_pIII_sse : FASTEST) diff --git a/include/asm-x86/xor_64.h b/include/asm-x86/xor_64.h index 1eee7fcb242..24957e39ac8 100644 --- a/include/asm-x86/xor_64.h +++ b/include/asm-x86/xor_64.h @@ -24,20 +24,23 @@ */ /* - * x86-64 changes / gcc fixes from Andi Kleen. + * x86-64 changes / gcc fixes from Andi Kleen. * Copyright 2002 Andi Kleen, SuSE Labs. * * This hasn't been optimized for the hammer yet, but there are likely * no advantages to be gotten from x86-64 here anyways. */ -typedef struct { unsigned long a,b; } __attribute__((aligned(16))) xmm_store_t; +typedef struct { + unsigned long a, b; +} __attribute__((aligned(16))) xmm_store_t; -/* Doesn't use gcc to save the XMM registers, because there is no easy way to +/* Doesn't use gcc to save the XMM registers, because there is no easy way to tell it to do a clts before the register saving. */ -#define XMMS_SAVE do { \ +#define XMMS_SAVE \ +do { \ preempt_disable(); \ - asm volatile ( \ + asm volatile( \ "movq %%cr0,%0 ;\n\t" \ "clts ;\n\t" \ "movups %%xmm0,(%1) ;\n\t" \ @@ -47,10 +50,11 @@ typedef struct { unsigned long a,b; } __attribute__((aligned(16))) xmm_store_t; : "=&r" (cr0) \ : "r" (xmm_save) \ : "memory"); \ -} while(0) +} while (0) -#define XMMS_RESTORE do { \ - asm volatile ( \ +#define XMMS_RESTORE \ +do { \ + asm volatile( \ "sfence ;\n\t" \ "movups (%1),%%xmm0 ;\n\t" \ "movups 0x10(%1),%%xmm1 ;\n\t" \ @@ -61,72 +65,72 @@ typedef struct { unsigned long a,b; } __attribute__((aligned(16))) xmm_store_t; : "r" (cr0), "r" (xmm_save) \ : "memory"); \ preempt_enable(); \ -} while(0) +} while (0) #define OFFS(x) "16*("#x")" #define PF_OFFS(x) "256+16*("#x")" #define PF0(x) " prefetchnta "PF_OFFS(x)"(%[p1]) ;\n" -#define LD(x,y) " movaps "OFFS(x)"(%[p1]), %%xmm"#y" ;\n" -#define ST(x,y) " movaps %%xmm"#y", "OFFS(x)"(%[p1]) ;\n" +#define LD(x, y) " movaps "OFFS(x)"(%[p1]), %%xmm"#y" ;\n" +#define ST(x, y) " movaps %%xmm"#y", "OFFS(x)"(%[p1]) ;\n" #define PF1(x) " prefetchnta "PF_OFFS(x)"(%[p2]) ;\n" #define PF2(x) " prefetchnta "PF_OFFS(x)"(%[p3]) ;\n" #define PF3(x) " prefetchnta "PF_OFFS(x)"(%[p4]) ;\n" #define PF4(x) " prefetchnta "PF_OFFS(x)"(%[p5]) ;\n" #define PF5(x) " prefetchnta "PF_OFFS(x)"(%[p6]) ;\n" -#define XO1(x,y) " xorps "OFFS(x)"(%[p2]), %%xmm"#y" ;\n" -#define XO2(x,y) " xorps "OFFS(x)"(%[p3]), %%xmm"#y" ;\n" -#define XO3(x,y) " xorps "OFFS(x)"(%[p4]), %%xmm"#y" ;\n" -#define XO4(x,y) " xorps "OFFS(x)"(%[p5]), %%xmm"#y" ;\n" -#define XO5(x,y) " xorps "OFFS(x)"(%[p6]), %%xmm"#y" ;\n" +#define XO1(x, y) " xorps "OFFS(x)"(%[p2]), %%xmm"#y" ;\n" +#define XO2(x, y) " xorps "OFFS(x)"(%[p3]), %%xmm"#y" ;\n" +#define XO3(x, y) " xorps "OFFS(x)"(%[p4]), %%xmm"#y" ;\n" +#define XO4(x, y) " xorps "OFFS(x)"(%[p5]), %%xmm"#y" ;\n" +#define XO5(x, y) " xorps "OFFS(x)"(%[p6]), %%xmm"#y" ;\n" static void xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) { - unsigned int lines = bytes >> 8; + unsigned int lines = bytes >> 8; unsigned long cr0; xmm_store_t xmm_save[4]; XMMS_SAVE; - asm volatile ( + asm volatile( #undef BLOCK #define BLOCK(i) \ - LD(i,0) \ - LD(i+1,1) \ + LD(i, 0) \ + LD(i + 1, 1) \ PF1(i) \ - PF1(i+2) \ - LD(i+2,2) \ - LD(i+3,3) \ - PF0(i+4) \ - PF0(i+6) \ - XO1(i,0) \ - XO1(i+1,1) \ - XO1(i+2,2) \ - XO1(i+3,3) \ - ST(i,0) \ - ST(i+1,1) \ - ST(i+2,2) \ - ST(i+3,3) \ + PF1(i + 2) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + PF0(i + 4) \ + PF0(i + 6) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + ST(i, 0) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ PF0(0) PF0(2) " .align 32 ;\n" - " 1: ;\n" + " 1: ;\n" BLOCK(0) BLOCK(4) BLOCK(8) BLOCK(12) - " addq %[inc], %[p1] ;\n" - " addq %[inc], %[p2] ;\n" + " addq %[inc], %[p1] ;\n" + " addq %[inc], %[p2] ;\n" " decl %[cnt] ; jnz 1b" : [p1] "+r" (p1), [p2] "+r" (p2), [cnt] "+r" (lines) - : [inc] "r" (256UL) - : "memory"); + : [inc] "r" (256UL) + : "memory"); XMMS_RESTORE; } @@ -141,52 +145,52 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, XMMS_SAVE; - __asm__ __volatile__ ( + asm volatile( #undef BLOCK #define BLOCK(i) \ PF1(i) \ - PF1(i+2) \ - LD(i,0) \ - LD(i+1,1) \ - LD(i+2,2) \ - LD(i+3,3) \ + PF1(i + 2) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ PF2(i) \ - PF2(i+2) \ - PF0(i+4) \ - PF0(i+6) \ - XO1(i,0) \ - XO1(i+1,1) \ - XO1(i+2,2) \ - XO1(i+3,3) \ - XO2(i,0) \ - XO2(i+1,1) \ - XO2(i+2,2) \ - XO2(i+3,3) \ - ST(i,0) \ - ST(i+1,1) \ - ST(i+2,2) \ - ST(i+3,3) \ + PF2(i + 2) \ + PF0(i + 4) \ + PF0(i + 6) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + XO2(i, 0) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ + ST(i, 0) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ PF0(0) PF0(2) " .align 32 ;\n" - " 1: ;\n" + " 1: ;\n" BLOCK(0) BLOCK(4) BLOCK(8) BLOCK(12) - " addq %[inc], %[p1] ;\n" - " addq %[inc], %[p2] ;\n" - " addq %[inc], %[p3] ;\n" + " addq %[inc], %[p1] ;\n" + " addq %[inc], %[p2] ;\n" + " addq %[inc], %[p3] ;\n" " decl %[cnt] ; jnz 1b" : [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3) : [inc] "r" (256UL) - : "memory"); + : "memory"); XMMS_RESTORE; } @@ -195,64 +199,64 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, unsigned long *p3, unsigned long *p4) { unsigned int lines = bytes >> 8; - xmm_store_t xmm_save[4]; + xmm_store_t xmm_save[4]; unsigned long cr0; XMMS_SAVE; - __asm__ __volatile__ ( + asm volatile( #undef BLOCK #define BLOCK(i) \ PF1(i) \ - PF1(i+2) \ - LD(i,0) \ - LD(i+1,1) \ - LD(i+2,2) \ - LD(i+3,3) \ + PF1(i + 2) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ PF2(i) \ - PF2(i+2) \ - XO1(i,0) \ - XO1(i+1,1) \ - XO1(i+2,2) \ - XO1(i+3,3) \ + PF2(i + 2) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ PF3(i) \ - PF3(i+2) \ - PF0(i+4) \ - PF0(i+6) \ - XO2(i,0) \ - XO2(i+1,1) \ - XO2(i+2,2) \ - XO2(i+3,3) \ - XO3(i,0) \ - XO3(i+1,1) \ - XO3(i+2,2) \ - XO3(i+3,3) \ - ST(i,0) \ - ST(i+1,1) \ - ST(i+2,2) \ - ST(i+3,3) \ + PF3(i + 2) \ + PF0(i + 4) \ + PF0(i + 6) \ + XO2(i, 0) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ + XO3(i, 0) \ + XO3(i + 1, 1) \ + XO3(i + 2, 2) \ + XO3(i + 3, 3) \ + ST(i, 0) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ PF0(0) PF0(2) " .align 32 ;\n" - " 1: ;\n" + " 1: ;\n" BLOCK(0) BLOCK(4) BLOCK(8) BLOCK(12) - " addq %[inc], %[p1] ;\n" - " addq %[inc], %[p2] ;\n" - " addq %[inc], %[p3] ;\n" - " addq %[inc], %[p4] ;\n" + " addq %[inc], %[p1] ;\n" + " addq %[inc], %[p2] ;\n" + " addq %[inc], %[p3] ;\n" + " addq %[inc], %[p4] ;\n" " decl %[cnt] ; jnz 1b" : [cnt] "+c" (lines), [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4) : [inc] "r" (256UL) - : "memory" ); + : "memory" ); XMMS_RESTORE; } @@ -261,70 +265,70 @@ static void xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, unsigned long *p3, unsigned long *p4, unsigned long *p5) { - unsigned int lines = bytes >> 8; + unsigned int lines = bytes >> 8; xmm_store_t xmm_save[4]; unsigned long cr0; XMMS_SAVE; - __asm__ __volatile__ ( + asm volatile( #undef BLOCK #define BLOCK(i) \ PF1(i) \ - PF1(i+2) \ - LD(i,0) \ - LD(i+1,1) \ - LD(i+2,2) \ - LD(i+3,3) \ + PF1(i + 2) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ PF2(i) \ - PF2(i+2) \ - XO1(i,0) \ - XO1(i+1,1) \ - XO1(i+2,2) \ - XO1(i+3,3) \ + PF2(i + 2) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ PF3(i) \ - PF3(i+2) \ - XO2(i,0) \ - XO2(i+1,1) \ - XO2(i+2,2) \ - XO2(i+3,3) \ + PF3(i + 2) \ + XO2(i, 0) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ PF4(i) \ - PF4(i+2) \ - PF0(i+4) \ - PF0(i+6) \ - XO3(i,0) \ - XO3(i+1,1) \ - XO3(i+2,2) \ - XO3(i+3,3) \ - XO4(i,0) \ - XO4(i+1,1) \ - XO4(i+2,2) \ - XO4(i+3,3) \ - ST(i,0) \ - ST(i+1,1) \ - ST(i+2,2) \ - ST(i+3,3) \ + PF4(i + 2) \ + PF0(i + 4) \ + PF0(i + 6) \ + XO3(i, 0) \ + XO3(i + 1, 1) \ + XO3(i + 2, 2) \ + XO3(i + 3, 3) \ + XO4(i, 0) \ + XO4(i + 1, 1) \ + XO4(i + 2, 2) \ + XO4(i + 3, 3) \ + ST(i, 0) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ PF0(0) PF0(2) " .align 32 ;\n" - " 1: ;\n" + " 1: ;\n" BLOCK(0) BLOCK(4) BLOCK(8) BLOCK(12) - " addq %[inc], %[p1] ;\n" - " addq %[inc], %[p2] ;\n" - " addq %[inc], %[p3] ;\n" - " addq %[inc], %[p4] ;\n" - " addq %[inc], %[p5] ;\n" + " addq %[inc], %[p1] ;\n" + " addq %[inc], %[p2] ;\n" + " addq %[inc], %[p3] ;\n" + " addq %[inc], %[p4] ;\n" + " addq %[inc], %[p5] ;\n" " decl %[cnt] ; jnz 1b" : [cnt] "+c" (lines), - [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4), + [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5) : [inc] "r" (256UL) : "memory"); @@ -333,18 +337,18 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static struct xor_block_template xor_block_sse = { - .name = "generic_sse", - .do_2 = xor_sse_2, - .do_3 = xor_sse_3, - .do_4 = xor_sse_4, - .do_5 = xor_sse_5, + .name = "generic_sse", + .do_2 = xor_sse_2, + .do_3 = xor_sse_3, + .do_4 = xor_sse_4, + .do_5 = xor_sse_5, }; #undef XOR_TRY_TEMPLATES -#define XOR_TRY_TEMPLATES \ - do { \ - xor_speed(&xor_block_sse); \ - } while (0) +#define XOR_TRY_TEMPLATES \ +do { \ + xor_speed(&xor_block_sse); \ +} while (0) /* We force the use of the SSE xor block because it can write around L2. We may also be able to load into the L1 only depending on how the cpu |