diff options
| -rw-r--r-- | include/asm-x86/paravirt.h | 21 | ||||
| -rw-r--r-- | include/asm-x86/pgtable-2level.h | 30 | ||||
| -rw-r--r-- | include/asm-x86/pgtable-3level.h | 15 | ||||
| -rw-r--r-- | include/asm-x86/pgtable.h | 63 | ||||
| -rw-r--r-- | include/asm-x86/pgtable_64.h | 64 |
5 files changed, 121 insertions, 72 deletions
diff --git a/include/asm-x86/paravirt.h b/include/asm-x86/paravirt.h index 24385decbd4..4aa06b929e4 100644 --- a/include/asm-x86/paravirt.h +++ b/include/asm-x86/paravirt.h @@ -1039,6 +1039,27 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) { PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, pmdval.pud.pgd.pgd); } + +static inline void pmd_clear(pmd_t *pmdp) +{ + set_pmd(pmdp, __pmd(0)); +} + +static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +{ + set_pte_at(mm, addr, ptep, __pte(0)); +} + +static inline void set_pte_atomic(pte_t *ptep, pte_t pte) +{ + set_pte(ptep, pte); +} + +static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + set_pte(ptep, pte); +} #endif /* CONFIG_X86_PAE */ /* Lazy mode for batching updates / context switch */ diff --git a/include/asm-x86/pgtable-2level.h b/include/asm-x86/pgtable-2level.h index 65cdc8f9e3b..ce486bdbbbb 100644 --- a/include/asm-x86/pgtable-2level.h +++ b/include/asm-x86/pgtable-2level.h @@ -15,29 +15,31 @@ static inline void native_set_pte(pte_t *ptep , pte_t pte) { *ptep = pte; } -static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep , pte_t pte) -{ - native_set_pte(ptep, pte); -} + static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) { *pmdp = pmd; } -#ifndef CONFIG_PARAVIRT -#define set_pte(pteptr, pteval) native_set_pte(pteptr, pteval) -#define set_pte_at(mm,addr,ptep,pteval) native_set_pte_at(mm, addr, ptep, pteval) -#define set_pmd(pmdptr, pmdval) native_set_pmd(pmdptr, pmdval) -#endif -#define set_pte_atomic(pteptr, pteval) set_pte(pteptr,pteval) -#define set_pte_present(mm,addr,ptep,pteval) set_pte_at(mm,addr,ptep,pteval) +static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) +{ + native_set_pte(ptep, pte); +} + +static inline void native_set_pte_present(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + native_set_pte(ptep, pte); +} -#define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0) +static inline void native_pmd_clear(pmd_t *pmdp) +{ + native_set_pmd(pmdp, __pmd(0)); +} static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *xp) { - *xp = __pte(0); + *xp = native_make_pte(0); } #ifdef CONFIG_SMP diff --git a/include/asm-x86/pgtable-3level.h b/include/asm-x86/pgtable-3level.h index a123e687112..5af7a44ed90 100644 --- a/include/asm-x86/pgtable-3level.h +++ b/include/asm-x86/pgtable-3level.h @@ -39,11 +39,6 @@ static inline void native_set_pte(pte_t *ptep, pte_t pte) smp_wmb(); ptep->pte_low = pte.pte_low; } -static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep , pte_t pte) -{ - native_set_pte(ptep, pte); -} /* * Since this is only called on user PTEs, and the page fault handler @@ -94,16 +89,6 @@ static inline void native_pmd_clear(pmd_t *pmd) *(tmp + 1) = 0; } -#ifndef CONFIG_PARAVIRT -#define set_pte(ptep, pte) native_set_pte(ptep, pte) -#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte) -#define set_pte_present(mm, addr, ptep, pte) native_set_pte_present(mm, addr, ptep, pte) -#define set_pte_atomic(ptep, pte) native_set_pte_atomic(ptep, pte) -#define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd) -#define set_pud(pudp, pud) native_set_pud(pudp, pud) -#define pmd_clear(pmd) native_pmd_clear(pmd) -#endif - /* * Pentium-II erratum A13: in PAE mode we explicitly have to flush * the TLB via cr3 if the top-level pgd is changed... diff --git a/include/asm-x86/pgtable.h b/include/asm-x86/pgtable.h index eb14a70a472..a1eeacdf9d8 100644 --- a/include/asm-x86/pgtable.h +++ b/include/asm-x86/pgtable.h @@ -178,6 +178,39 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) return __pte(val); } +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else /* !CONFIG_PARAVIRT */ +#define set_pte(ptep, pte) native_set_pte(ptep, pte) +#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte) + +#define set_pte_present(mm, addr, ptep, pte) \ + native_set_pte_present(mm, addr, ptep, pte) +#define set_pte_atomic(ptep, pte) \ + native_set_pte_atomic(ptep, pte) + +#define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd) + +#ifndef __PAGETABLE_PUD_FOLDED +#define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd) +#define pgd_clear(pgd) native_pgd_clear(pgd) +#endif + +#ifndef set_pud +# define set_pud(pudp, pud) native_set_pud(pudp, pud) +#endif + +#ifndef __PAGETABLE_PMD_FOLDED +#define pud_clear(pud) native_pud_clear(pud) +#endif + +#define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep) +#define pmd_clear(pmd) native_pmd_clear(pmd) + +#define pte_update(mm, addr, ptep) do { } while (0) +#define pte_update_defer(mm, addr, ptep) do { } while (0) +#endif /* CONFIG_PARAVIRT */ + #endif /* __ASSEMBLY__ */ #ifdef CONFIG_X86_32 @@ -188,6 +221,22 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #ifndef __ASSEMBLY__ +/* local pte updates need not use xchg for locking */ +static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep) +{ + pte_t res = *ptep; + + /* Pure native function needs no input for mm, addr */ + native_pte_clear(NULL, 0, ptep); + return res; +} + +static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep , pte_t pte) +{ + native_set_pte(ptep, pte); +} + #ifndef CONFIG_PARAVIRT /* * Rules for using pte_update - it must be called after any PTE update which @@ -206,16 +255,6 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #define pte_update_defer(mm, addr, ptep) do { } while (0) #endif -/* local pte updates need not use xchg for locking */ -static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep) -{ - pte_t res = *ptep; - - /* Pure native function needs no input for mm, addr */ - native_pte_clear(NULL, 0, ptep); - return res; -} - /* * We only update the dirty/accessed state if we set * the dirty bit by hand in the kernel, since the hardware @@ -287,10 +326,6 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_update(mm, addr, ptep); } -#ifndef CONFIG_PARAVIRT -#define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep) -#endif /* !CONFIG_PARAVIRT */ - #include <asm-generic/pgtable.h> #endif /* __ASSEMBLY__ */ diff --git a/include/asm-x86/pgtable_64.h b/include/asm-x86/pgtable_64.h index 435e17187eb..29fdeb8111b 100644 --- a/include/asm-x86/pgtable_64.h +++ b/include/asm-x86/pgtable_64.h @@ -63,51 +63,59 @@ extern void clear_kernel_mapping(unsigned long addr, unsigned long size); #define pgd_none(x) (!pgd_val(x)) #define pud_none(x) (!pud_val(x)) -static inline void set_pte(pte_t *dst, pte_t val) +struct mm_struct; + +static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) { - *dst = val; -} -#define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) + *ptep = native_make_pte(0); +} -static inline void set_pmd(pmd_t *dst, pmd_t val) +static inline void native_set_pte(pte_t *ptep, pte_t pte) { - *dst = val; -} + *ptep = pte; +} -static inline void set_pud(pud_t *dst, pud_t val) +static inline pte_t native_ptep_get_and_clear(pte_t *xp) { - *dst = val; +#ifdef CONFIG_SMP + return native_make_pte(xchg(&xp->pte, 0)); +#else + /* native_local_ptep_get_and_clear, but duplicated because of cyclic dependency */ + pte_t ret = *xp; + native_pte_clear(NULL, 0, xp); + return ret; +#endif } -static inline void pud_clear (pud_t *pud) +static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) { - set_pud(pud, __pud(0)); + *pmdp = pmd; } -static inline void set_pgd(pgd_t *dst, pgd_t val) +static inline void native_pmd_clear(pmd_t *pmd) { - *dst = val; -} + native_set_pmd(pmd, native_make_pmd(0)); +} -static inline void pgd_clear (pgd_t * pgd) +static inline void native_set_pud(pud_t *pudp, pud_t pud) { - set_pgd(pgd, __pgd(0)); + *pudp = pud; } -#define native_ptep_get_and_clear(xp) __pte(xchg(&(xp)->pte, 0)) +static inline void native_pud_clear(pud_t *pud) +{ + native_set_pud(pud, native_make_pud(0)); +} -struct mm_struct; +static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) +{ + *pgdp = pgd; +} -static inline pte_t native_ptep_get_and_clear_full(struct mm_struct *mm, unsigned long addr, pte_t *ptep, int full) +static inline void native_pgd_clear(pgd_t * pgd) { - pte_t pte; - if (full) { - pte = *ptep; - *ptep = __pte(0); - } else { - pte = native_ptep_get_and_clear(ptep); - } - return pte; + native_set_pgd(pgd, native_make_pgd(0)); } #define pte_same(a, b) ((a).pte == (b).pte) @@ -151,7 +159,6 @@ static inline unsigned long pmd_bad(pmd_t pmd) #define pte_none(x) (!pte_val(x)) #define pte_present(x) (pte_val(x) & (_PAGE_PRESENT | _PAGE_PROTNONE)) -#define native_pte_clear(mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0) #define pages_to_mb(x) ((x) >> (20-PAGE_SHIFT)) /* FIXME: is this right? */ #define pte_page(x) pfn_to_page(pte_pfn(x)) @@ -196,7 +203,6 @@ static inline unsigned long pmd_bad(pmd_t pmd) pmd_index(address)) #define pmd_none(x) (!pmd_val(x)) #define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT) -#define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0) #define pfn_pmd(nr,prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val(prot))) #define pmd_pfn(x) ((pmd_val(x) & __PHYSICAL_MASK) >> PAGE_SHIFT) |