From 0ba3418b8b1c85ee1771c63f1dd12041614e56ff Mon Sep 17 00:00:00 2001 From: Kumar Gala Date: Tue, 15 Jul 2008 16:12:25 -0500 Subject: powerpc: Introduce local (non-broadcast) forms of tlb invalidates Introduced a new set of low level tlb invalidate functions that do not broadcast invalidates on the bus: _tlbil_all - invalidate all _tlbil_pid - invalidate based on process id (or mm context) _tlbil_va - invalidate based on virtual address (ea + pid) On non-SMP configs _tlbil_all should be functionally equivalent to _tlbia and _tlbil_va should be functionally equivalent to _tlbie. The intent of this change is to handle SMP based invalidates via IPIs instead of broadcasts as the mechanism scales better for larger number of cores. On e500 (fsl-booke mmu) based cores move to using MMUCSR for invalidate alls and tlbsx/tlbwe for invalidate virtual address. Signed-off-by: Kumar Gala --- arch/powerpc/include/asm/reg_booke.h | 7 +++++++ arch/powerpc/include/asm/tlbflush.h | 13 ++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) (limited to 'arch/powerpc/include') diff --git a/arch/powerpc/include/asm/reg_booke.h b/arch/powerpc/include/asm/reg_booke.h index be980f4ee49..67453766bff 100644 --- a/arch/powerpc/include/asm/reg_booke.h +++ b/arch/powerpc/include/asm/reg_booke.h @@ -109,6 +109,7 @@ #define SPRN_EVPR 0x3D6 /* Exception Vector Prefix Register */ #define SPRN_L1CSR0 0x3F2 /* L1 Cache Control and Status Register 0 */ #define SPRN_L1CSR1 0x3F3 /* L1 Cache Control and Status Register 1 */ +#define SPRN_MMUCSR0 0x3F4 /* MMU Control and Status Register 0 */ #define SPRN_PIT 0x3DB /* Programmable Interval Timer */ #define SPRN_BUCSR 0x3F5 /* Branch Unit Control and Status */ #define SPRN_L2CSR0 0x3F9 /* L2 Data Cache Control and Status Register 0 */ @@ -410,6 +411,12 @@ #define L2CSR0_L2LOA 0x00000080 /* L2 Cache Lock Overflow Allocate */ #define L2CSR0_L2LO 0x00000020 /* L2 Cache Lock Overflow */ +/* Bit definitions for MMUCSR0 */ +#define MMUCSR0_TLB1FI 0x00000002 /* TLB1 Flash invalidate */ +#define MMUCSR0_TLB0FI 0x00000004 /* TLB0 Flash invalidate */ +#define MMUCSR0_TLB2FI 0x00000040 /* TLB2 Flash invalidate */ +#define MMUCSR0_TLB3FI 0x00000020 /* TLB3 Flash invalidate */ + /* Bit definitions for SGR. */ #define SGR_NORMAL 0 /* Speculative fetching allowed. */ #define SGR_GUARDED 1 /* Speculative fetching disallowed. */ diff --git a/arch/powerpc/include/asm/tlbflush.h b/arch/powerpc/include/asm/tlbflush.h index 361cd5c7a32..a2c6bfd85fb 100644 --- a/arch/powerpc/include/asm/tlbflush.h +++ b/arch/powerpc/include/asm/tlbflush.h @@ -29,6 +29,9 @@ #include extern void _tlbie(unsigned long address, unsigned int pid); +extern void _tlbil_all(void); +extern void _tlbil_pid(unsigned int pid); +extern void _tlbil_va(unsigned long address, unsigned int pid); #if defined(CONFIG_40x) || defined(CONFIG_8xx) #define _tlbia() asm volatile ("tlbia; sync" : : : "memory") @@ -38,31 +41,31 @@ extern void _tlbia(void); static inline void flush_tlb_mm(struct mm_struct *mm) { - _tlbia(); + _tlbil_pid(mm->context.id); } static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) { - _tlbie(vmaddr, vma ? vma->vm_mm->context.id : 0); + _tlbil_va(vmaddr, vma ? vma->vm_mm->context.id : 0); } static inline void flush_tlb_page_nohash(struct vm_area_struct *vma, unsigned long vmaddr) { - _tlbie(vmaddr, vma ? vma->vm_mm->context.id : 0); + flush_tlb_page(vma, vmaddr); } static inline void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - _tlbia(); + _tlbil_pid(vma->vm_mm->context.id); } static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) { - _tlbia(); + _tlbil_pid(0); } #elif defined(CONFIG_PPC32) -- cgit v1.2.3 From 9bf2b5cdc5fe5e3136ceeecc85141765023ded6e Mon Sep 17 00:00:00 2001 From: Kumar Gala Date: Wed, 16 Jul 2008 15:54:21 -0500 Subject: powerpc: Fixes for CONFIG_PTE_64BIT for SMP support There are some minor issues with support 64-bit PTEs on a 32-bit processor when dealing with SMP. * We need to order the stores in set_pte_at to make sure the flag word is set second. * Change pte_clear to use pte_update so only the flag word is cleared * Added a WARN_ON to set_pte_at to ensure the pte isn't present for the 64-bit pte/SMP case (to ensure our assumption of this fact). Signed-off-by: Kumar Gala Acked-by: Becky Bruce --- arch/powerpc/include/asm/highmem.h | 2 +- arch/powerpc/include/asm/pgtable-ppc32.h | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) (limited to 'arch/powerpc/include') diff --git a/arch/powerpc/include/asm/highmem.h b/arch/powerpc/include/asm/highmem.h index 5d99b6489d5..91c589520c0 100644 --- a/arch/powerpc/include/asm/highmem.h +++ b/arch/powerpc/include/asm/highmem.h @@ -84,7 +84,7 @@ static inline void *kmap_atomic_prot(struct page *page, enum km_type type, pgpro #ifdef CONFIG_DEBUG_HIGHMEM BUG_ON(!pte_none(*(kmap_pte-idx))); #endif - set_pte_at(&init_mm, vaddr, kmap_pte-idx, mk_pte(page, prot)); + __set_pte_at(&init_mm, vaddr, kmap_pte-idx, mk_pte(page, prot)); flush_tlb_page(NULL, vaddr); return (void*) vaddr; diff --git a/arch/powerpc/include/asm/pgtable-ppc32.h b/arch/powerpc/include/asm/pgtable-ppc32.h index 6fe39e32704..1b119555547 100644 --- a/arch/powerpc/include/asm/pgtable-ppc32.h +++ b/arch/powerpc/include/asm/pgtable-ppc32.h @@ -517,7 +517,8 @@ extern unsigned long bad_call_to_PMD_PAGE_SIZE(void); #define pte_none(pte) ((pte_val(pte) & ~_PTE_NONE_MASK) == 0) #define pte_present(pte) (pte_val(pte) & _PAGE_PRESENT) -#define pte_clear(mm,addr,ptep) do { set_pte_at((mm), (addr), (ptep), __pte(0)); } while (0) +#define pte_clear(mm, addr, ptep) \ + do { pte_update(ptep, ~_PAGE_HASHPTE, 0); } while (0) #define pmd_none(pmd) (!pmd_val(pmd)) #define pmd_bad(pmd) (pmd_val(pmd) & _PMD_BAD) @@ -612,9 +613,6 @@ static inline unsigned long pte_update(pte_t *p, return old; } #else /* CONFIG_PTE_64BIT */ -/* TODO: Change that to only modify the low word and move set_pte_at() - * out of line - */ static inline unsigned long long pte_update(pte_t *p, unsigned long clr, unsigned long set) @@ -652,16 +650,33 @@ static inline unsigned long long pte_update(pte_t *p, * On machines which use an MMU hash table we avoid changing the * _PAGE_HASHPTE bit. */ -static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + +static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { #if _PAGE_HASHPTE != 0 pte_update(ptep, ~_PAGE_HASHPTE, pte_val(pte) & ~_PAGE_HASHPTE); +#elif defined(CONFIG_PTE_64BIT) && defined(CONFIG_SMP) + __asm__ __volatile__("\ + stw%U0%X0 %2,%0\n\ + eieio\n\ + stw%U0%X0 %L2,%1" + : "=m" (*ptep), "=m" (*((unsigned char *)ptep+4)) + : "r" (pte) : "memory"); #else *ptep = pte; #endif } +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ +#if defined(CONFIG_PTE_64BIT) && defined(CONFIG_SMP) + WARN_ON(pte_present(*ptep)); +#endif + __set_pte_at(mm, addr, ptep, pte); +} + /* * 2.6 calls this without flushing the TLB entry; this is wrong * for our hash-based implementation, we fix that up here. -- cgit v1.2.3 From 9a62c05180ff55fdaa517370c6f077402820406c Mon Sep 17 00:00:00 2001 From: Kumar Gala Date: Thu, 31 Jul 2008 08:41:10 -0500 Subject: powerpc/mm: Implement _PAGE_SPECIAL & pte_special() for 32-bit Implement _PAGE_SPECIAL and pte_special() for 32-bit powerpc. This bit will be used by the fast get_user_pages() to differenciate PTEs that correspond to a valid struct page from special mappings that don't such as IO mappings obtained via io_remap_pfn_ranges(). We currently only implement this on sub-arch that support SMP or will so in the future (6xx, 44x, FSL-BookE) and not (8xx, 40x). Signed-off-by: Kumar Gala Acked-by: Benjamin Herrenschmidt --- arch/powerpc/include/asm/pgtable-ppc32.h | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'arch/powerpc/include') diff --git a/arch/powerpc/include/asm/pgtable-ppc32.h b/arch/powerpc/include/asm/pgtable-ppc32.h index 1b119555547..c2e58b41bc7 100644 --- a/arch/powerpc/include/asm/pgtable-ppc32.h +++ b/arch/powerpc/include/asm/pgtable-ppc32.h @@ -261,6 +261,7 @@ extern int icache_44x_need_flush; #define _PAGE_HWEXEC 0x00000004 /* H: Execute permission */ #define _PAGE_ACCESSED 0x00000008 /* S: Page referenced */ #define _PAGE_DIRTY 0x00000010 /* S: Page dirty */ +#define _PAGE_SPECIAL 0x00000020 /* S: Special page */ #define _PAGE_USER 0x00000040 /* S: User page */ #define _PAGE_ENDIAN 0x00000080 /* H: E bit */ #define _PAGE_GUARDED 0x00000100 /* H: G bit */ @@ -276,6 +277,7 @@ extern int icache_44x_need_flush; /* ERPN in a PTE never gets cleared, ignore it */ #define _PTE_NONE_MASK 0xffffffff00000000ULL +#define __HAVE_ARCH_PTE_SPECIAL #elif defined(CONFIG_FSL_BOOKE) /* @@ -305,6 +307,7 @@ extern int icache_44x_need_flush; #define _PAGE_COHERENT 0x00100 /* H: M bit */ #define _PAGE_NO_CACHE 0x00200 /* H: I bit */ #define _PAGE_WRITETHRU 0x00400 /* H: W bit */ +#define _PAGE_SPECIAL 0x00800 /* S: Special page */ #ifdef CONFIG_PTE_64BIT /* ERPN in a PTE never gets cleared, ignore it */ @@ -315,6 +318,8 @@ extern int icache_44x_need_flush; #define _PMD_PRESENT_MASK (PAGE_MASK) #define _PMD_BAD (~PAGE_MASK) +#define __HAVE_ARCH_PTE_SPECIAL + #elif defined(CONFIG_8xx) /* Definitions for 8xx embedded chips. */ #define _PAGE_PRESENT 0x0001 /* Page is valid */ @@ -362,6 +367,7 @@ extern int icache_44x_need_flush; #define _PAGE_ACCESSED 0x100 /* R: page referenced */ #define _PAGE_EXEC 0x200 /* software: i-cache coherency required */ #define _PAGE_RW 0x400 /* software: user write access allowed */ +#define _PAGE_SPECIAL 0x800 /* software: Special page */ #define _PTE_NONE_MASK _PAGE_HASHPTE @@ -372,6 +378,8 @@ extern int icache_44x_need_flush; /* Hash table based platforms need atomic updates of the linux PTE */ #define PTE_ATOMIC_UPDATES 1 +#define __HAVE_ARCH_PTE_SPECIAL + #endif /* @@ -404,6 +412,9 @@ extern int icache_44x_need_flush; #ifndef _PAGE_WRITETHRU #define _PAGE_WRITETHRU 0 #endif +#ifndef _PAGE_SPECIAL +#define _PAGE_SPECIAL 0 +#endif #ifndef _PMD_PRESENT_MASK #define _PMD_PRESENT_MASK _PMD_PRESENT #endif @@ -534,7 +545,7 @@ static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_RW; } static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } static inline int pte_file(pte_t pte) { return pte_val(pte) & _PAGE_FILE; } -static inline int pte_special(pte_t pte) { return 0; } +static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; } static inline void pte_uncache(pte_t pte) { pte_val(pte) |= _PAGE_NO_CACHE; } static inline void pte_cache(pte_t pte) { pte_val(pte) &= ~_PAGE_NO_CACHE; } @@ -553,7 +564,7 @@ static inline pte_t pte_mkdirty(pte_t pte) { static inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= _PAGE_ACCESSED; return pte; } static inline pte_t pte_mkspecial(pte_t pte) { - return pte; } + pte_val(pte) |= _PAGE_SPECIAL; return pte; } static inline unsigned long pte_pgprot(pte_t pte) { return __pgprot(pte_val(pte)) & PAGE_PROT_BITS; -- cgit v1.2.3 From 4ee7084eb11e00eb02dc8435fd18273a61ffa9bf Mon Sep 17 00:00:00 2001 From: Becky Bruce Date: Wed, 24 Sep 2008 11:01:24 -0500 Subject: POWERPC: Allow 32-bit hashed pgtable code to support 36-bit physical This rearranges a bit of code, and adds support for 36-bit physical addressing for configs that use a hashed page table. The 36b physical support is not enabled by default on any config - it must be explicitly enabled via the config system. This patch *only* expands the page table code to accomodate large physical addresses on 32-bit systems and enables the PHYS_64BIT config option for 86xx. It does *not* allow you to boot a board with more than about 3.5GB of RAM - for that, SWIOTLB support is also required (and coming soon). Signed-off-by: Becky Bruce Signed-off-by: Kumar Gala --- arch/powerpc/include/asm/io.h | 2 +- arch/powerpc/include/asm/page_32.h | 8 +++++++- arch/powerpc/include/asm/pgtable-ppc32.h | 17 +++++++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/include') diff --git a/arch/powerpc/include/asm/io.h b/arch/powerpc/include/asm/io.h index 77c7fa025e6..08266d2728b 100644 --- a/arch/powerpc/include/asm/io.h +++ b/arch/powerpc/include/asm/io.h @@ -711,7 +711,7 @@ static inline void * phys_to_virt(unsigned long address) /* * Change "struct page" to physical address. */ -#define page_to_phys(page) (page_to_pfn(page) << PAGE_SHIFT) +#define page_to_phys(page) ((phys_addr_t)page_to_pfn(page) << PAGE_SHIFT) /* We do NOT want virtual merging, it would put too much pressure on * our iommu allocator. Instead, we want drivers to be smart enough diff --git a/arch/powerpc/include/asm/page_32.h b/arch/powerpc/include/asm/page_32.h index ebfae530a37..d77072a32cc 100644 --- a/arch/powerpc/include/asm/page_32.h +++ b/arch/powerpc/include/asm/page_32.h @@ -13,10 +13,16 @@ #define ARCH_KMALLOC_MINALIGN L1_CACHE_BYTES #endif +#ifdef CONFIG_PTE_64BIT +#define PTE_FLAGS_OFFSET 4 /* offset of PTE flags, in bytes */ +#else +#define PTE_FLAGS_OFFSET 0 +#endif + #ifndef __ASSEMBLY__ /* * The basic type of a PTE - 64 bits for those CPUs with > 32 bit - * physical addressing. For now this just the IBM PPC440. + * physical addressing. */ #ifdef CONFIG_PTE_64BIT typedef unsigned long long pte_basic_t; diff --git a/arch/powerpc/include/asm/pgtable-ppc32.h b/arch/powerpc/include/asm/pgtable-ppc32.h index c2e58b41bc7..29c83d85b04 100644 --- a/arch/powerpc/include/asm/pgtable-ppc32.h +++ b/arch/powerpc/include/asm/pgtable-ppc32.h @@ -369,7 +369,12 @@ extern int icache_44x_need_flush; #define _PAGE_RW 0x400 /* software: user write access allowed */ #define _PAGE_SPECIAL 0x800 /* software: Special page */ +#ifdef CONFIG_PTE_64BIT +/* We never clear the high word of the pte */ +#define _PTE_NONE_MASK (0xffffffff00000000ULL | _PAGE_HASHPTE) +#else #define _PTE_NONE_MASK _PAGE_HASHPTE +#endif #define _PMD_PRESENT 0 #define _PMD_PRESENT_MASK (PAGE_MASK) @@ -587,6 +592,10 @@ extern int flush_hash_pages(unsigned context, unsigned long va, extern void add_hash_page(unsigned context, unsigned long va, unsigned long pmdval); +/* Flush an entry from the TLB/hash table */ +extern void flush_hash_entry(struct mm_struct *mm, pte_t *ptep, + unsigned long address); + /* * Atomic PTE updates. * @@ -665,9 +674,13 @@ static inline unsigned long long pte_update(pte_t *p, static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { -#if _PAGE_HASHPTE != 0 +#if (_PAGE_HASHPTE != 0) && defined(CONFIG_SMP) && !defined(CONFIG_PTE_64BIT) pte_update(ptep, ~_PAGE_HASHPTE, pte_val(pte) & ~_PAGE_HASHPTE); #elif defined(CONFIG_PTE_64BIT) && defined(CONFIG_SMP) +#if _PAGE_HASHPTE != 0 + if (pte_val(*ptep) & _PAGE_HASHPTE) + flush_hash_entry(mm, ptep, addr); +#endif __asm__ __volatile__("\ stw%U0%X0 %2,%0\n\ eieio\n\ @@ -675,7 +688,7 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, : "=m" (*ptep), "=m" (*((unsigned char *)ptep+4)) : "r" (pte) : "memory"); #else - *ptep = pte; + *ptep = (*ptep & _PAGE_HASHPTE) | (pte & ~_PAGE_HASHPTE); #endif } -- cgit v1.2.3