24 files changed, 513 insertions, 853 deletions

diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile
index b56a0abdc3d..b0178da019f 100644
--- a/arch/mips/mm/Makefile
+++ b/arch/mips/mm/Makefile
@@ -22,7 +22,7 @@ obj-$(CONFIG_CPU_R8000)		+= c-r4k.o cex-gen.o pg-r4k.o tlb-r8k.o
 obj-$(CONFIG_CPU_RM7000)	+= c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
 obj-$(CONFIG_CPU_RM9000)	+= c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
 obj-$(CONFIG_CPU_SB1)		+= c-sb1.o cerr-sb1.o cex-sb1.o pg-sb1.o \
-				   tlb-sb1.o
+				   tlb-r4k.o
 obj-$(CONFIG_CPU_TX39XX)	+= c-tx39.o pg-r4k.o tlb-r3k.o
 obj-$(CONFIG_CPU_TX49XX)	+= c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
 obj-$(CONFIG_CPU_VR41XX)	+= c-r4k.o cex-gen.o pg-r4k.o tlb-r4k.o
diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c
index c659f99eb39..27f4fa25e8c 100644
--- a/arch/mips/mm/c-r3k.c
+++ b/arch/mips/mm/c-r3k.c
@@ -221,12 +221,14 @@ static inline unsigned long get_phys_page (unsigned long addr,
 					   struct mm_struct *mm)
 {
 	pgd_t *pgd;
+	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
 	unsigned long physpage;
 
 	pgd = pgd_offset(mm, addr);
-	pmd = pmd_offset(pgd, addr);
+	pud = pud_offset(pgd, addr);
+	pmd = pmd_offset(pud, addr);
 	pte = pte_offset(pmd, addr);
 
 	if ((physpage = pte_val(*pte)) & _PAGE_VALID)
@@ -317,7 +319,7 @@ static void r3k_dma_cache_wback_inv(unsigned long start, unsigned long size)
 	r3k_flush_dcache_range(start, start + size);
 }
 
-void __init ld_mmu_r23000(void)
+void __init r3k_cache_init(void)
 {
 	extern void build_clear_page(void);
 	extern void build_copy_page(void);
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index 5ea84bc98c6..38223b44d96 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -16,6 +16,7 @@
 
 #include <asm/bcache.h>
 #include <asm/bootinfo.h>
+#include <asm/cache.h>
 #include <asm/cacheops.h>
 #include <asm/cpu.h>
 #include <asm/cpu-features.h>
@@ -26,8 +27,14 @@
 #include <asm/system.h>
 #include <asm/mmu_context.h>
 #include <asm/war.h>
+#include <asm/cacheflush.h> /* for run_uncached() */
 
-static unsigned long icache_size, dcache_size, scache_size;
+/*
+ * Must die.
+ */
+static unsigned long icache_size __read_mostly;
+static unsigned long dcache_size __read_mostly;
+static unsigned long scache_size __read_mostly;
 
 /*
  * Dummy cache handling routines for machines without boardcaches
@@ -43,8 +50,8 @@ static struct bcache_ops no_sc_ops = {
 
 struct bcache_ops *bcops = &no_sc_ops;
 
-#define cpu_is_r4600_v1_x()	((read_c0_prid() & 0xfffffff0) == 0x2010)
-#define cpu_is_r4600_v2_x()	((read_c0_prid() & 0xfffffff0) == 0x2020)
+#define cpu_is_r4600_v1_x()	((read_c0_prid() & 0xfffffff0) == 0x00002010)
+#define cpu_is_r4600_v2_x()	((read_c0_prid() & 0xfffffff0) == 0x00002020)
 
 #define R4600_HIT_CACHEOP_WAR_IMPL					\
 do {									\
@@ -190,12 +197,12 @@ static inline void r4k_blast_icache_page_indexed_setup(void)
 	if (ic_lsize == 16)
 		r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
 	else if (ic_lsize == 32) {
-		if (TX49XX_ICACHE_INDEX_INV_WAR)
-			r4k_blast_icache_page_indexed =
-				tx49_blast_icache32_page_indexed;
-		else if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
+		if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
 			r4k_blast_icache_page_indexed =
 				blast_icache32_r4600_v1_page_indexed;
+		else if (TX49XX_ICACHE_INDEX_INV_WAR)
+			r4k_blast_icache_page_indexed =
+				tx49_blast_icache32_page_indexed;
 		else
 			r4k_blast_icache_page_indexed =
 				blast_icache32_page_indexed;
@@ -361,24 +368,33 @@ static void r4k_flush_cache_mm(struct mm_struct *mm)
 
 struct flush_cache_page_args {
 	struct vm_area_struct *vma;
-	unsigned long page;
+	unsigned long addr;
 };
 
 static inline void local_r4k_flush_cache_page(void *args)
 {
 	struct flush_cache_page_args *fcp_args = args;
 	struct vm_area_struct *vma = fcp_args->vma;
-	unsigned long page = fcp_args->page;
+	unsigned long addr = fcp_args->addr;
 	int exec = vma->vm_flags & VM_EXEC;
 	struct mm_struct *mm = vma->vm_mm;
 	pgd_t *pgdp;
+	pud_t *pudp;
 	pmd_t *pmdp;
 	pte_t *ptep;
 
-	page &= PAGE_MASK;
-	pgdp = pgd_offset(mm, page);
-	pmdp = pmd_offset(pgdp, page);
-	ptep = pte_offset(pmdp, page);
+	/*
+	 * If ownes no valid ASID yet, cannot possibly have gotten
+	 * this page into the cache.
+	 */
+	if (cpu_context(smp_processor_id(), mm) == 0)
+		return;
+
+	addr &= PAGE_MASK;
+	pgdp = pgd_offset(mm, addr);
+	pudp = pud_offset(pgdp, addr);
+	pmdp = pmd_offset(pudp, addr);
+	ptep = pte_offset(pmdp, addr);
 
 	/*
 	 * If the page isn't marked valid, the page cannot possibly be
@@ -395,12 +411,12 @@ static inline void local_r4k_flush_cache_page(void *args)
 	 */
 	if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
 		if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
-			r4k_blast_dcache_page(page);
+			r4k_blast_dcache_page(addr);
 			if (exec && !cpu_icache_snoops_remote_store)
-				r4k_blast_scache_page(page);
+				r4k_blast_scache_page(addr);
 		}
 		if (exec)
-			r4k_blast_icache_page(page);
+			r4k_blast_icache_page(addr);
 
 		return;
 	}
@@ -409,36 +425,30 @@ static inline void local_r4k_flush_cache_page(void *args)
 	 * Do indexed flush, too much work to get the (possible) TLB refills
 	 * to work correctly.
 	 */
-	page = INDEX_BASE + (page & (dcache_size - 1));
+	addr = INDEX_BASE + (addr & (dcache_size - 1));
 	if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
-		r4k_blast_dcache_page_indexed(page);
+		r4k_blast_dcache_page_indexed(addr);
 		if (exec && !cpu_icache_snoops_remote_store)
-			r4k_blast_scache_page_indexed(page);
+			r4k_blast_scache_page_indexed(addr);
 	}
 	if (exec) {
 		if (cpu_has_vtag_icache) {
 			int cpu = smp_processor_id();
 
-			if (cpu_context(cpu, vma->vm_mm) != 0)
-				drop_mmu_context(vma->vm_mm, cpu);
+			if (cpu_context(cpu, mm) != 0)
+				drop_mmu_context(mm, cpu);
 		} else
-			r4k_blast_icache_page_indexed(page);
+			r4k_blast_icache_page_indexed(addr);
 	}
 }
 
-static void r4k_flush_cache_page(struct vm_area_struct *vma, unsigned long page, unsigned long pfn)
+static void r4k_flush_cache_page(struct vm_area_struct *vma,
+	unsigned long addr, unsigned long pfn)
 {
 	struct flush_cache_page_args args;
 
-	/*
-	 * If ownes no valid ASID yet, cannot possibly have gotten
-	 * this page into the cache.
-	 */
-	if (cpu_context(smp_processor_id(), vma->vm_mm) == 0)
-		return;
-
 	args.vma = vma;
-	args.page = page;
+	args.addr = addr;
 
 	on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1);
 }
@@ -454,16 +464,16 @@ static void r4k_flush_data_cache_page(unsigned long addr)
 }
 
 struct flush_icache_range_args {
-	unsigned long start;
-	unsigned long end;
+	unsigned long __user start;
+	unsigned long __user end;
 };
 
 static inline void local_r4k_flush_icache_range(void *args)
 {
 	struct flush_icache_range_args *fir_args = args;
-	unsigned long dc_lsize = current_cpu_data.dcache.linesz;
-	unsigned long ic_lsize = current_cpu_data.icache.linesz;
-	unsigned long sc_lsize = current_cpu_data.scache.linesz;
+	unsigned long dc_lsize = cpu_dcache_line_size();
+	unsigned long ic_lsize = cpu_icache_line_size();
+	unsigned long sc_lsize = cpu_scache_line_size();
 	unsigned long start = fir_args->start;
 	unsigned long end = fir_args->end;
 	unsigned long addr, aend;
@@ -472,6 +482,7 @@ static inline void local_r4k_flush_icache_range(void *args)
 		if (end - start > dcache_size) {
 			r4k_blast_dcache();
 		} else {
+			R4600_HIT_CACHEOP_WAR_IMPL;
 			addr = start & ~(dc_lsize - 1);
 			aend = (end - 1) & ~(dc_lsize - 1);
 
@@ -492,7 +503,7 @@ static inline void local_r4k_flush_icache_range(void *args)
 				aend = (end - 1) & ~(sc_lsize - 1);
 
 				while (1) {
-					/* Hit_Writeback_Inv_D */
+					/* Hit_Writeback_Inv_SD */
 					protected_writeback_scache_line(addr);
 					if (addr == aend)
 						break;
@@ -517,7 +528,8 @@ static inline void local_r4k_flush_icache_range(void *args)
 	}
 }
 
-static void r4k_flush_icache_range(unsigned long start, unsigned long end)
+static void r4k_flush_icache_range(unsigned long __user start,
+	unsigned long __user end)
 {
 	struct flush_icache_range_args args;
 
@@ -525,6 +537,7 @@ static void r4k_flush_icache_range(unsigned long start, unsigned long end)
 	args.end = end;
 
 	on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1);
+	instruction_hazard();
 }
 
 /*
@@ -613,7 +626,7 @@ static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
 	BUG_ON(size == 0);
 
 	if (cpu_has_subset_pcaches) {
-		unsigned long sc_lsize = current_cpu_data.scache.linesz;
+		unsigned long sc_lsize = cpu_scache_line_size();
 
 		if (size >= scache_size) {
 			r4k_blast_scache();
@@ -639,7 +652,7 @@ static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
 	if (size >= dcache_size) {
 		r4k_blast_dcache();
 	} else {
-		unsigned long dc_lsize = current_cpu_data.dcache.linesz;
+		unsigned long dc_lsize = cpu_dcache_line_size();
 
 		R4600_HIT_CACHEOP_WAR_IMPL;
 		a = addr & ~(dc_lsize - 1);
@@ -663,7 +676,7 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
 	BUG_ON(size == 0);
 
 	if (cpu_has_subset_pcaches) {
-		unsigned long sc_lsize = current_cpu_data.scache.linesz;
+		unsigned long sc_lsize = cpu_scache_line_size();
 
 		if (size >= scache_size) {
 			r4k_blast_scache();
@@ -684,7 +697,7 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
 	if (size >= dcache_size) {
 		r4k_blast_dcache();
 	} else {
-		unsigned long dc_lsize = current_cpu_data.dcache.linesz;
+		unsigned long dc_lsize = cpu_dcache_line_size();
 
 		R4600_HIT_CACHEOP_WAR_IMPL;
 		a = addr & ~(dc_lsize - 1);
@@ -708,9 +721,9 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
  */
 static void local_r4k_flush_cache_sigtramp(void * arg)
 {
-	unsigned long ic_lsize = current_cpu_data.icache.linesz;
-	unsigned long dc_lsize = current_cpu_data.dcache.linesz;
-	unsigned long sc_lsize = current_cpu_data.scache.linesz;
+	unsigned long ic_lsize = cpu_icache_line_size();
+	unsigned long dc_lsize = cpu_dcache_line_size();
+	unsigned long sc_lsize = cpu_scache_line_size();
 	unsigned long addr = (unsigned long) arg;
 
 	R4600_HIT_CACHEOP_WAR_IMPL;
@@ -762,6 +775,7 @@ static inline void rm7k_erratum31(void)
 
 	for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
 		__asm__ __volatile__ (
+			".set push\n\t"
 			".set noreorder\n\t"
 			".set mips3\n\t"
 			"cache\t%1, 0(%0)\n\t"
@@ -776,8 +790,7 @@ static inline void rm7k_erratum31(void)
 			"cache\t%1, 0x1000(%0)\n\t"
 			"cache\t%1, 0x2000(%0)\n\t"
 			"cache\t%1, 0x3000(%0)\n\t"
-			".set\tmips0\n\t"
-			".set\treorder\n\t"
+			".set pop\n"
 			:
 			: "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
 	}
@@ -1011,9 +1024,19 @@ static void __init probe_pcache(void)
 	 * normally they'd suffer from aliases but magic in the hardware deals
 	 * with that for us so we don't need to take care ourselves.
 	 */
-	if (c->cputype != CPU_R10000 && c->cputype != CPU_R12000)
-		if (c->dcache.waysize > PAGE_SIZE)
-		        c->dcache.flags |= MIPS_CACHE_ALIASES;
+	switch (c->cputype) {
+	case CPU_20KC:
+	case CPU_25KF:
+	case CPU_R10000:
+	case CPU_R12000:
+	case CPU_SB1:
+		break;
+	case CPU_24K:
+		if (!(read_c0_config7() & (1 << 16)))
+	default:
+			if (c->dcache.waysize > PAGE_SIZE)
+				c->dcache.flags |= MIPS_CACHE_ALIASES;
+	}
 
 	switch (c->cputype) {
 	case CPU_20KC:
@@ -1024,7 +1047,11 @@ static void __init probe_pcache(void)
 		c->icache.flags |= MIPS_CACHE_VTAG;
 		break;
 
+	case CPU_AU1000:
 	case CPU_AU1500:
+	case CPU_AU1100:
+	case CPU_AU1550:
+	case CPU_AU1200:
 		c->icache.flags |= MIPS_CACHE_IC_F_DC;
 		break;
 	}
@@ -1102,7 +1129,6 @@ static int __init probe_scache(void)
 	return 1;
 }
 
-typedef int (*probe_func_t)(unsigned long);
 extern int r5k_sc_init(void);
 extern int rm7k_sc_init(void);
 
@@ -1110,7 +1136,6 @@ static void __init setup_scache(void)
 {
 	struct cpuinfo_mips *c = &current_cpu_data;
 	unsigned int config = read_c0_config();
-	probe_func_t probe_scache_kseg1;
 	int sc_present = 0;
 
 	/*
@@ -1123,8 +1148,7 @@ static void __init setup_scache(void)
 	case CPU_R4000MC:
 	case CPU_R4400SC:
 	case CPU_R4400MC:
-		probe_scache_kseg1 = (probe_func_t) (CKSEG1ADDR(&probe_scache));
-		sc_present = probe_scache_kseg1(config);
+		sc_present = run_uncached(probe_scache);
 		if (sc_present)
 			c->options |= MIPS_CPU_CACHE_CDEX_S;
 		break;
@@ -1198,7 +1222,7 @@ static inline void coherency_setup(void)
 	}
 }
 
-void __init ld_mmu_r4xx0(void)
+void __init r4k_cache_init(void)
 {
 	extern void build_clear_page(void);
 	extern void build_copy_page(void);
@@ -1206,15 +1230,11 @@ void __init ld_mmu_r4xx0(void)
 	struct cpuinfo_mips *c = &current_cpu_data;
 
 	/* Default cache error handler for R4000 and R5000 family */
-	memcpy((void *)(CAC_BASE   + 0x100), &except_vec2_generic, 0x80);
-	memcpy((void *)(UNCAC_BASE + 0x100), &except_vec2_generic, 0x80);
+	set_uncached_handler (0x100, &except_vec2_generic, 0x80);
 
 	probe_pcache();
 	setup_scache();
 
-	if (c->dcache.sets * c->dcache.ways > PAGE_SIZE)
-		c->dcache.flags |= MIPS_CACHE_ALIASES;
-
 	r4k_blast_dcache_page_setup();
 	r4k_blast_dcache_page_indexed_setup();
 	r4k_blast_dcache_setup();
@@ -1252,9 +1272,8 @@ void __init ld_mmu_r4xx0(void)
 	_dma_cache_inv		= r4k_dma_cache_inv;
 #endif
 
-	__flush_cache_all();
-	coherency_setup();
-
 	build_clear_page();
 	build_copy_page();
+	local_r4k___flush_cache_all(NULL);
+	coherency_setup();
 }
diff --git a/arch/mips/mm/c-sb1.c b/arch/mips/mm/c-sb1.c
index 502f68c664b..2f08b535f20 100644
--- a/arch/mips/mm/c-sb1.c
+++ b/arch/mips/mm/c-sb1.c
@@ -235,7 +235,7 @@ static inline void __sb1_flush_icache_range(unsigned long start,
 /*
  * Invalidate all caches on this CPU
  */
-static void local_sb1___flush_cache_all(void)
+static void __attribute_used__ local_sb1___flush_cache_all(void)
 {
 	__sb1_writeback_inv_dcache_all();
 	__sb1_flush_icache_all();
@@ -492,19 +492,17 @@ static __init void probe_cache_sizes(void)
 }
 
 /*
- * This is called from loadmmu.c.  We have to set up all the
+ * This is called from cache.c.  We have to set up all the
  * memory management function pointers, as well as initialize
  * the caches and tlbs
  */
-void ld_mmu_sb1(void)
+void sb1_cache_init(void)
 {
 	extern char except_vec2_sb1;
 	extern char handle_vec2_sb1;
 
 	/* Special cache error handler for SB1 */
-	memcpy((void *)(CAC_BASE   + 0x100), &except_vec2_sb1, 0x80);
-	memcpy((void *)(UNCAC_BASE + 0x100), &except_vec2_sb1, 0x80);
-	memcpy((void *)CKSEG1ADDR(&handle_vec2_sb1), &handle_vec2_sb1, 0x80);
+	set_uncached_handler (0x100, &except_vec2_sb1, 0x80);
 
 	probe_cache_sizes();
 
diff --git a/arch/mips/mm/c-tx39.c b/arch/mips/mm/c-tx39.c
index ff5afab64b2..0a97a9434eb 100644
--- a/arch/mips/mm/c-tx39.c
+++ b/arch/mips/mm/c-tx39.c
@@ -167,15 +167,16 @@ static void tx39_flush_cache_mm(struct mm_struct *mm)
 static void tx39_flush_cache_range(struct vm_area_struct *vma,
 	unsigned long start, unsigned long end)
 {
-	struct mm_struct *mm = vma->vm_mm;
+	int exec;
 
-	if (!cpu_has_dc_aliases)
+	if (!(cpu_context(smp_processor_id(), vma->vm_mm)))
 		return;
 
-	if (cpu_context(smp_processor_id(), mm) != 0) {
+	exec = vma->vm_flags & VM_EXEC;
+	if (cpu_has_dc_aliases || exec)
 		tx39_blast_dcache();
+	if (exec)
 		tx39_blast_icache();
-	}
 }
 
 static void tx39_flush_cache_page(struct vm_area_struct *vma, unsigned long page, unsigned long pfn)
@@ -183,6 +184,7 @@ static void tx39_flush_cache_page(struct vm_area_struct *vma, unsigned long page
 	int exec = vma->vm_flags & VM_EXEC;
 	struct mm_struct *mm = vma->vm_mm;
 	pgd_t *pgdp;
+	pud_t *pudp;
 	pmd_t *pmdp;
 	pte_t *ptep;
 
@@ -195,7 +197,8 @@ static void tx39_flush_cache_page(struct vm_area_struct *vma, unsigned long page
 
 	page &= PAGE_MASK;
 	pgdp = pgd_offset(mm, page);
-	pmdp = pmd_offset(pgdp, page);
+	pudp = pud_offset(pgdp, page);
+	pmdp = pmd_offset(pudp, page);
 	ptep = pte_offset(pmdp, page);
 
 	/*
@@ -407,7 +410,7 @@ static __init void tx39_probe_cache(void)
 	}
 }
 
-void __init ld_mmu_tx39(void)
+void __init tx39_cache_init(void)
 {
 	extern void build_clear_page(void);
 	extern void build_copy_page(void);
@@ -490,4 +493,5 @@ void __init ld_mmu_tx39(void)
 
 	build_clear_page();
 	build_copy_page();
+	tx39h_flush_icache_all();
 }
diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c
index 1d95cdb77be..314701a66b1 100644
--- a/arch/mips/mm/cache.c
+++ b/arch/mips/mm/cache.c
@@ -23,8 +23,10 @@ void (*__flush_cache_all)(void);
 void (*flush_cache_mm)(struct mm_struct *mm);
 void (*flush_cache_range)(struct vm_area_struct *vma, unsigned long start,
 	unsigned long end);
-void (*flush_cache_page)(struct vm_area_struct *vma, unsigned long page, unsigned long pfn);
-void (*flush_icache_range)(unsigned long start, unsigned long end);
+void (*flush_cache_page)(struct vm_area_struct *vma, unsigned long page,
+	unsigned long pfn);
+void (*flush_icache_range)(unsigned long __user start,
+	unsigned long __user end);
 void (*flush_icache_page)(struct vm_area_struct *vma, struct page *page);
 
 /* MIPS specific cache operations */
@@ -32,6 +34,8 @@ void (*flush_cache_sigtramp)(unsigned long addr);
 void (*flush_data_cache_page)(unsigned long addr);
 void (*flush_icache_all)(void);
 
+EXPORT_SYMBOL(flush_data_cache_page);
+
 #ifdef CONFIG_DMA_NONCOHERENT
 
 /* DMA cache operations. */
@@ -49,10 +53,12 @@ EXPORT_SYMBOL(_dma_cache_inv);
  * We could optimize the case where the cache argument is not BCACHE but
  * that seems very atypical use ...
  */
-asmlinkage int sys_cacheflush(unsigned long addr, unsigned long int bytes,
-	unsigned int cache)
+asmlinkage int sys_cacheflush(unsigned long __user addr,
+	unsigned long bytes, unsigned int cache)
 {
-	if (!access_ok(VERIFY_WRITE, (void *) addr, bytes))
+	if (bytes == 0)
+		return 0;
+	if (!access_ok(VERIFY_WRITE, (void __user *) addr, bytes))
 		return -EFAULT;
 
 	flush_icache_range(addr, addr + bytes);
@@ -100,58 +106,48 @@ void __update_cache(struct vm_area_struct *vma, unsigned long address,
 	}
 }
 
-extern void ld_mmu_r23000(void);
-extern void ld_mmu_r4xx0(void);
-extern void ld_mmu_tx39(void);
-extern void ld_mmu_r6000(void);
-extern void ld_mmu_tfp(void);
-extern void ld_mmu_andes(void);
-extern void ld_mmu_sb1(void);
+#define __weak __attribute__((weak))
+
+static char cache_panic[] __initdata = "Yeee, unsupported cache architecture.";
 
 void __init cpu_cache_init(void)
 {
-	if (cpu_has_4ktlb) {
-#if defined(CONFIG_CPU_R4X00)  || defined(CONFIG_CPU_VR41XX) || \
-    defined(CONFIG_CPU_R4300)  || defined(CONFIG_CPU_R5000)  || \
-    defined(CONFIG_CPU_NEVADA) || defined(CONFIG_CPU_R5432)  || \
-    defined(CONFIG_CPU_R5500)  || defined(CONFIG_CPU_MIPS32) || \
-    defined(CONFIG_CPU_MIPS64) || defined(CONFIG_CPU_TX49XX) || \
-    defined(CONFIG_CPU_RM7000) || defined(CONFIG_CPU_RM9000)
-		ld_mmu_r4xx0();
-#endif
-	} else switch (current_cpu_data.cputype) {
-#ifdef CONFIG_CPU_R3000
-	case CPU_R2000:
-	case CPU_R3000:
-	case CPU_R3000A:
-	case CPU_R3081E:
-		ld_mmu_r23000();
-		break;
-#endif
-#ifdef CONFIG_CPU_TX39XX
-	case CPU_TX3912:
-	case CPU_TX3922:
-	case CPU_TX3927:
-		ld_mmu_tx39();
-		break;
-#endif
-#ifdef CONFIG_CPU_R10000
-	case CPU_R10000:
-	case CPU_R12000:
-		ld_mmu_r4xx0();
-		break;
-#endif
-#ifdef CONFIG_CPU_SB1
-	case CPU_SB1:
-		ld_mmu_sb1();
-		break;
-#endif
-
-	case CPU_R8000:
-		panic("R8000 is unsupported");
-		break;
-
-	default:
-		panic("Yeee, unsupported cache architecture.");
+	if (cpu_has_3k_cache) {
+		extern void __weak r3k_cache_init(void);
+
+		r3k_cache_init();
+		return;
+	}
+	if (cpu_has_6k_cache) {
+		extern void __weak r6k_cache_init(void);
+
+		r6k_cache_init();
+		return;
+	}
+	if (cpu_has_4k_cache) {
+		extern void __weak r4k_cache_init(void);
+
+		r4k_cache_init();
+		return;
 	}
+	if (cpu_has_8k_cache) {
+		extern void __weak r8k_cache_init(void);
+
+		r8k_cache_init();
+		return;
+	}
+	if (cpu_has_tx39_cache) {
+		extern void __weak tx39_cache_init(void);
+
+		tx39_cache_init();
+		return;
+	}
+	if (cpu_has_sb1_cache) {
+		extern void __weak sb1_cache_init(void);
+
+		sb1_cache_init();
+		return;
+	}
+
+	panic(cache_panic);
 }
diff --git a/arch/mips/mm/cerr-sb1.c b/arch/mips/mm/cerr-sb1.c
index 7166ffe6350..1cf3c6006cc 100644
--- a/arch/mips/mm/cerr-sb1.c
+++ b/arch/mips/mm/cerr-sb1.c
@@ -19,13 +19,19 @@
 #include <linux/sched.h>
 #include <asm/mipsregs.h>
 #include <asm/sibyte/sb1250.h>
+#include <asm/sibyte/sb1250_regs.h>
 
-#ifndef CONFIG_SIBYTE_BUS_WATCHER
+#if !defined(CONFIG_SIBYTE_BUS_WATCHER) || defined(CONFIG_SIBYTE_BW_TRACE)
 #include <asm/io.h>
-#include <asm/sibyte/sb1250_regs.h>
 #include <asm/sibyte/sb1250_scd.h>
 #endif
 
+/*
+ * We'd like to dump the L2_ECC_TAG register on errors, but errata make
+ * that unsafe... So for now we don't.  (BCM1250/BCM112x erratum SOC-48.)
+ */
+#undef DUMP_L2_ECC_TAG_ON_ERROR
+
 /* SB1 definitions */
 
 /* XXX should come from config1 XXX */
@@ -139,12 +145,18 @@ static inline void breakout_cerrd(unsigned int val)
 static void check_bus_watcher(void)
 {
 	uint32_t status, l2_err, memio_err;
+#ifdef DUMP_L2_ECC_TAG_ON_ERROR
+	uint64_t l2_tag;
+#endif
 
 	/* Destructive read, clears register and interrupt */
 	status = csr_in32(IOADDR(A_SCD_BUS_ERR_STATUS));
 	/* Bit 31 is always on, but there's no #define for that */
 	if (status & ~(1UL << 31)) {
 		l2_err = csr_in32(IOADDR(A_BUS_L2_ERRORS));
+#ifdef DUMP_L2_ECC_TAG_ON_ERROR
+		l2_tag = in64(IO_SPACE_BASE | A_L2_ECC_TAG);
+#endif
 		memio_err = csr_in32(IOADDR(A_BUS_MEM_IO_ERRORS));
 		prom_printf("Bus watcher error counters: %08x %08x\n", l2_err, memio_err);
 		prom_printf("\nLast recorded signature:\n");
@@ -153,6 +165,9 @@ static void check_bus_watcher(void)
 		       (int)(G_SCD_BERR_TID(status) >> 6),
 		       (int)G_SCD_BERR_RID(status),
 		       (int)G_SCD_BERR_DCODE(status));
+#ifdef DUMP_L2_ECC_TAG_ON_ERROR
+		prom_printf("Last L2 tag w/ bad ECC: %016llx\n", l2_tag);
+#endif
 	} else {
 		prom_printf("Bus watcher indicates no error\n");
 	}
@@ -166,6 +181,16 @@ asmlinkage void sb1_cache_error(void)
 	uint64_t cerr_dpa;
 	uint32_t errctl, cerr_i, cerr_d, dpalo, dpahi, eepc, res;
 
+#ifdef CONFIG_SIBYTE_BW_TRACE
+	/* Freeze the trace buffer now */
+#if defined(CONFIG_SIBYTE_BCM1x55) || defined(CONFIG_SIBYTE_BCM1x80)
+	csr_out32(M_BCM1480_SCD_TRACE_CFG_FREEZE, IO_SPACE_BASE | A_SCD_TRACE_CFG);
+#else
+	csr_out32(M_SCD_TRACE_CFG_FREEZE, IO_SPACE_BASE | A_SCD_TRACE_CFG);
+#endif
+	prom_printf("Trace buffer frozen\n");
+#endif
+
 	prom_printf("Cache error exception on CPU %x:\n",
 		    (read_c0_prid() >> 25) & 0x7);
 
@@ -229,11 +254,19 @@ asmlinkage void sb1_cache_error(void)
 
 	check_bus_watcher();
 
-	while (1);
 	/*
-	 * This tends to make things get really ugly; let's just stall instead.
-	 *    panic("Can't handle the cache error!");
+	 * Calling panic() when a fatal cache error occurs scrambles the
+	 * state of the system (and the cache), making it difficult to
+	 * investigate after the fact.  However, if you just stall the CPU,
+	 * the other CPU may keep on running, which is typically very
+	 * undesirable.
 	 */
+#ifdef CONFIG_SB1_CERR_STALL
+	while (1)
+		;
+#else
+	panic("unhandled cache error");
+#endif
 }
 
 
@@ -434,7 +467,8 @@ static struct dc_state dc_states[] = {
 };
 
 #define DC_TAG_VALID(state) \
-    (((state) == 0xf) || ((state) == 0x13) || ((state) == 0x19) || ((state == 0x16)) || ((state) == 0x1c))
+    (((state) == 0x0) || ((state) == 0xf) || ((state) == 0x13) || \
+     ((state) == 0x19) || ((state) == 0x16) || ((state) == 0x1c))
 
 static char *dc_state_str(unsigned char state)
 {
@@ -505,6 +539,7 @@ static uint32_t extract_dc(unsigned short addr, int data)
 			uint64_t datalo;
 			uint32_t datalohi, datalolo, datahi;
 			int offset;
+			char bad_ecc = 0;
 
 			for (offset = 0; offset < 4; offset++) {
 				/* Index-load-data-D */
@@ -525,8 +560,7 @@ static uint32_t extract_dc(unsigned short addr, int data)
 				ecc = dc_ecc(datalo);
 				if (ecc != datahi) {
 					int bits = 0;
-					prom_printf("  ** bad ECC (%02x %02x) ->",
-						    datahi, ecc);
+					bad_ecc |= 1 << (3-offset);
 					ecc ^= datahi;
 					while (ecc) {
 						if (ecc & 1) bits++;
@@ -537,6 +571,10 @@ static uint32_t extract_dc(unsigned short addr, int data)
 				prom_printf("  %02X-%016llX", datahi, datalo);
 			}
 			prom_printf("\n");
+			if (bad_ecc)
+				prom_printf("  dwords w/ bad ECC: %d %d %d %d\n",
+					    !!(bad_ecc & 8), !!(bad_ecc & 4),
+					    !!(bad_ecc & 2), !!(bad_ecc & 1));
 		}
 	}
 	return res;
diff --git a/arch/mips/mm/cex-sb1.S b/arch/mips/mm/cex-sb1.S
index 2c3a23aa88c..0e71580774f 100644
--- a/arch/mips/mm/cex-sb1.S
+++ b/arch/mips/mm/cex-sb1.S
@@ -64,6 +64,10 @@ LEAF(except_vec2_sb1)
 	sd	k0,0x170($0)
 	sd	k1,0x178($0)
 
+#if CONFIG_SB1_CEX_ALWAYS_FATAL
+	j	handle_vec2_sb1
+	 nop
+#else
 	/*
 	 * M_ERRCTL_RECOVERABLE is bit 31, which makes it easy to tell
 	 * if we can fast-path out of here for a h/w-recovered error.
@@ -134,6 +138,7 @@ unrecoverable:
 	/* Unrecoverable Icache or Dcache error; log it and/or fail */
 	j	handle_vec2_sb1
 	 nop
+#endif
 
 END(except_vec2_sb1)
 
diff --git a/arch/mips/mm/dma-coherent.c b/arch/mips/mm/dma-coherent.c
index 97a50d38c98..f6b3c722230 100644
--- a/arch/mips/mm/dma-coherent.c
+++ b/arch/mips/mm/dma-coherent.c
@@ -9,16 +9,16 @@
  */
 #include <linux/config.h>
 #include <linux/types.h>
+#include <linux/dma-mapping.h>
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/string.h>
-#include <linux/pci.h>
 
 #include <asm/cache.h>
 #include <asm/io.h>
 
 void *dma_alloc_noncoherent(struct device *dev, size_t size,
-	dma_addr_t * dma_handle, int gfp)
+	dma_addr_t * dma_handle, gfp_t gfp)
 {
 	void *ret;
 	/* ignore region specifiers */
@@ -39,7 +39,7 @@ void *dma_alloc_noncoherent(struct device *dev, size_t size,
 EXPORT_SYMBOL(dma_alloc_noncoherent);
 
 void *dma_alloc_coherent(struct device *dev, size_t size,
-	dma_addr_t * dma_handle, int gfp)
+	dma_addr_t * dma_handle, gfp_t gfp)
 	__attribute__((alias("dma_alloc_noncoherent")));
 
 EXPORT_SYMBOL(dma_alloc_coherent);
diff --git a/arch/mips/mm/dma-ip27.c b/arch/mips/mm/dma-ip27.c
index aa7c94b5d78..8da19fd22ac 100644
--- a/arch/mips/mm/dma-ip27.c
+++ b/arch/mips/mm/dma-ip27.c
@@ -22,7 +22,7 @@
 	pdev_to_baddr(to_pci_dev(dev), (addr))
 
 void *dma_alloc_noncoherent(struct device *dev, size_t size,
-	dma_addr_t * dma_handle, int gfp)
+	dma_addr_t * dma_handle, gfp_t gfp)
 {
 	void *ret;
 
@@ -44,7 +44,7 @@ void *dma_alloc_noncoherent(struct device *dev, size_t size,
 EXPORT_SYMBOL(dma_alloc_noncoherent);
 
 void *dma_alloc_coherent(struct device *dev, size_t size,
-	dma_addr_t * dma_handle, int gfp)
+	dma_addr_t * dma_handle, gfp_t gfp)
 	__attribute__((alias("dma_alloc_noncoherent")));
 
 EXPORT_SYMBOL(dma_alloc_coherent);
diff --git a/arch/mips/mm/dma-ip32.c b/arch/mips/mm/dma-ip32.c
index 2cbe196c35f..a7e3072ff78 100644
--- a/arch/mips/mm/dma-ip32.c
+++ b/arch/mips/mm/dma-ip32.c
@@ -37,7 +37,7 @@
 #define RAM_OFFSET_MASK	0x3fffffff
 
 void *dma_alloc_noncoherent(struct device *dev, size_t size,
-	dma_addr_t * dma_handle, int gfp)
+	dma_addr_t * dma_handle, gfp_t gfp)
 {
 	void *ret;
 	/* ignore region specifiers */
@@ -61,7 +61,7 @@ void *dma_alloc_noncoherent(struct device *dev, size_t size,
 EXPORT_SYMBOL(dma_alloc_noncoherent);
 
 void *dma_alloc_coherent(struct device *dev, size_t size,
-	dma_addr_t * dma_handle, int gfp)
+	dma_addr_t * dma_handle, gfp_t gfp)
 {
 	void *ret;
 
diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c
index 59e54f12212..cd4ea8474f8 100644
--- a/arch/mips/mm/dma-noncoherent.c
+++ b/arch/mips/mm/dma-noncoherent.c
@@ -24,7 +24,7 @@
  */
 
 void *dma_alloc_noncoherent(struct device *dev, size_t size,
-	dma_addr_t * dma_handle, int gfp)
+	dma_addr_t * dma_handle, gfp_t gfp)
 {
 	void *ret;
 	/* ignore region specifiers */
@@ -45,7 +45,7 @@ void *dma_alloc_noncoherent(struct device *dev, size_t size,
 EXPORT_SYMBOL(dma_alloc_noncoherent);
 
 void *dma_alloc_coherent(struct device *dev, size_t size,
-	dma_addr_t * dma_handle, int gfp)
+	dma_addr_t * dma_handle, gfp_t gfp)
 {
 	void *ret;
 
@@ -105,22 +105,7 @@ dma_addr_t dma_map_single(struct device *dev, void *ptr, size_t size,
 {
 	unsigned long addr = (unsigned long) ptr;
 
-	switch (direction) {
-	case DMA_TO_DEVICE:
-		dma_cache_wback(addr, size);
-		break;
-
-	case DMA_FROM_DEVICE:
-		dma_cache_inv(addr, size);
-		break;
-
-	case DMA_BIDIRECTIONAL:
-		dma_cache_wback_inv(addr, size);
-		break;
-
-	default:
-		BUG();
-	}
+	__dma_sync(addr, size, direction);
 
 	return virt_to_phys(ptr);
 }
@@ -133,22 +118,7 @@ void dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
 	unsigned long addr;
 	addr = dma_addr + PAGE_OFFSET;
 
-	switch (direction) {
-	case DMA_TO_DEVICE:
-		//dma_cache_wback(addr, size);
-		break;
-
-	case DMA_FROM_DEVICE:
-		//dma_cache_inv(addr, size);
-		break;
-
-	case DMA_BIDIRECTIONAL:
-		//dma_cache_wback_inv(addr, size);
-		break;
-
-	default:
-		BUG();
-	}
+	//__dma_sync(addr, size, direction);
 }
 
 EXPORT_SYMBOL(dma_unmap_single);
@@ -164,10 +134,11 @@ int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
 		unsigned long addr;
 
 		addr = (unsigned long) page_address(sg->page);
-		if (addr)
+		if (addr) {
 			__dma_sync(addr + sg->offset, sg->length, direction);
-		sg->dma_address = (dma_addr_t)
-			(page_to_phys(sg->page) + sg->offset);
+			sg->dma_address = (dma_addr_t)page_to_phys(sg->page)
+					  + sg->offset;
+		}
 	}
 
 	return nents;
@@ -218,9 +189,8 @@ void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
 
 	for (i = 0; i < nhwentries; i++, sg++) {
 		addr = (unsigned long) page_address(sg->page);
-		if (!addr)
-			continue;
-		dma_cache_wback_inv(addr + sg->offset, sg->length);
+		if (addr)
+			__dma_sync(addr + sg->offset, sg->length, direction);
 	}
 }
 
diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
index ec8077c74e9..2d9624fd10e 100644
--- a/arch/mips/mm/fault.c
+++ b/arch/mips/mm/fault.c
@@ -25,6 +25,7 @@
 #include <asm/system.h>
 #include <asm/uaccess.h>
 #include <asm/ptrace.h>
+#include <asm/highmem.h>		/* For VMALLOC_END */
 
 /*
  * This routine handles page faults.  It determines the address,
@@ -57,7 +58,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
 	 * only copy the information from the master page table,
 	 * nothing more.
 	 */
-	if (unlikely(address >= VMALLOC_START))
+	if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END))
 		goto vmalloc_fault;
 
 	/*
@@ -140,7 +141,7 @@ bad_area_nosemaphore:
 		info.si_signo = SIGSEGV;
 		info.si_errno = 0;
 		/* info.si_code has been set above */
-		info.si_addr = (void *) address;
+		info.si_addr = (void __user *) address;
 		force_sig_info(SIGSEGV, &info, tsk);
 		return;
 	}
@@ -196,7 +197,7 @@ do_sigbus:
 	info.si_signo = SIGBUS;
 	info.si_errno = 0;
 	info.si_code = BUS_ADRERR;
-	info.si_addr = (void *) address;
+	info.si_addr = (void __user *) address;
 	force_sig_info(SIGBUS, &info, tsk);
 
 	return;
@@ -212,6 +213,7 @@ vmalloc_fault:
 		 */
 		int offset = __pgd_offset(address);
 		pgd_t *pgd, *pgd_k;
+		pud_t *pud, *pud_k;
 		pmd_t *pmd, *pmd_k;
 		pte_t *pte_k;
 
@@ -222,8 +224,13 @@ vmalloc_fault:
 			goto no_context;
 		set_pgd(pgd, *pgd_k);
 
-		pmd = pmd_offset(pgd, address);
-		pmd_k = pmd_offset(pgd_k, address);
+		pud = pud_offset(pgd, address);
+		pud_k = pud_offset(pgd_k, address);
+		if (!pud_present(*pud_k))
+			goto no_context;
+
+		pmd = pmd_offset(pud, address);
+		pmd_k = pmd_offset(pud_k, address);
 		if (!pmd_present(*pmd_k))
 			goto no_context;
 		set_pmd(pmd, *pmd_k);
diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c
index dd5e2e31885..1f7b37b38f5 100644
--- a/arch/mips/mm/highmem.c
+++ b/arch/mips/mm/highmem.c
@@ -83,6 +83,25 @@ void __kunmap_atomic(void *kvaddr, enum km_type type)
 	preempt_check_resched();
 }
 
+/*
+ * This is the same as kmap_atomic() but can map memory that doesn't
+ * have a struct page associated with it.
+ */
+void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
+{
+	enum fixed_addresses idx;
+	unsigned long vaddr;
+
+	inc_preempt_count();
+
+	idx = type + KM_TYPE_NR*smp_processor_id();
+	vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
+	set_pte(kmap_pte-idx, pfn_pte(pfn, kmap_prot));
+	flush_tlb_one(vaddr);
+
+	return (void*) vaddr;
+}
+
 struct page *__kmap_atomic_to_page(void *ptr)
 {
 	unsigned long idx, vaddr = (unsigned long)ptr;
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index dc6830b10fa..f75ab748e8c 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -83,7 +83,7 @@ pte_t *kmap_pte;
 pgprot_t kmap_prot;
 
 #define kmap_get_fixmap_pte(vaddr)					\
-	pte_offset_kernel(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr))
+	pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr))
 
 static void __init kmap_init(void)
 {
@@ -96,36 +96,42 @@ static void __init kmap_init(void)
 	kmap_prot = PAGE_KERNEL;
 }
 
-#ifdef CONFIG_64BIT
-static void __init fixrange_init(unsigned long start, unsigned long end,
+#ifdef CONFIG_32BIT
+void __init fixrange_init(unsigned long start, unsigned long end,
 	pgd_t *pgd_base)
 {
 	pgd_t *pgd;
+	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
-	int i, j;
+	int i, j, k;
 	unsigned long vaddr;
 
 	vaddr = start;
 	i = __pgd_offset(vaddr);
-	j = __pmd_offset(vaddr);
+	j = __pud_offset(vaddr);
+	k = __pmd_offset(vaddr);
 	pgd = pgd_base + i;
 
 	for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) {
-		pmd = (pmd_t *)pgd;
-		for (; (j < PTRS_PER_PMD) && (vaddr != end); pmd++, j++) {
-			if (pmd_none(*pmd)) {
-				pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
-				set_pmd(pmd, __pmd(pte));
-				if (pte != pte_offset_kernel(pmd, 0))
-					BUG();
+		pud = (pud_t *)pgd;
+		for ( ; (j < PTRS_PER_PUD) && (vaddr != end); pud++, j++) {
+			pmd = (pmd_t *)pud;
+			for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {
+				if (pmd_none(*pmd)) {
+					pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
+					set_pmd(pmd, __pmd(pte));
+					if (pte != pte_offset_kernel(pmd, 0))
+						BUG();
+				}
+				vaddr += PMD_SIZE;
 			}
-			vaddr += PMD_SIZE;
+			k = 0;
 		}
 		j = 0;
 	}
 }
-#endif /* CONFIG_64BIT */
+#endif /* CONFIG_32BIT */
 #endif /* CONFIG_HIGHMEM */
 
 #ifndef CONFIG_NEED_MULTIPLE_NODES
diff --git a/arch/mips/mm/ioremap.c b/arch/mips/mm/ioremap.c
index adf352273f6..3101d1db559 100644
--- a/arch/mips/mm/ioremap.c
+++ b/arch/mips/mm/ioremap.c
@@ -55,7 +55,7 @@ static inline int remap_area_pmd(pmd_t * pmd, unsigned long address,
 	if (address >= end)
 		BUG();
 	do {
-		pte_t * pte = pte_alloc_kernel(&init_mm, pmd, address);
+		pte_t * pte = pte_alloc_kernel(pmd, address);
 		if (!pte)
 			return -ENOMEM;
 		remap_area_pte(pte, address, end - address, address + phys_addr, flags);
@@ -77,11 +77,15 @@ static int remap_area_pages(unsigned long address, phys_t phys_addr,
 	flush_cache_all();
 	if (address >= end)
 		BUG();
-	spin_lock(&init_mm.page_table_lock);
 	do {
+		pud_t *pud;
 		pmd_t *pmd;
-		pmd = pmd_alloc(&init_mm, dir, address);
+
 		error = -ENOMEM;
+		pud = pud_alloc(&init_mm, dir, address);
+		if (!pud)
+			break;
+		pmd = pmd_alloc(&init_mm, pud, address);
 		if (!pmd)
 			break;
 		if (remap_area_pmd(pmd, address, end - address,
@@ -91,21 +95,11 @@ static int remap_area_pages(unsigned long address, phys_t phys_addr,
 		address = (address + PGDIR_SIZE) & PGDIR_MASK;
 		dir++;
 	} while (address && (address < end));
-	spin_unlock(&init_mm.page_table_lock);
 	flush_tlb_all();
 	return error;
 }
 
 /*
- * Allow physical addresses to be fixed up to help 36 bit peripherals.
- */
-phys_t __attribute__ ((weak))
-fixup_bigphys_addr(phys_t phys_addr, phys_t size)
-{
-	return phys_addr;
-}
-
-/*
  * Generic mapping function (not visible outside):
  */
 
@@ -121,7 +115,7 @@ fixup_bigphys_addr(phys_t phys_addr, phys_t size)
 
 #define IS_LOW512(addr) (!((phys_t)(addr) & (phys_t) ~0x1fffffffULL))
 
-void * __ioremap(phys_t phys_addr, phys_t size, unsigned long flags)
+void __iomem * __ioremap(phys_t phys_addr, phys_t size, unsigned long flags)
 {
 	struct vm_struct * area;
 	unsigned long offset;
@@ -141,7 +135,7 @@ void * __ioremap(phys_t phys_addr, phys_t size, unsigned long flags)
 	 */
 	if (IS_LOW512(phys_addr) && IS_LOW512(last_addr) &&
 	    flags == _CACHE_UNCACHED)
-		return (void *) KSEG1ADDR(phys_addr);
+		return (void __iomem *) CKSEG1ADDR(phys_addr);
 
 	/*
 	 * Don't allow anybody to remap normal RAM that we're using..
@@ -177,10 +171,10 @@ void * __ioremap(phys_t phys_addr, phys_t size, unsigned long flags)
 		return NULL;
 	}
 
-	return (void *) (offset + (char *)addr);
+	return (void __iomem *) (offset + (char *)addr);
 }
 
-#define IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == KSEG1)
+#define IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == CKSEG1)
 
 void __iounmap(volatile void __iomem *addr)
 {
@@ -190,10 +184,8 @@ void __iounmap(volatile void __iomem *addr)
 		return;
 
 	p = remove_vm_area((void *) (PAGE_MASK & (unsigned long __force) addr));
-	if (!p) {
+	if (!p)
 		printk(KERN_ERR "iounmap: bad address %p\n", addr);
-		return;
-	}
 
         kfree(p);
 }
diff --git a/arch/mips/mm/pg-r4k.c b/arch/mips/mm/pg-r4k.c
index 9f8b1654157..f51e180072e 100644
--- a/arch/mips/mm/pg-r4k.c
+++ b/arch/mips/mm/pg-r4k.c
@@ -25,7 +25,10 @@
 #include <asm/cpu.h>
 #include <asm/war.h>
 
-#define half_scache_line_size()		(cpu_scache_line_size() >> 1)
+#define half_scache_line_size()	(cpu_scache_line_size() >> 1)
+#define cpu_is_r4600_v1_x()	((read_c0_prid() & 0xfffffff0) == 0x00002010)
+#define cpu_is_r4600_v2_x()	((read_c0_prid() & 0xfffffff0) == 0x00002020)
+
 
 /*
  * Maximum sizes:
@@ -198,15 +201,15 @@ static inline void build_cdex_p(void)
 	if (store_offset & (cpu_dcache_line_size() - 1))
 		return;
 
-	if (R4600_V1_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2010)) {
+	if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) {
 		build_nop();
 		build_nop();
 		build_nop();
 		build_nop();
 	}
 
-	if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
-		build_insn_word(0x8c200000);	/* lw      $zero, ($at) */
+	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
+		build_insn_word(0x3c01a000);	/* lui     $at, 0xa000  */
 
 	mi.c_format.opcode     = cache_op;
 	mi.c_format.rs         = 4;		/* $a0 */
@@ -361,7 +364,7 @@ void __init build_clear_page(void)
 
 	build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_clear : 0));
 
-	if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
+	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
 		build_insn_word(0x3c01a000);	/* lui     $at, 0xa000  */
 
 dest = label();
@@ -404,9 +407,6 @@ dest = label();
 
 	build_jr_ra();
 
-	flush_icache_range((unsigned long)&clear_page_array,
-	                   (unsigned long) epc);
-
 	BUG_ON(epc > clear_page_array + ARRAY_SIZE(clear_page_array));
 }
 
@@ -420,7 +420,7 @@ void __init build_copy_page(void)
 
 	build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_copy : 0));
 
-	if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
+	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
 		build_insn_word(0x3c01a000);	/* lui     $at, 0xa000  */
 
 dest = label();
@@ -482,8 +482,5 @@ dest = label();
 
 	build_jr_ra();
 
-	flush_icache_range((unsigned long)&copy_page_array,
-	                   (unsigned long) epc);
-
 	BUG_ON(epc > copy_page_array + ARRAY_SIZE(copy_page_array));
 }
diff --git a/arch/mips/mm/pg-sb1.c b/arch/mips/mm/pg-sb1.c
index 1b6df7133c1..148c65b9cd8 100644
--- a/arch/mips/mm/pg-sb1.c
+++ b/arch/mips/mm/pg-sb1.c
@@ -60,7 +60,8 @@ static inline void clear_page_cpu(void *page)
 	"	.set	noreorder	\n"
 #ifdef CONFIG_CPU_HAS_PREFETCH
 	"	daddiu	%0, %0, 128	\n"
-	"	pref	" SB1_PREF_STORE_STREAMED_HINT ", -128(%0)  \n"  /* Prefetch the first 4 lines */
+	"	pref	" SB1_PREF_STORE_STREAMED_HINT ", -128(%0)  \n"
+					     /* Prefetch the first 4 lines */
 	"	pref	" SB1_PREF_STORE_STREAMED_HINT ",  -96(%0)  \n"
 	"	pref	" SB1_PREF_STORE_STREAMED_HINT ",  -64(%0)  \n"
 	"	pref	" SB1_PREF_STORE_STREAMED_HINT ",  -32(%0)  \n"
@@ -106,7 +107,8 @@ static inline void copy_page_cpu(void *to, void *from)
 #ifdef CONFIG_CPU_HAS_PREFETCH
 	"	daddiu	%0, %0, 128	\n"
 	"	daddiu	%1, %1, 128	\n"
-	"	pref	" SB1_PREF_LOAD_STREAMED_HINT  ", -128(%0)\n"  /* Prefetch the first 4 lines */
+	"	pref	" SB1_PREF_LOAD_STREAMED_HINT  ", -128(%0)\n"
+					     /* Prefetch the first 4 lines */
 	"	pref	" SB1_PREF_STORE_STREAMED_HINT ", -128(%1)\n"
 	"	pref	" SB1_PREF_LOAD_STREAMED_HINT  ",  -96(%0)\n"
 	"	pref	" SB1_PREF_STORE_STREAMED_HINT ",  -96(%1)\n"
@@ -207,66 +209,73 @@ typedef struct dmadscr_s {
 	u64 pad_b;
 } dmadscr_t;
 
-static dmadscr_t page_descr[NR_CPUS] __attribute__((aligned(SMP_CACHE_BYTES)));
+static dmadscr_t page_descr[DM_NUM_CHANNELS]
+	__attribute__((aligned(SMP_CACHE_BYTES)));
 
 void sb1_dma_init(void)
 {
-	int cpu = smp_processor_id();
-	u64 base_val = CPHYSADDR(&page_descr[cpu]) | V_DM_DSCR_BASE_RINGSZ(1);
+	int i;
 
-	bus_writeq(base_val,
-		   (void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
-	bus_writeq(base_val | M_DM_DSCR_BASE_RESET,
-		   (void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
-	bus_writeq(base_val | M_DM_DSCR_BASE_ENABL,
-		   (void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
+	for (i = 0; i < DM_NUM_CHANNELS; i++) {
+		const u64 base_val = CPHYSADDR(&page_descr[i]) |
+				     V_DM_DSCR_BASE_RINGSZ(1);
+		volatile void *base_reg =
+			IOADDR(A_DM_REGISTER(i, R_DM_DSCR_BASE));
+
+		__raw_writeq(base_val, base_reg);
+		__raw_writeq(base_val | M_DM_DSCR_BASE_RESET, base_reg);
+		__raw_writeq(base_val | M_DM_DSCR_BASE_ENABL, base_reg);
+	}
 }
 
 void clear_page(void *page)
 {
-	int cpu = smp_processor_id();
+	u64 to_phys = CPHYSADDR(page);
+	unsigned int cpu = smp_processor_id();
 
-	/* if the page is above Kseg0, use old way */
+	/* if the page is not in KSEG0, use old way */
 	if ((long)KSEGX(page) != (long)CKSEG0)
 		return clear_page_cpu(page);
 
-	page_descr[cpu].dscr_a = CPHYSADDR(page) | M_DM_DSCRA_ZERO_MEM | M_DM_DSCRA_L2C_DEST | M_DM_DSCRA_INTERRUPT;
+	page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_ZERO_MEM |
+				 M_DM_DSCRA_L2C_DEST | M_DM_DSCRA_INTERRUPT;
 	page_descr[cpu].dscr_b = V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE);
-	bus_writeq(1, (void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
+	__raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
 
 	/*
 	 * Don't really want to do it this way, but there's no
 	 * reliable way to delay completion detection.
 	 */
-	while (!(bus_readq((void *)(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)) &
-			   M_DM_DSCR_BASE_INTERRUPT))))
+	while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)))
+		 & M_DM_DSCR_BASE_INTERRUPT))
 		;
-	bus_readq((void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
+	__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
 }
 
 void copy_page(void *to, void *from)
 {
-	unsigned long from_phys = CPHYSADDR(from);
-	unsigned long to_phys = CPHYSADDR(to);
-	int cpu = smp_processor_id();
+	u64 from_phys = CPHYSADDR(from);
+	u64 to_phys = CPHYSADDR(to);
+	unsigned int cpu = smp_processor_id();
 
-	/* if either page is above Kseg0, use old way */
+	/* if any page is not in KSEG0, use old way */
 	if ((long)KSEGX(to) != (long)CKSEG0
 	    || (long)KSEGX(from) != (long)CKSEG0)
 		return copy_page_cpu(to, from);
 
-	page_descr[cpu].dscr_a = CPHYSADDR(to_phys) | M_DM_DSCRA_L2C_DEST | M_DM_DSCRA_INTERRUPT;
-	page_descr[cpu].dscr_b = CPHYSADDR(from_phys) | V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE);
-	bus_writeq(1, (void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
+	page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_L2C_DEST |
+				 M_DM_DSCRA_INTERRUPT;
+	page_descr[cpu].dscr_b = from_phys | V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE);
+	__raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
 
 	/*
 	 * Don't really want to do it this way, but there's no
 	 * reliable way to delay completion detection.
 	 */
-	while (!(bus_readq((void *)(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)) &
-				    M_DM_DSCR_BASE_INTERRUPT))))
+	while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)))
+		 & M_DM_DSCR_BASE_INTERRUPT))
 		;
-	bus_readq((void *)IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
+	__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
 }
 
 #else /* !CONFIG_SIBYTE_DMA_PAGEOPS */
diff --git a/arch/mips/mm/pgtable-32.c b/arch/mips/mm/pgtable-32.c
index 4f07f81e850..4a3c4919e31 100644
--- a/arch/mips/mm/pgtable-32.c
+++ b/arch/mips/mm/pgtable-32.c
@@ -10,6 +10,7 @@
 #include <linux/mm.h>
 #include <linux/bootmem.h>
 #include <linux/highmem.h>
+#include <asm/fixmap.h>
 #include <asm/pgtable.h>
 
 void pgd_init(unsigned long page)
@@ -29,42 +30,12 @@ void pgd_init(unsigned long page)
 	}
 }
 
-#ifdef CONFIG_HIGHMEM
-static void __init fixrange_init (unsigned long start, unsigned long end,
-	pgd_t *pgd_base)
-{
-	pgd_t *pgd;
-	pmd_t *pmd;
-	pte_t *pte;
-	int i, j;
-	unsigned long vaddr;
-
-	vaddr = start;
-	i = __pgd_offset(vaddr);
-	j = __pmd_offset(vaddr);
-	pgd = pgd_base + i;
-
-	for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) {
-		pmd = (pmd_t *)pgd;
-		for (; (j < PTRS_PER_PMD) && (vaddr != end); pmd++, j++) {
-			if (pmd_none(*pmd)) {
-				pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
-				set_pmd(pmd, __pmd((unsigned long)pte));
-				if (pte != pte_offset_kernel(pmd, 0))
-					BUG();
-			}
-			vaddr += PMD_SIZE;
-		}
-		j = 0;
-	}
-}
-#endif
-
 void __init pagetable_init(void)
 {
 #ifdef CONFIG_HIGHMEM
 	unsigned long vaddr;
 	pgd_t *pgd, *pgd_base;
+	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
 #endif
@@ -90,7 +61,8 @@ void __init pagetable_init(void)
 	fixrange_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
 
 	pgd = swapper_pg_dir + __pgd_offset(vaddr);
-	pmd = pmd_offset(pgd, vaddr);
+	pud = pud_offset(pgd, vaddr);
+	pmd = pmd_offset(pud, vaddr);
 	pte = pte_offset_kernel(pmd, vaddr);
 	pkmap_page_table = pte;
 #endif
diff --git a/arch/mips/mm/sc-rm7k.c b/arch/mips/mm/sc-rm7k.c
index 4e92f931aab..9e8ff8badb1 100644
--- a/arch/mips/mm/sc-rm7k.c
+++ b/arch/mips/mm/sc-rm7k.c
@@ -15,6 +15,7 @@
 #include <asm/cacheops.h>
 #include <asm/mipsregs.h>
 #include <asm/processor.h>
+#include <asm/cacheflush.h> /* for run_uncached() */
 
 /* Primary cache parameters. */
 #define sc_lsize	32
@@ -96,25 +97,13 @@ static void rm7k_sc_inv(unsigned long addr, unsigned long size)
 }
 
 /*
- * This function is executed in the uncached segment CKSEG1.
- * It must not touch the stack, because the stack pointer still points
- * into CKSEG0.
- *
- * Three options:
- *	- Write it in assembly and guarantee that we don't use the stack.
- *	- Disable caching for CKSEG0 before calling it.
- *	- Pray that GCC doesn't randomly start using the stack.
- *
- * This being Linux, we obviously take the least sane of those options -
- * following DaveM's lead in c-r4k.c
- *
- * It seems we get our kicks from relying on unguaranteed behaviour in GCC
+ * This function is executed in uncached address space.
  */
 static __init void __rm7k_sc_enable(void)
 {
 	int i;
 
-	set_c0_config(1 << 3);				/* CONF_SE */
+	set_c0_config(RM7K_CONF_SE);
 
 	write_c0_taglo(0);
 	write_c0_taghi(0);
@@ -127,24 +116,22 @@ static __init void __rm7k_sc_enable(void)
 		      ".set mips0\n\t"
 		      ".set reorder"
 		      :
-		      : "r" (KSEG0ADDR(i)), "i" (Index_Store_Tag_SD));
+		      : "r" (CKSEG0ADDR(i)), "i" (Index_Store_Tag_SD));
 	}
 }
 
 static __init void rm7k_sc_enable(void)
 {
-	void (*func)(void) = (void *) KSEG1ADDR(&__rm7k_sc_enable);
-
-	if (read_c0_config() & 0x08)			/* CONF_SE */
+	if (read_c0_config() & RM7K_CONF_SE)
 		return;
 
-	printk(KERN_INFO "Enabling secondary cache...");
-	func();
+	printk(KERN_INFO "Enabling secondary cache...\n");
+	run_uncached(__rm7k_sc_enable);
 }
 
 static void rm7k_sc_disable(void)
 {
-	clear_c0_config(1<<3);				/* CONF_SE */
+	clear_c0_config(RM7K_CONF_SE);
 }
 
 struct bcache_ops rm7k_sc_ops = {
@@ -158,19 +145,19 @@ void __init rm7k_sc_init(void)
 {
 	unsigned int config = read_c0_config();
 
-	if ((config >> 31) & 1)		/* Bit 31 set -> no S-Cache */
+	if ((config & RM7K_CONF_SC))
 		return;
 
 	printk(KERN_INFO "Secondary cache size %dK, linesize %d bytes.\n",
 	       (scache_size >> 10), sc_lsize);
 
-	if (!((config >> 3) & 1))	/* CONF_SE */
+	if (!(config & RM7K_CONF_SE))
 		rm7k_sc_enable();
 
 	/*
 	 * While we're at it let's deal with the tertiary cache.
 	 */
-	if (!((config >> 17) & 1)) {
+	if (!(config & RM7K_CONF_TC)) {
 
 		/*
 		 * We can't enable the L3 cache yet. There may be board-specific
@@ -183,9 +170,9 @@ void __init rm7k_sc_init(void)
 		 * to probe it.
 		 */
 		printk(KERN_INFO "Tertiary cache present, %s enabled\n",
-		       config&(1<<12) ? "already" : "not (yet)");
+		       (config & RM7K_CONF_TE) ? "already" : "not (yet)");
 
-		if ((config >> 12) & 1)
+		if ((config & RM7K_CONF_TE))
 			rm7k_tcache_enabled = 1;
 	}
 
diff --git a/arch/mips/mm/tlb-andes.c b/arch/mips/mm/tlb-andes.c
index 167e08e9661..3f422a849c4 100644
--- a/arch/mips/mm/tlb-andes.c
+++ b/arch/mips/mm/tlb-andes.c
@@ -195,6 +195,7 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 {
 	unsigned long flags;
 	pgd_t *pgdp;
+	pud_t *pudp;
 	pmd_t *pmdp;
 	pte_t *ptep;
 	int idx, pid;
@@ -220,7 +221,8 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 	write_c0_entryhi(address | (pid));
 	pgdp = pgd_offset(vma->vm_mm, address);
 	tlb_probe();
-	pmdp = pmd_offset(pgdp, address);
+	pudp = pud_offset(pgdp, address);
+	pmdp = pmd_offset(pudp, address);
 	idx = read_c0_index();
 	ptep = pte_offset_map(pmdp, address);
 	write_c0_entrylo0(pte_val(*ptep++) >> 6);
diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c
index 59d38bc05b6..8297970f0bb 100644
--- a/arch/mips/mm/tlb-r4k.c
+++ b/arch/mips/mm/tlb-r4k.c
@@ -21,6 +21,12 @@
 
 extern void build_tlb_refill_handler(void);
 
+/*
+ * Make sure all entries differ.  If they're not different
+ * MIPS32 will take revenge ...
+ */
+#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
+
 /* CP0 hazard avoidance. */
 #define BARRIER __asm__ __volatile__(".set noreorder\n\t" \
 				     "nop; nop; nop; nop; nop; nop;\n\t" \
@@ -42,11 +48,8 @@ void local_flush_tlb_all(void)
 
 	/* Blast 'em all away. */
 	while (entry < current_cpu_data.tlbsize) {
-		/*
-		 * Make sure all entries differ.  If they're not different
-		 * MIPS32 will take revenge ...
-		 */
-		write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1)));
+		/* Make sure all entries differ. */
+		write_c0_entryhi(UNIQUE_ENTRYHI(entry));
 		write_c0_index(entry);
 		mtc0_tlbw_hazard();
 		tlb_write_indexed();
@@ -57,12 +60,21 @@ void local_flush_tlb_all(void)
 	local_irq_restore(flags);
 }
 
+/* All entries common to a mm share an asid.  To effectively flush
+   these entries, we just bump the asid. */
 void local_flush_tlb_mm(struct mm_struct *mm)
 {
-	int cpu = smp_processor_id();
+	int cpu;
+
+	preempt_disable();
 
-	if (cpu_context(cpu, mm) != 0)
-		drop_mmu_context(mm,cpu);
+	cpu = smp_processor_id();
+
+	if (cpu_context(cpu, mm) != 0) {
+		drop_mmu_context(mm, cpu);
+	}
+
+	preempt_enable();
 }
 
 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
@@ -75,9 +87,9 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 		unsigned long flags;
 		int size;
 
-		local_irq_save(flags);
 		size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 		size = (size + 1) >> 1;
+		local_irq_save(flags);
 		if (size <= current_cpu_data.tlbsize/2) {
 			int oldpid = read_c0_entryhi();
 			int newpid = cpu_asid(cpu, mm);
@@ -99,8 +111,7 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 				if (idx < 0)
 					continue;
 				/* Make sure all entries differ. */
-				write_c0_entryhi(CKSEG0 +
-				                 (idx << (PAGE_SHIFT + 1)));
+				write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 				mtc0_tlbw_hazard();
 				tlb_write_indexed();
 			}
@@ -118,9 +129,9 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 	unsigned long flags;
 	int size;
 
-	local_irq_save(flags);
 	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 	size = (size + 1) >> 1;
+	local_irq_save(flags);
 	if (size <= current_cpu_data.tlbsize / 2) {
 		int pid = read_c0_entryhi();
 
@@ -142,7 +153,7 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 			if (idx < 0)
 				continue;
 			/* Make sure all entries differ. */
-			write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
+			write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 			mtc0_tlbw_hazard();
 			tlb_write_indexed();
 		}
@@ -176,7 +187,7 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 		if (idx < 0)
 			goto finish;
 		/* Make sure all entries differ. */
-		write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
+		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 		mtc0_tlbw_hazard();
 		tlb_write_indexed();
 		tlbw_use_hazard();
@@ -197,8 +208,8 @@ void local_flush_tlb_one(unsigned long page)
 	int oldpid, idx;
 
 	local_irq_save(flags);
-	page &= (PAGE_MASK << 1);
 	oldpid = read_c0_entryhi();
+	page &= (PAGE_MASK << 1);
 	write_c0_entryhi(page);
 	mtc0_tlbw_hazard();
 	tlb_probe();
@@ -208,7 +219,7 @@ void local_flush_tlb_one(unsigned long page)
 	write_c0_entrylo1(0);
 	if (idx >= 0) {
 		/* Make sure all entries differ. */
-		write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
+		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 		mtc0_tlbw_hazard();
 		tlb_write_indexed();
 		tlbw_use_hazard();
@@ -227,6 +238,7 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 {
 	unsigned long flags;
 	pgd_t *pgdp;
+	pud_t *pudp;
 	pmd_t *pmdp;
 	pte_t *ptep;
 	int idx, pid;
@@ -237,35 +249,34 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 	if (current->active_mm != vma->vm_mm)
 		return;
 
-	pid = read_c0_entryhi() & ASID_MASK;
-
 	local_irq_save(flags);
+
+	pid = read_c0_entryhi() & ASID_MASK;
 	address &= (PAGE_MASK << 1);
 	write_c0_entryhi(address | pid);
 	pgdp = pgd_offset(vma->vm_mm, address);
 	mtc0_tlbw_hazard();
 	tlb_probe();
 	BARRIER;
-	pmdp = pmd_offset(pgdp, address);
+	pudp = pud_offset(pgdp, address);
+	pmdp = pmd_offset(pudp, address);
 	idx = read_c0_index();
 	ptep = pte_offset_map(pmdp, address);
 
- #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
- 	write_c0_entrylo0(ptep->pte_high);
- 	ptep++;
- 	write_c0_entrylo1(ptep->pte_high);
+#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32_R1)
+	write_c0_entrylo0(ptep->pte_high);
+	ptep++;
+	write_c0_entrylo1(ptep->pte_high);
 #else
-  	write_c0_entrylo0(pte_val(*ptep++) >> 6);
-  	write_c0_entrylo1(pte_val(*ptep) >> 6);
+	write_c0_entrylo0(pte_val(*ptep++) >> 6);
+	write_c0_entrylo1(pte_val(*ptep) >> 6);
 #endif
-	write_c0_entryhi(address | pid);
 	mtc0_tlbw_hazard();
 	if (idx < 0)
 		tlb_write_random();
 	else
 		tlb_write_indexed();
 	tlbw_use_hazard();
-	write_c0_entryhi(pid);
 	local_irq_restore(flags);
 }
 
@@ -357,7 +368,8 @@ __init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1,
 	old_pagemask = read_c0_pagemask();
 	wired = read_c0_wired();
 	if (--temp_tlb_entry < wired) {
-		printk(KERN_WARNING "No TLB space left for add_temporary_entry\n");
+		printk(KERN_WARNING
+		       "No TLB space left for add_temporary_entry\n");
 		ret = -ENOSPC;
 		goto out;
 	}
@@ -388,7 +400,7 @@ static void __init probe_tlb(unsigned long config)
 	 * is not supported, we assume R4k style.  Cpu probing already figured
 	 * out the number of tlb entries.
 	 */
-	if ((c->processor_id  & 0xff0000) == PRID_COMP_LEGACY)
+	if ((c->processor_id & 0xff0000) == PRID_COMP_LEGACY)
 		return;
 
 	reg = read_c0_config1();
diff --git a/arch/mips/mm/tlb-sb1.c b/arch/mips/mm/tlb-sb1.c
deleted file mode 100644
index 6256cafcf3a..00000000000
--- a/arch/mips/mm/tlb-sb1.c
+++ /dev/null
@@ -1,376 +0,0 @@
-/*
- * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
- * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org)
- * Copyright (C) 2000, 2001, 2002, 2003 Broadcom Corporation
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
- */
-#include <linux/init.h>
-#include <asm/mmu_context.h>
-#include <asm/bootinfo.h>
-#include <asm/cpu.h>
-
-extern void build_tlb_refill_handler(void);
-
-#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
-
-/* Dump the current entry* and pagemask registers */
-static inline void dump_cur_tlb_regs(void)
-{
-	unsigned int entryhihi, entryhilo, entrylo0hi, entrylo0lo, entrylo1hi;
-	unsigned int entrylo1lo, pagemask;
-
-	__asm__ __volatile__ (
-		".set push             \n"
-		".set noreorder        \n"
-		".set mips64           \n"
-		".set noat             \n"
-		"     tlbr             \n"
-		"     dmfc0  $1, $10   \n"
-		"     dsrl32 %0, $1, 0 \n"
-		"     sll    %1, $1, 0 \n"
-		"     dmfc0  $1, $2    \n"
-		"     dsrl32 %2, $1, 0 \n"
-		"     sll    %3, $1, 0 \n"
-		"     dmfc0  $1, $3    \n"
-		"     dsrl32 %4, $1, 0 \n"
-		"     sll    %5, $1, 0 \n"
-		"     mfc0   %6, $5    \n"
-		".set pop              \n"
-		: "=r" (entryhihi), "=r" (entryhilo),
-		  "=r" (entrylo0hi), "=r" (entrylo0lo),
-		  "=r" (entrylo1hi), "=r" (entrylo1lo),
-		  "=r" (pagemask));
-
-	printk("%08X%08X %08X%08X %08X%08X %08X",
-	       entryhihi, entryhilo,
-	       entrylo0hi, entrylo0lo,
-	       entrylo1hi, entrylo1lo,
-	       pagemask);
-}
-
-void sb1_dump_tlb(void)
-{
-	unsigned long old_ctx;
-	unsigned long flags;
-	int entry;
-	local_irq_save(flags);
-	old_ctx = read_c0_entryhi();
-	printk("Current TLB registers state:\n"
-	       "      EntryHi       EntryLo0          EntryLo1     PageMask  Index\n"
-	       "--------------------------------------------------------------------\n");
-	dump_cur_tlb_regs();
-	printk(" %08X\n", read_c0_index());
-	printk("\n\nFull TLB Dump:\n"
-	       "Idx      EntryHi       EntryLo0          EntryLo1     PageMask\n"
-	       "--------------------------------------------------------------\n");
-	for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
-		write_c0_index(entry);
-		printk("\n%02i ", entry);
-		dump_cur_tlb_regs();
-	}
-	printk("\n");
-	write_c0_entryhi(old_ctx);
-	local_irq_restore(flags);
-}
-
-void local_flush_tlb_all(void)
-{
-	unsigned long flags;
-	unsigned long old_ctx;
-	int entry;
-
-	local_irq_save(flags);
-	/* Save old context and create impossible VPN2 value */
-	old_ctx = read_c0_entryhi() & ASID_MASK;
-	write_c0_entrylo0(0);
-	write_c0_entrylo1(0);
-
-	entry = read_c0_wired();
-	while (entry < current_cpu_data.tlbsize) {
-		write_c0_entryhi(UNIQUE_ENTRYHI(entry));
-		write_c0_index(entry);
-		tlb_write_indexed();
-		entry++;
-	}
-	write_c0_entryhi(old_ctx);
-	local_irq_restore(flags);
-}
-
-
-/*
- * Use a bogus region of memory (starting at 0) to sanitize the TLB's.
- * Use increments of the maximum page size (16MB), and check for duplicate
- * entries before doing a given write.  Then, when we're safe from collisions
- * with the firmware, go back and give all the entries invalid addresses with
- * the normal flush routine.  Wired entries will be killed as well!
- */
-static void __init sb1_sanitize_tlb(void)
-{
-	int entry;
-	long addr = 0;
-
-	long inc = 1<<24;  /* 16MB */
-	/* Save old context and create impossible VPN2 value */
-	write_c0_entrylo0(0);
-	write_c0_entrylo1(0);
-	for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
-		do {
-			addr += inc;
-			write_c0_entryhi(addr);
-			tlb_probe();
-		} while ((int)(read_c0_index()) >= 0);
-		write_c0_index(entry);
-		tlb_write_indexed();
-	}
-	/* Now that we know we're safe from collisions, we can safely flush
-	   the TLB with the "normal" routine. */
-	local_flush_tlb_all();
-}
-
-void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
-	unsigned long end)
-{
-	struct mm_struct *mm = vma->vm_mm;
-	unsigned long flags;
-	int cpu;
-
-	local_irq_save(flags);
-	cpu = smp_processor_id();
-	if (cpu_context(cpu, mm) != 0) {
-		int size;
-		size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
-		size = (size + 1) >> 1;
-		if (size <= (current_cpu_data.tlbsize/2)) {
-			int oldpid = read_c0_entryhi() & ASID_MASK;
-			int newpid = cpu_asid(cpu, mm);
-
-			start &= (PAGE_MASK << 1);
-			end += ((PAGE_SIZE << 1) - 1);
-			end &= (PAGE_MASK << 1);
-			while (start < end) {
-				int idx;
-
-				write_c0_entryhi(start | newpid);
-				start += (PAGE_SIZE << 1);
-				tlb_probe();
-				idx = read_c0_index();
-				write_c0_entrylo0(0);
-				write_c0_entrylo1(0);
-				write_c0_entryhi(UNIQUE_ENTRYHI(idx));
-				if (idx < 0)
-					continue;
-				tlb_write_indexed();
-			}
-			write_c0_entryhi(oldpid);
-		} else {
-			drop_mmu_context(mm, cpu);
-		}
-	}
-	local_irq_restore(flags);
-}
-
-void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
-{
-	unsigned long flags;
-	int size;
-
-	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
-	size = (size + 1) >> 1;
-
-	local_irq_save(flags);
-	if (size <= (current_cpu_data.tlbsize/2)) {
-		int pid = read_c0_entryhi();
-
-		start &= (PAGE_MASK << 1);
-		end += ((PAGE_SIZE << 1) - 1);
-		end &= (PAGE_MASK << 1);
-
-		while (start < end) {
-			int idx;
-
-			write_c0_entryhi(start);
-			start += (PAGE_SIZE << 1);
-			tlb_probe();
-			idx = read_c0_index();
-			write_c0_entrylo0(0);
-			write_c0_entrylo1(0);
-			write_c0_entryhi(UNIQUE_ENTRYHI(idx));
-			if (idx < 0)
-				continue;
-			tlb_write_indexed();
-		}
-		write_c0_entryhi(pid);
-	} else {
-		local_flush_tlb_all();
-	}
-	local_irq_restore(flags);
-}
-
-void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
-{
-	unsigned long flags;
-	int cpu = smp_processor_id();
-
-	local_irq_save(flags);
-	if (cpu_context(cpu, vma->vm_mm) != 0) {
-		int oldpid, newpid, idx;
-		newpid = cpu_asid(cpu, vma->vm_mm);
-		page &= (PAGE_MASK << 1);
-		oldpid = read_c0_entryhi() & ASID_MASK;
-		write_c0_entryhi(page | newpid);
-		tlb_probe();
-		idx = read_c0_index();
-		write_c0_entrylo0(0);
-		write_c0_entrylo1(0);
-		if (idx < 0)
-			goto finish;
-		/* Make sure all entries differ. */
-		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
-		tlb_write_indexed();
-	finish:
-		write_c0_entryhi(oldpid);
-	}
-	local_irq_restore(flags);
-}
-
-/*
- * Remove one kernel space TLB entry.  This entry is assumed to be marked
- * global so we don't do the ASID thing.
- */
-void local_flush_tlb_one(unsigned long page)
-{
-	unsigned long flags;
-	int oldpid, idx;
-
-	page &= (PAGE_MASK << 1);
-	oldpid = read_c0_entryhi() & ASID_MASK;
-
-	local_irq_save(flags);
-	write_c0_entryhi(page);
-	tlb_probe();
-	idx = read_c0_index();
-	if (idx >= 0) {
-		/* Make sure all entries differ. */
-		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
-		write_c0_entrylo0(0);
-		write_c0_entrylo1(0);
-		tlb_write_indexed();
-	}
-
-	write_c0_entryhi(oldpid);
-	local_irq_restore(flags);
-}
-
-/* All entries common to a mm share an asid.  To effectively flush
-   these entries, we just bump the asid. */
-void local_flush_tlb_mm(struct mm_struct *mm)
-{
-	int cpu;
-
-	preempt_disable();
-
-	cpu = smp_processor_id();
-
-	if (cpu_context(cpu, mm) != 0) {
-		drop_mmu_context(mm, cpu);
-	}
-
-	preempt_enable();
-}
-
-/* Stolen from mips32 routines */
-
-void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
-{
-	unsigned long flags;
-	pgd_t *pgdp;
-	pmd_t *pmdp;
-	pte_t *ptep;
-	int idx, pid;
-
-	/*
-	 * Handle debugger faulting in for debugee.
-	 */
-	if (current->active_mm != vma->vm_mm)
-		return;
-
-	local_irq_save(flags);
-
-	pid = read_c0_entryhi() & ASID_MASK;
-	address &= (PAGE_MASK << 1);
-	write_c0_entryhi(address | (pid));
-	pgdp = pgd_offset(vma->vm_mm, address);
-	tlb_probe();
-	pmdp = pmd_offset(pgdp, address);
-	idx = read_c0_index();
-	ptep = pte_offset_map(pmdp, address);
-	write_c0_entrylo0(pte_val(*ptep++) >> 6);
-	write_c0_entrylo1(pte_val(*ptep) >> 6);
-	if (idx < 0) {
-		tlb_write_random();
-	} else {
-		tlb_write_indexed();
-	}
-	local_irq_restore(flags);
-}
-
-void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
-	unsigned long entryhi, unsigned long pagemask)
-{
-	unsigned long flags;
-	unsigned long wired;
-	unsigned long old_pagemask;
-	unsigned long old_ctx;
-
-	local_irq_save(flags);
-	old_ctx = read_c0_entryhi() & 0xff;
-	old_pagemask = read_c0_pagemask();
-	wired = read_c0_wired();
-	write_c0_wired(wired + 1);
-	write_c0_index(wired);
-
-	write_c0_pagemask(pagemask);
-	write_c0_entryhi(entryhi);
-	write_c0_entrylo0(entrylo0);
-	write_c0_entrylo1(entrylo1);
-	tlb_write_indexed();
-
-	write_c0_entryhi(old_ctx);
-	write_c0_pagemask(old_pagemask);
-
-	local_flush_tlb_all();
-	local_irq_restore(flags);
-}
-
-/*
- * This is called from loadmmu.c.  We have to set up all the
- * memory management function pointers, as well as initialize
- * the caches and tlbs
- */
-void tlb_init(void)
-{
-	write_c0_pagemask(PM_DEFAULT_MASK);
-	write_c0_wired(0);
-
-	/*
-	 * We don't know what state the firmware left the TLB's in, so this is
-	 * the ultra-conservative way to flush the TLB's and avoid machine
-	 * check exceptions due to duplicate TLB entries
-	 */
-	sb1_sanitize_tlb();
-
-	build_tlb_refill_handler();
-}
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 6569be3983c..0f9485806ba 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -6,6 +6,7 @@
  * Synthesize TLB refill handlers at runtime.
  *
  * Copyright (C) 2004,2005 by Thiemo Seufer
+ * Copyright (C) 2005  Maciej W. Rozycki
  */
 
 #include <stdarg.h>
@@ -91,7 +92,7 @@ enum opcode {
 	insn_addu, insn_addiu, insn_and, insn_andi, insn_beq,
 	insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
 	insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0,
-	insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32,
+	insn_dsll, insn_dsll32, insn_dsra, insn_dsrl,
 	insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld,
 	insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0,
 	insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll,
@@ -134,7 +135,6 @@ static __initdata struct insn insn_table[] = {
 	{ insn_dsll32, M(spec_op,0,0,0,0,dsll32_op), RT | RD | RE },
 	{ insn_dsra, M(spec_op,0,0,0,0,dsra_op), RT | RD | RE },
 	{ insn_dsrl, M(spec_op,0,0,0,0,dsrl_op), RT | RD | RE },
-	{ insn_dsrl32, M(spec_op,0,0,0,0,dsrl32_op), RT | RD | RE },
 	{ insn_dsubu, M(spec_op,0,0,0,0,dsubu_op), RS | RT | RD },
 	{ insn_eret, M(cop0_op,cop_op,0,0,0,eret_op), 0 },
 	{ insn_j, M(j_op,0,0,0,0,0), JIMM },
@@ -366,7 +366,6 @@ I_u2u1u3(_dsll);
 I_u2u1u3(_dsll32);
 I_u2u1u3(_dsra);
 I_u2u1u3(_dsrl);
-I_u2u1u3(_dsrl32);
 I_u3u1u2(_dsubu);
 I_0(_eret);
 I_u1(_j);
@@ -412,7 +411,6 @@ enum label_id {
 	label_nopage_tlbm,
 	label_smp_pgtable_change,
 	label_r3000_write_probe_fail,
-	label_r3000_write_probe_ok
 };
 
 struct label {
@@ -445,7 +443,6 @@ L_LA(_nopage_tlbs)
 L_LA(_nopage_tlbm)
 L_LA(_smp_pgtable_change)
 L_LA(_r3000_write_probe_fail)
-L_LA(_r3000_write_probe_ok)
 
 /* convenience macros for instructions */
 #ifdef CONFIG_64BIT
@@ -490,7 +487,7 @@ L_LA(_r3000_write_probe_ok)
 static __init int __attribute__((unused)) in_compat_space_p(long addr)
 {
 	/* Is this address in 32bit compat space? */
-	return (((addr) & 0xffffffff00000000) == 0xffffffff00000000);
+	return (((addr) & 0xffffffff00000000L) == 0xffffffff00000000L);
 }
 
 static __init int __attribute__((unused)) rel_highest(long val)
@@ -734,7 +731,7 @@ static void __init build_r3000_tlb_refill_handler(void)
 	if (p > tlb_handler + 32)
 		panic("TLB refill handler space exceeded");
 
-	printk("Synthesized TLB handler (%u instructions).\n",
+	printk("Synthesized TLB refill handler (%u instructions).\n",
 	       (unsigned int)(p - tlb_handler));
 #ifdef DEBUG_TLB
 	{
@@ -746,7 +743,6 @@ static void __init build_r3000_tlb_refill_handler(void)
 #endif
 
 	memcpy((void *)CAC_BASE, tlb_handler, 0x80);
-	flush_icache_range(CAC_BASE, CAC_BASE + 0x80);
 }
 
 /*
@@ -783,6 +779,8 @@ static __initdata u32 final_handler[64];
 static __init void __attribute__((unused)) build_tlb_probe_entry(u32 **p)
 {
 	switch (current_cpu_data.cputype) {
+	/* Found by experiment: R4600 v2.0 needs this, too.  */
+	case CPU_R4600:
 	case CPU_R5000:
 	case CPU_R5000A:
 	case CPU_NEVADA:
@@ -834,12 +832,20 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l,
 	case CPU_R4700:
 	case CPU_R5000:
 	case CPU_R5000A:
+		i_nop(p);
+		tlbw(p);
+		i_nop(p);
+		break;
+
+	case CPU_R4300:
 	case CPU_5KC:
 	case CPU_TX49XX:
 	case CPU_AU1000:
 	case CPU_AU1100:
 	case CPU_AU1500:
 	case CPU_AU1550:
+	case CPU_AU1200:
+	case CPU_PR4450:
 		i_nop(p);
 		tlbw(p);
 		break;
@@ -848,6 +854,7 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l,
 	case CPU_R12000:
 	case CPU_4KC:
 	case CPU_SB1:
+	case CPU_SB1A:
 	case CPU_4KSC:
 	case CPU_20KC:
 	case CPU_25KF:
@@ -875,6 +882,7 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l,
 
 	case CPU_4KEC:
 	case CPU_24K:
+	case CPU_34K:
 		i_ehb(p);
 		tlbw(p);
 		break;
@@ -911,6 +919,7 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l,
 
 	case CPU_VR4131:
 	case CPU_VR4133:
+	case CPU_R5432:
 		i_nop(p);
 		i_nop(p);
 		tlbw(p);
@@ -942,34 +951,29 @@ build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
 	/* No i_nop needed here, since the next insn doesn't touch TMP. */
 
 #ifdef CONFIG_SMP
+# ifdef CONFIG_BUILD_ELF64
 	/*
-	 * 64 bit SMP has the lower part of &pgd_current[smp_processor_id()]
+	 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
 	 * stored in CONTEXT.
 	 */
-	if (in_compat_space_p(pgdc)) {
-		i_dmfc0(p, ptr, C0_CONTEXT);
-		i_dsra(p, ptr, ptr, 23);
-		i_ld(p, ptr, 0, ptr);
-	} else {
-#ifdef CONFIG_BUILD_ELF64
-		i_dmfc0(p, ptr, C0_CONTEXT);
-		i_dsrl(p, ptr, ptr, 23);
-		i_dsll(p, ptr, ptr, 3);
-		i_LA_mostly(p, tmp, pgdc);
-		i_daddu(p, ptr, ptr, tmp);
-		i_dmfc0(p, tmp, C0_BADVADDR);
-		i_ld(p, ptr, rel_lo(pgdc), ptr);
-#else
-		i_dmfc0(p, ptr, C0_CONTEXT);
-		i_lui(p, tmp, rel_highest(pgdc));
-		i_dsll(p, ptr, ptr, 9);
-		i_daddiu(p, tmp, tmp, rel_higher(pgdc));
-		i_dsrl32(p, ptr, ptr, 0);
-		i_and(p, ptr, ptr, tmp);
-		i_dmfc0(p, tmp, C0_BADVADDR);
-		i_ld(p, ptr, 0, ptr);
-#endif
-	}
+	i_dmfc0(p, ptr, C0_CONTEXT);
+	i_dsrl(p, ptr, ptr, 23);
+	i_LA_mostly(p, tmp, pgdc);
+	i_daddu(p, ptr, ptr, tmp);
+	i_dmfc0(p, tmp, C0_BADVADDR);
+	i_ld(p, ptr, rel_lo(pgdc), ptr);
+# else
+	/*
+	 * 64 bit SMP running in compat space has the lower part of
+	 * &pgd_current[smp_processor_id()] stored in CONTEXT.
+	 */
+	if (!in_compat_space_p(pgdc))
+		panic("Invalid page directory address!");
+
+	i_dmfc0(p, ptr, C0_CONTEXT);
+	i_dsra(p, ptr, ptr, 23);
+	i_ld(p, ptr, 0, ptr);
+# endif
 #else
 	i_LA_mostly(p, ptr, pgdc);
 	i_ld(p, ptr, rel_lo(pgdc), ptr);
@@ -1026,7 +1030,6 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
 	i_mfc0(p, ptr, C0_CONTEXT);
 	i_LA_mostly(p, tmp, pgdc);
 	i_srl(p, ptr, ptr, 23);
-	i_sll(p, ptr, ptr, 2);
 	i_addu(p, ptr, tmp, ptr);
 #else
 	i_LA_mostly(p, ptr, pgdc);
@@ -1245,13 +1248,19 @@ static void __init build_r4000_tlb_refill_handler(void)
 	{
 		int i;
 
-		for (i = 0; i < 64; i++)
-			printk("%08x\n", final_handler[i]);
+		f = final_handler;
+#ifdef CONFIG_64BIT
+		if (final_len > 32)
+			final_len = 64;
+		else
+			f = final_handler + 32;
+#endif /* CONFIG_64BIT */
+		for (i = 0; i < final_len; i++)
+			printk("%08x\n", f[i]);
 	}
 #endif
 
 	memcpy((void *)CAC_BASE, final_handler, 0x100);
-	flush_icache_range(CAC_BASE, CAC_BASE + 0x100);
 }
 
 /*
@@ -1277,37 +1286,41 @@ u32 __tlb_handler_align handle_tlbs[FASTPATH_SIZE];
 u32 __tlb_handler_align handle_tlbm[FASTPATH_SIZE];
 
 static void __init
-iPTE_LW(u32 **p, struct label **l, unsigned int pte, int offset,
-	unsigned int ptr)
+iPTE_LW(u32 **p, struct label **l, unsigned int pte, unsigned int ptr)
 {
 #ifdef CONFIG_SMP
 # ifdef CONFIG_64BIT_PHYS_ADDR
 	if (cpu_has_64bits)
-		i_lld(p, pte, offset, ptr);
+		i_lld(p, pte, 0, ptr);
 	else
 # endif
-		i_LL(p, pte, offset, ptr);
+		i_LL(p, pte, 0, ptr);
 #else
 # ifdef CONFIG_64BIT_PHYS_ADDR
 	if (cpu_has_64bits)
-		i_ld(p, pte, offset, ptr);
+		i_ld(p, pte, 0, ptr);
 	else
 # endif
-		i_LW(p, pte, offset, ptr);
+		i_LW(p, pte, 0, ptr);
 #endif
 }
 
 static void __init
-iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, int offset,
-	unsigned int ptr)
+iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, unsigned int ptr,
+	unsigned int mode)
 {
+#ifdef CONFIG_64BIT_PHYS_ADDR
+	unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
+#endif
+
+	i_ori(p, pte, pte, mode);
 #ifdef CONFIG_SMP
 # ifdef CONFIG_64BIT_PHYS_ADDR
 	if (cpu_has_64bits)
-		i_scd(p, pte, offset, ptr);
+		i_scd(p, pte, 0, ptr);
 	else
 # endif
-		i_SC(p, pte, offset, ptr);
+		i_SC(p, pte, 0, ptr);
 
 	if (r10000_llsc_war())
 		il_beqzl(p, r, pte, label_smp_pgtable_change);
@@ -1318,7 +1331,7 @@ iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, int offset,
 	if (!cpu_has_64bits) {
 		/* no i_nop needed */
 		i_ll(p, pte, sizeof(pte_t) / 2, ptr);
-		i_ori(p, pte, pte, _PAGE_VALID);
+		i_ori(p, pte, pte, hwmode);
 		i_sc(p, pte, sizeof(pte_t) / 2, ptr);
 		il_beqz(p, r, pte, label_smp_pgtable_change);
 		/* no i_nop needed */
@@ -1331,15 +1344,15 @@ iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, int offset,
 #else
 # ifdef CONFIG_64BIT_PHYS_ADDR
 	if (cpu_has_64bits)
-		i_sd(p, pte, offset, ptr);
+		i_sd(p, pte, 0, ptr);
 	else
 # endif
-		i_SW(p, pte, offset, ptr);
+		i_SW(p, pte, 0, ptr);
 
 # ifdef CONFIG_64BIT_PHYS_ADDR
 	if (!cpu_has_64bits) {
 		i_lw(p, pte, sizeof(pte_t) / 2, ptr);
-		i_ori(p, pte, pte, _PAGE_VALID);
+		i_ori(p, pte, pte, hwmode);
 		i_sw(p, pte, sizeof(pte_t) / 2, ptr);
 		i_lw(p, pte, 0, ptr);
 	}
@@ -1359,7 +1372,7 @@ build_pte_present(u32 **p, struct label **l, struct reloc **r,
 	i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
 	i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
 	il_bnez(p, r, pte, lid);
-	iPTE_LW(p, l, pte, 0, ptr);
+	iPTE_LW(p, l, pte, ptr);
 }
 
 /* Make PTE valid, store result in PTR. */
@@ -1367,8 +1380,9 @@ static void __init
 build_make_valid(u32 **p, struct reloc **r, unsigned int pte,
 		 unsigned int ptr)
 {
-	i_ori(p, pte, pte, _PAGE_VALID | _PAGE_ACCESSED);
-	iPTE_SW(p, r, pte, 0, ptr);
+	unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
+
+	iPTE_SW(p, r, pte, ptr, mode);
 }
 
 /*
@@ -1382,7 +1396,7 @@ build_pte_writable(u32 **p, struct label **l, struct reloc **r,
 	i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
 	i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
 	il_bnez(p, r, pte, lid);
-	iPTE_LW(p, l, pte, 0, ptr);
+	iPTE_LW(p, l, pte, ptr);
 }
 
 /* Make PTE writable, update software status bits as well, then store
@@ -1392,9 +1406,10 @@ static void __init
 build_make_write(u32 **p, struct reloc **r, unsigned int pte,
 		 unsigned int ptr)
 {
-	i_ori(p, pte, pte,
-	      _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
-	iPTE_SW(p, r, pte, 0, ptr);
+	unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
+			     | _PAGE_DIRTY);
+
+	iPTE_SW(p, r, pte, ptr, mode);
 }
 
 /*
@@ -1407,41 +1422,48 @@ build_pte_modifiable(u32 **p, struct label **l, struct reloc **r,
 {
 	i_andi(p, pte, pte, _PAGE_WRITE);
 	il_beqz(p, r, pte, lid);
-	iPTE_LW(p, l, pte, 0, ptr);
+	iPTE_LW(p, l, pte, ptr);
 }
 
 /*
  * R3000 style TLB load/store/modify handlers.
  */
 
-/* This places the pte in the page table at PTR into ENTRYLO0. */
+/*
+ * This places the pte into ENTRYLO0 and writes it with tlbwi.
+ * Then it returns.
+ */
 static void __init
-build_r3000_pte_reload(u32 **p, unsigned int ptr)
+build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
 {
-	i_lw(p, ptr, 0, ptr);
-	i_nop(p); /* load delay */
-	i_mtc0(p, ptr, C0_ENTRYLO0);
-	i_nop(p); /* cp0 delay */
+	i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
+	i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
+	i_tlbwi(p);
+	i_jr(p, tmp);
+	i_rfe(p); /* branch delay */
 }
 
 /*
- * The index register may have the probe fail bit set,
- * because we would trap on access kseg2, i.e. without refill.
+ * This places the pte into ENTRYLO0 and writes it with tlbwi
+ * or tlbwr as appropriate.  This is because the index register
+ * may have the probe fail bit set as a result of a trap on a
+ * kseg2 access, i.e. without refill.  Then it returns.
  */
 static void __init
-build_r3000_tlb_write(u32 **p, struct label **l, struct reloc **r,
-		      unsigned int tmp)
+build_r3000_tlb_reload_write(u32 **p, struct label **l, struct reloc **r,
+			     unsigned int pte, unsigned int tmp)
 {
 	i_mfc0(p, tmp, C0_INDEX);
-	i_nop(p); /* cp0 delay */
-	il_bltz(p, r, tmp, label_r3000_write_probe_fail);
-	i_nop(p); /* branch delay */
-	i_tlbwi(p);
-	il_b(p, r, label_r3000_write_probe_ok);
-	i_nop(p); /* branch delay */
+	i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
+	il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
+	i_mfc0(p, tmp, C0_EPC); /* branch delay */
+	i_tlbwi(p); /* cp0 delay */
+	i_jr(p, tmp);
+	i_rfe(p); /* branch delay */
 	l_r3000_write_probe_fail(l, *p);
-	i_tlbwr(p);
-	l_r3000_write_probe_ok(l, *p);
+	i_tlbwr(p); /* cp0 delay */
+	i_jr(p, tmp);
+	i_rfe(p); /* branch delay */
 }
 
 static void __init
@@ -1461,17 +1483,7 @@ build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
 	i_andi(p, pte, pte, 0xffc); /* load delay */
 	i_addu(p, ptr, ptr, pte);
 	i_lw(p, pte, 0, ptr);
-	i_nop(p); /* load delay */
-	i_tlbp(p);
-}
-
-static void __init
-build_r3000_tlbchange_handler_tail(u32 **p, unsigned int tmp)
-{
-	i_mfc0(p, tmp, C0_EPC);
-	i_nop(p); /* cp0 delay */
-	i_jr(p, tmp);
-	i_rfe(p); /* branch delay */
+	i_tlbp(p); /* load delay */
 }
 
 static void __init build_r3000_tlb_load_handler(void)
@@ -1486,10 +1498,9 @@ static void __init build_r3000_tlb_load_handler(void)
 
 	build_r3000_tlbchange_handler_head(&p, K0, K1);
 	build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
+	i_nop(&p); /* load delay */
 	build_make_valid(&p, &r, K0, K1);
-	build_r3000_pte_reload(&p, K1);
-	build_r3000_tlb_write(&p, &l, &r, K0);
-	build_r3000_tlbchange_handler_tail(&p, K0);
+	build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
 
 	l_nopage_tlbl(&l, p);
 	i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
@@ -1506,13 +1517,10 @@ static void __init build_r3000_tlb_load_handler(void)
 	{
 		int i;
 
-		for (i = 0; i < FASTPATH_SIZE; i++)
+		for (i = 0; i < (p - handle_tlbl); i++)
 			printk("%08x\n", handle_tlbl[i]);
 	}
 #endif
-
-	flush_icache_range((unsigned long)handle_tlbl,
-			   (unsigned long)handle_tlbl + FASTPATH_SIZE * sizeof(u32));
 }
 
 static void __init build_r3000_tlb_store_handler(void)
@@ -1527,10 +1535,9 @@ static void __init build_r3000_tlb_store_handler(void)
 
 	build_r3000_tlbchange_handler_head(&p, K0, K1);
 	build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
+	i_nop(&p); /* load delay */
 	build_make_write(&p, &r, K0, K1);
-	build_r3000_pte_reload(&p, K1);
-	build_r3000_tlb_write(&p, &l, &r, K0);
-	build_r3000_tlbchange_handler_tail(&p, K0);
+	build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
 
 	l_nopage_tlbs(&l, p);
 	i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
@@ -1547,13 +1554,10 @@ static void __init build_r3000_tlb_store_handler(void)
 	{
 		int i;
 
-		for (i = 0; i < FASTPATH_SIZE; i++)
+		for (i = 0; i < (p - handle_tlbs); i++)
 			printk("%08x\n", handle_tlbs[i]);
 	}
 #endif
-
-	flush_icache_range((unsigned long)handle_tlbs,
-			   (unsigned long)handle_tlbs + FASTPATH_SIZE * sizeof(u32));
 }
 
 static void __init build_r3000_tlb_modify_handler(void)
@@ -1568,10 +1572,9 @@ static void __init build_r3000_tlb_modify_handler(void)
 
 	build_r3000_tlbchange_handler_head(&p, K0, K1);
 	build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
+	i_nop(&p); /* load delay */
 	build_make_write(&p, &r, K0, K1);
-	build_r3000_pte_reload(&p, K1);
-	i_tlbwi(&p);
-	build_r3000_tlbchange_handler_tail(&p, K0);
+	build_r3000_pte_reload_tlbwi(&p, K0, K1);
 
 	l_nopage_tlbm(&l, p);
 	i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
@@ -1588,13 +1591,10 @@ static void __init build_r3000_tlb_modify_handler(void)
 	{
 		int i;
 
-		for (i = 0; i < FASTPATH_SIZE; i++)
+		for (i = 0; i < (p - handle_tlbm); i++)
 			printk("%08x\n", handle_tlbm[i]);
 	}
 #endif
-
-	flush_icache_range((unsigned long)handle_tlbm,
-			   (unsigned long)handle_tlbm + FASTPATH_SIZE * sizeof(u32));
 }
 
 /*
@@ -1620,7 +1620,7 @@ build_r4000_tlbchange_handler_head(u32 **p, struct label **l,
 #ifdef CONFIG_SMP
 	l_smp_pgtable_change(l, *p);
 # endif
-	iPTE_LW(p, l, pte, 0, ptr); /* get even pte */
+	iPTE_LW(p, l, pte, ptr); /* get even pte */
 	build_tlb_probe_entry(p);
 }
 
@@ -1680,13 +1680,10 @@ static void __init build_r4000_tlb_load_handler(void)
 	{
 		int i;
 
-		for (i = 0; i < FASTPATH_SIZE; i++)
+		for (i = 0; i < (p - handle_tlbl); i++)
 			printk("%08x\n", handle_tlbl[i]);
 	}
 #endif
-
-	flush_icache_range((unsigned long)handle_tlbl,
-			   (unsigned long)handle_tlbl + FASTPATH_SIZE * sizeof(u32));
 }
 
 static void __init build_r4000_tlb_store_handler(void)
@@ -1719,13 +1716,10 @@ static void __init build_r4000_tlb_store_handler(void)
 	{
 		int i;
 
-		for (i = 0; i < FASTPATH_SIZE; i++)
+		for (i = 0; i < (p - handle_tlbs); i++)
 			printk("%08x\n", handle_tlbs[i]);
 	}
 #endif
-
-	flush_icache_range((unsigned long)handle_tlbs,
-			   (unsigned long)handle_tlbs + FASTPATH_SIZE * sizeof(u32));
 }
 
 static void __init build_r4000_tlb_modify_handler(void)
@@ -1759,13 +1753,10 @@ static void __init build_r4000_tlb_modify_handler(void)
 	{
 		int i;
 
-		for (i = 0; i < FASTPATH_SIZE; i++)
+		for (i = 0; i < (p - handle_tlbm); i++)
 			printk("%08x\n", handle_tlbm[i]);
 	}
 #endif
-
-	flush_icache_range((unsigned long)handle_tlbm,
-			   (unsigned long)handle_tlbm + FASTPATH_SIZE * sizeof(u32));
 }
 
 void __init build_tlb_refill_handler(void)
@@ -1813,3 +1804,13 @@ void __init build_tlb_refill_handler(void)
 		}
 	}
 }
+
+void __init flush_tlb_handlers(void)
+{
+	flush_icache_range((unsigned long)handle_tlbl,
+			   (unsigned long)handle_tlbl + sizeof(handle_tlbl));
+	flush_icache_range((unsigned long)handle_tlbs,
+			   (unsigned long)handle_tlbs + sizeof(handle_tlbs));
+	flush_icache_range((unsigned long)handle_tlbm,
+			   (unsigned long)handle_tlbm + sizeof(handle_tlbm));
+}