From 0eb948dd7f7c3cec37440c16a6c738c8e75efcda Mon Sep 17 00:00:00 2001
From: Russell King <rmk+kernel@arm.linux.org.uk>
Date: Thu, 19 Nov 2009 11:12:15 +0000
Subject: ARM: cache-l2x0: avoid taking spinlock for every iteration

Taking the spinlock for every iteration is very expensive; instead,
batch iterations up into 4K blocks, releasing and reacquiring the
spinlock between each block.

Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
---
 arch/arm/mm/cache-l2x0.c | 65 ++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 52 insertions(+), 13 deletions(-)

(limited to 'arch/arm')

diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index b480f1d3591..c1b7bfff47f 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -31,14 +31,10 @@ static DEFINE_SPINLOCK(l2x0_lock);
 static inline void sync_writel(unsigned long val, unsigned long reg,
 			       unsigned long complete_mask)
 {
-	unsigned long flags;
-
-	spin_lock_irqsave(&l2x0_lock, flags);
 	writel(val, l2x0_base + reg);
 	/* wait for the operation to complete */
 	while (readl(l2x0_base + reg) & complete_mask)
 		;
-	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static inline void cache_sync(void)
@@ -48,15 +44,20 @@ static inline void cache_sync(void)
 
 static inline void l2x0_inv_all(void)
 {
+	unsigned long flags;
+
 	/* invalidate all ways */
+	spin_lock_irqsave(&l2x0_lock, flags);
 	sync_writel(0xff, L2X0_INV_WAY, 0xff);
 	cache_sync();
+	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_inv_range(unsigned long start, unsigned long end)
 {
-	unsigned long addr;
+	unsigned long flags;
 
+	spin_lock_irqsave(&l2x0_lock, flags);
 	if (start & (CACHE_LINE_SIZE - 1)) {
 		start &= ~(CACHE_LINE_SIZE - 1);
 		sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1);
@@ -68,29 +69,67 @@ static void l2x0_inv_range(unsigned long start, unsigned long end)
 		sync_writel(end, L2X0_CLEAN_INV_LINE_PA, 1);
 	}
 
-	for (addr = start; addr < end; addr += CACHE_LINE_SIZE)
-		sync_writel(addr, L2X0_INV_LINE_PA, 1);
+	while (start < end) {
+		unsigned long blk_end = start + min(end - start, 4096UL);
+
+		while (start < blk_end) {
+			sync_writel(start, L2X0_INV_LINE_PA, 1);
+			start += CACHE_LINE_SIZE;
+		}
+
+		if (blk_end < end) {
+			spin_unlock_irqrestore(&l2x0_lock, flags);
+			spin_lock_irqsave(&l2x0_lock, flags);
+		}
+	}
 	cache_sync();
+	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_clean_range(unsigned long start, unsigned long end)
 {
-	unsigned long addr;
+	unsigned long flags;
 
+	spin_lock_irqsave(&l2x0_lock, flags);
 	start &= ~(CACHE_LINE_SIZE - 1);
-	for (addr = start; addr < end; addr += CACHE_LINE_SIZE)
-		sync_writel(addr, L2X0_CLEAN_LINE_PA, 1);
+	while (start < end) {
+		unsigned long blk_end = start + min(end - start, 4096UL);
+
+		while (start < blk_end) {
+			sync_writel(start, L2X0_CLEAN_LINE_PA, 1);
+			start += CACHE_LINE_SIZE;
+		}
+
+		if (blk_end < end) {
+			spin_unlock_irqrestore(&l2x0_lock, flags);
+			spin_lock_irqsave(&l2x0_lock, flags);
+		}
+	}
 	cache_sync();
+	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_flush_range(unsigned long start, unsigned long end)
 {
-	unsigned long addr;
+	unsigned long flags;
 
+	spin_lock_irqsave(&l2x0_lock, flags);
 	start &= ~(CACHE_LINE_SIZE - 1);
-	for (addr = start; addr < end; addr += CACHE_LINE_SIZE)
-		sync_writel(addr, L2X0_CLEAN_INV_LINE_PA, 1);
+	while (start < end) {
+		unsigned long blk_end = start + min(end - start, 4096UL);
+
+		while (start < blk_end) {
+			sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1);
+			start += CACHE_LINE_SIZE;
+		}
+
+		if (blk_end < end) {
+			spin_unlock_irqrestore(&l2x0_lock, flags);
+			spin_lock_irqsave(&l2x0_lock, flags);
+		}
+	}
 	cache_sync();
+	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
-- 
cgit v1.2.3


From 3d1074349b22c9653e746282564136c87668c2b8 Mon Sep 17 00:00:00 2001
From: Russell King <rmk+kernel@arm.linux.org.uk>
Date: Thu, 19 Nov 2009 11:41:09 +0000
Subject: ARM: cache-l2x0: make better use of background cache handling

There's no point having the hardware support background operations
if we issue a cache operation, and then wait for it to complete
before calculating the address of the next operation.  We gain no
advantage in the cache controller stalling the bus until completion.

What we should be doing is using the 'wait' time productively by
calculating the address of the next operation, and only then waiting
for the previous operation to complete.  This means that cache
operations can occur in parallel with the CPU calculating the next
address.

Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
---
 arch/arm/mm/cache-l2x0.c | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

(limited to 'arch/arm')

diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index c1b7bfff47f..ec85dda1e73 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -28,18 +28,18 @@
 static void __iomem *l2x0_base;
 static DEFINE_SPINLOCK(l2x0_lock);
 
-static inline void sync_writel(unsigned long val, unsigned long reg,
-			       unsigned long complete_mask)
+static inline void cache_wait(void __iomem *reg, unsigned long mask)
 {
-	writel(val, l2x0_base + reg);
 	/* wait for the operation to complete */
-	while (readl(l2x0_base + reg) & complete_mask)
+	while (readl(reg) & mask)
 		;
 }
 
 static inline void cache_sync(void)
 {
-	sync_writel(0, L2X0_CACHE_SYNC, 1);
+	void __iomem *base = l2x0_base;
+	writel(0, base + L2X0_CACHE_SYNC);
+	cache_wait(base + L2X0_CACHE_SYNC, 1);
 }
 
 static inline void l2x0_inv_all(void)
@@ -48,32 +48,37 @@ static inline void l2x0_inv_all(void)
 
 	/* invalidate all ways */
 	spin_lock_irqsave(&l2x0_lock, flags);
-	sync_writel(0xff, L2X0_INV_WAY, 0xff);
+	writel(0xff, l2x0_base + L2X0_INV_WAY);
+	cache_wait(l2x0_base + L2X0_INV_WAY, 0xff);
 	cache_sync();
 	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_inv_range(unsigned long start, unsigned long end)
 {
+	void __iomem *base = l2x0_base;
 	unsigned long flags;
 
 	spin_lock_irqsave(&l2x0_lock, flags);
 	if (start & (CACHE_LINE_SIZE - 1)) {
 		start &= ~(CACHE_LINE_SIZE - 1);
-		sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1);
+		cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+		writel(start, base + L2X0_CLEAN_INV_LINE_PA);
 		start += CACHE_LINE_SIZE;
 	}
 
 	if (end & (CACHE_LINE_SIZE - 1)) {
 		end &= ~(CACHE_LINE_SIZE - 1);
-		sync_writel(end, L2X0_CLEAN_INV_LINE_PA, 1);
+		cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+		writel(end, base + L2X0_CLEAN_INV_LINE_PA);
 	}
 
 	while (start < end) {
 		unsigned long blk_end = start + min(end - start, 4096UL);
 
 		while (start < blk_end) {
-			sync_writel(start, L2X0_INV_LINE_PA, 1);
+			cache_wait(base + L2X0_INV_LINE_PA, 1);
+			writel(start, base + L2X0_INV_LINE_PA);
 			start += CACHE_LINE_SIZE;
 		}
 
@@ -82,12 +87,14 @@ static void l2x0_inv_range(unsigned long start, unsigned long end)
 			spin_lock_irqsave(&l2x0_lock, flags);
 		}
 	}
+	cache_wait(base + L2X0_INV_LINE_PA, 1);
 	cache_sync();
 	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_clean_range(unsigned long start, unsigned long end)
 {
+	void __iomem *base = l2x0_base;
 	unsigned long flags;
 
 	spin_lock_irqsave(&l2x0_lock, flags);
@@ -96,7 +103,8 @@ static void l2x0_clean_range(unsigned long start, unsigned long end)
 		unsigned long blk_end = start + min(end - start, 4096UL);
 
 		while (start < blk_end) {
-			sync_writel(start, L2X0_CLEAN_LINE_PA, 1);
+			cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
+			writel(start, base + L2X0_CLEAN_LINE_PA);
 			start += CACHE_LINE_SIZE;
 		}
 
@@ -105,12 +113,14 @@ static void l2x0_clean_range(unsigned long start, unsigned long end)
 			spin_lock_irqsave(&l2x0_lock, flags);
 		}
 	}
+	cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
 	cache_sync();
 	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_flush_range(unsigned long start, unsigned long end)
 {
+	void __iomem *base = l2x0_base;
 	unsigned long flags;
 
 	spin_lock_irqsave(&l2x0_lock, flags);
@@ -119,7 +129,8 @@ static void l2x0_flush_range(unsigned long start, unsigned long end)
 		unsigned long blk_end = start + min(end - start, 4096UL);
 
 		while (start < blk_end) {
-			sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1);
+			cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+			writel(start, base + L2X0_CLEAN_INV_LINE_PA);
 			start += CACHE_LINE_SIZE;
 		}
 
@@ -128,6 +139,7 @@ static void l2x0_flush_range(unsigned long start, unsigned long end)
 			spin_lock_irqsave(&l2x0_lock, flags);
 		}
 	}
+	cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
 	cache_sync();
 	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
-- 
cgit v1.2.3


From ccaf5f05b218e5eb41e2f5cdfd26b18dce4a0218 Mon Sep 17 00:00:00 2001
From: Nicolas Pitre <nico@fluxnic.net>
Date: Fri, 11 Dec 2009 02:21:57 +0100
Subject: ARM: 5848/1: kill flush_ioremap_region()

There is not enough users to warrant its existence, and it is actually
an obstacle to progress with the new DMA API which cannot cover this
case properly.

To keep backward compatibility, let's perform the necessary custom
cache maintenance locally in the only driver affected.

Signed-off-by: Nicolas Pitre <nico@marvell.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
---
 arch/arm/include/asm/cacheflush.h | 7 -------
 arch/arm/mm/proc-syms.c           | 1 -
 2 files changed, 8 deletions(-)

(limited to 'arch/arm')

diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h
index 73eceb87e58..3db7acd39a6 100644
--- a/arch/arm/include/asm/cacheflush.h
+++ b/arch/arm/include/asm/cacheflush.h
@@ -465,13 +465,6 @@ static inline void flush_kernel_dcache_page(struct page *page)
  */
 #define flush_icache_page(vma,page)	do { } while (0)
 
-static inline void flush_ioremap_region(unsigned long phys, void __iomem *virt,
-	unsigned offset, size_t size)
-{
-	const void *start = (void __force *)virt + offset;
-	dmac_inv_range(start, start + size);
-}
-
 /*
  * flush_cache_vmap() is used when creating mappings (eg, via vmap,
  * vmalloc, ioremap etc) in kernel space for pages.  On non-VIPT
diff --git a/arch/arm/mm/proc-syms.c b/arch/arm/mm/proc-syms.c
index ac5c80062b7..f604aa8acae 100644
--- a/arch/arm/mm/proc-syms.c
+++ b/arch/arm/mm/proc-syms.c
@@ -28,7 +28,6 @@ EXPORT_SYMBOL(__cpuc_flush_user_all);
 EXPORT_SYMBOL(__cpuc_flush_user_range);
 EXPORT_SYMBOL(__cpuc_coherent_kern_range);
 EXPORT_SYMBOL(__cpuc_flush_dcache_page);
-EXPORT_SYMBOL(dmac_inv_range);  /* because of flush_ioremap_region() */
 #else
 EXPORT_SYMBOL(cpu_cache);
 #endif
-- 
cgit v1.2.3


From 2c9b9c8490b60428fa2d1c64042f7c7caed93940 Mon Sep 17 00:00:00 2001
From: Russell King <rmk+kernel@arm.linux.org.uk>
Date: Thu, 26 Nov 2009 12:56:21 +0000
Subject: ARM: add size argument to __cpuc_flush_dcache_page

... and rename the function since it no longer operates on just
pages.

Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
---
 arch/arm/include/asm/cacheflush.h | 10 +++++-----
 arch/arm/mm/cache-fa.S            | 11 ++++++-----
 arch/arm/mm/cache-v3.S            |  9 +++++----
 arch/arm/mm/cache-v4.S            |  9 +++++----
 arch/arm/mm/cache-v4wb.S          | 11 ++++++-----
 arch/arm/mm/cache-v4wt.S          | 11 ++++++-----
 arch/arm/mm/cache-v6.S            | 11 ++++++-----
 arch/arm/mm/cache-v7.S            | 13 +++++++------
 arch/arm/mm/flush.c               |  4 ++--
 arch/arm/mm/highmem.c             |  2 +-
 arch/arm/mm/nommu.c               |  2 +-
 arch/arm/mm/proc-arm1020.S        | 11 ++++++-----
 arch/arm/mm/proc-arm1020e.S       | 11 ++++++-----
 arch/arm/mm/proc-arm1022.S        | 11 ++++++-----
 arch/arm/mm/proc-arm1026.S        | 11 ++++++-----
 arch/arm/mm/proc-arm920.S         | 11 ++++++-----
 arch/arm/mm/proc-arm922.S         | 11 ++++++-----
 arch/arm/mm/proc-arm925.S         | 11 ++++++-----
 arch/arm/mm/proc-arm926.S         | 11 ++++++-----
 arch/arm/mm/proc-arm940.S         |  9 +++++----
 arch/arm/mm/proc-arm946.S         | 11 ++++++-----
 arch/arm/mm/proc-feroceon.S       | 15 ++++++++-------
 arch/arm/mm/proc-mohawk.S         | 11 ++++++-----
 arch/arm/mm/proc-syms.c           |  2 +-
 arch/arm/mm/proc-xsc3.S           | 11 ++++++-----
 arch/arm/mm/proc-xscale.S         | 13 +++++++------
 26 files changed, 137 insertions(+), 116 deletions(-)

(limited to 'arch/arm')

diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h
index 3db7acd39a6..730aefcfbee 100644
--- a/arch/arm/include/asm/cacheflush.h
+++ b/arch/arm/include/asm/cacheflush.h
@@ -211,7 +211,7 @@ struct cpu_cache_fns {
 
 	void (*coherent_kern_range)(unsigned long, unsigned long);
 	void (*coherent_user_range)(unsigned long, unsigned long);
-	void (*flush_kern_dcache_page)(void *);
+	void (*flush_kern_dcache_area)(void *, size_t);
 
 	void (*dma_inv_range)(const void *, const void *);
 	void (*dma_clean_range)(const void *, const void *);
@@ -236,7 +236,7 @@ extern struct cpu_cache_fns cpu_cache;
 #define __cpuc_flush_user_range		cpu_cache.flush_user_range
 #define __cpuc_coherent_kern_range	cpu_cache.coherent_kern_range
 #define __cpuc_coherent_user_range	cpu_cache.coherent_user_range
-#define __cpuc_flush_dcache_page	cpu_cache.flush_kern_dcache_page
+#define __cpuc_flush_dcache_area	cpu_cache.flush_kern_dcache_area
 
 /*
  * These are private to the dma-mapping API.  Do not use directly.
@@ -255,14 +255,14 @@ extern struct cpu_cache_fns cpu_cache;
 #define __cpuc_flush_user_range		__glue(_CACHE,_flush_user_cache_range)
 #define __cpuc_coherent_kern_range	__glue(_CACHE,_coherent_kern_range)
 #define __cpuc_coherent_user_range	__glue(_CACHE,_coherent_user_range)
-#define __cpuc_flush_dcache_page	__glue(_CACHE,_flush_kern_dcache_page)
+#define __cpuc_flush_dcache_area	__glue(_CACHE,_flush_kern_dcache_area)
 
 extern void __cpuc_flush_kern_all(void);
 extern void __cpuc_flush_user_all(void);
 extern void __cpuc_flush_user_range(unsigned long, unsigned long, unsigned int);
 extern void __cpuc_coherent_kern_range(unsigned long, unsigned long);
 extern void __cpuc_coherent_user_range(unsigned long, unsigned long);
-extern void __cpuc_flush_dcache_page(void *);
+extern void __cpuc_flush_dcache_area(void *, size_t);
 
 /*
  * These are private to the dma-mapping API.  Do not use directly.
@@ -448,7 +448,7 @@ static inline void flush_kernel_dcache_page(struct page *page)
 {
 	/* highmem pages are always flushed upon kunmap already */
 	if ((cache_is_vivt() || cache_is_vipt_aliasing()) && !PageHighMem(page))
-		__cpuc_flush_dcache_page(page_address(page));
+		__cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
 }
 
 #define flush_dcache_mmap_lock(mapping) \
diff --git a/arch/arm/mm/cache-fa.S b/arch/arm/mm/cache-fa.S
index b63a8f7b95c..a89444a3c01 100644
--- a/arch/arm/mm/cache-fa.S
+++ b/arch/arm/mm/cache-fa.S
@@ -127,15 +127,16 @@ ENTRY(fa_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(kaddr)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure that the data held in the page kaddr is written back
  *	to the page in question.
  *
- *	- kaddr   - kernel address (guaranteed to be page aligned)
+ *	- addr	- kernel address
+ *	- size	- size of region
  */
-ENTRY(fa_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(fa_flush_kern_dcache_area)
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean & invalidate D line
 	add	r0, r0, #CACHE_DLINESIZE
 	cmp	r0, r1
@@ -213,7 +214,7 @@ ENTRY(fa_cache_fns)
 	.long	fa_flush_user_cache_range
 	.long	fa_coherent_kern_range
 	.long	fa_coherent_user_range
-	.long	fa_flush_kern_dcache_page
+	.long	fa_flush_kern_dcache_area
 	.long	fa_dma_inv_range
 	.long	fa_dma_clean_range
 	.long	fa_dma_flush_range
diff --git a/arch/arm/mm/cache-v3.S b/arch/arm/mm/cache-v3.S
index 8a4abebc478..2a482731ea3 100644
--- a/arch/arm/mm/cache-v3.S
+++ b/arch/arm/mm/cache-v3.S
@@ -72,14 +72,15 @@ ENTRY(v3_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *page, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(v3_flush_kern_dcache_page)
+ENTRY(v3_flush_kern_dcache_area)
 	/* FALLTHROUGH */
 
 /*
@@ -129,7 +130,7 @@ ENTRY(v3_cache_fns)
 	.long	v3_flush_user_cache_range
 	.long	v3_coherent_kern_range
 	.long	v3_coherent_user_range
-	.long	v3_flush_kern_dcache_page
+	.long	v3_flush_kern_dcache_area
 	.long	v3_dma_inv_range
 	.long	v3_dma_clean_range
 	.long	v3_dma_flush_range
diff --git a/arch/arm/mm/cache-v4.S b/arch/arm/mm/cache-v4.S
index 3668611cb40..5c7da3e372e 100644
--- a/arch/arm/mm/cache-v4.S
+++ b/arch/arm/mm/cache-v4.S
@@ -82,14 +82,15 @@ ENTRY(v4_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(v4_flush_kern_dcache_page)
+ENTRY(v4_flush_kern_dcache_area)
 	/* FALLTHROUGH */
 
 /*
@@ -141,7 +142,7 @@ ENTRY(v4_cache_fns)
 	.long	v4_flush_user_cache_range
 	.long	v4_coherent_kern_range
 	.long	v4_coherent_user_range
-	.long	v4_flush_kern_dcache_page
+	.long	v4_flush_kern_dcache_area
 	.long	v4_dma_inv_range
 	.long	v4_dma_clean_range
 	.long	v4_dma_flush_range
diff --git a/arch/arm/mm/cache-v4wb.S b/arch/arm/mm/cache-v4wb.S
index 2ebc1b3bf85..3dbedf1ec0e 100644
--- a/arch/arm/mm/cache-v4wb.S
+++ b/arch/arm/mm/cache-v4wb.S
@@ -114,15 +114,16 @@ ENTRY(v4wb_flush_user_cache_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(v4wb_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(v4wb_flush_kern_dcache_area)
+	add	r1, r0, r1
 	/* fall through */
 
 /*
@@ -224,7 +225,7 @@ ENTRY(v4wb_cache_fns)
 	.long	v4wb_flush_user_cache_range
 	.long	v4wb_coherent_kern_range
 	.long	v4wb_coherent_user_range
-	.long	v4wb_flush_kern_dcache_page
+	.long	v4wb_flush_kern_dcache_area
 	.long	v4wb_dma_inv_range
 	.long	v4wb_dma_clean_range
 	.long	v4wb_dma_flush_range
diff --git a/arch/arm/mm/cache-v4wt.S b/arch/arm/mm/cache-v4wt.S
index c54fa2cc40e..b3b7410270b 100644
--- a/arch/arm/mm/cache-v4wt.S
+++ b/arch/arm/mm/cache-v4wt.S
@@ -117,17 +117,18 @@ ENTRY(v4wt_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(v4wt_flush_kern_dcache_page)
+ENTRY(v4wt_flush_kern_dcache_area)
 	mov	r2, #0
 	mcr	p15, 0, r2, c7, c5, 0		@ invalidate I cache
-	add	r1, r0, #PAGE_SZ
+	add	r1, r0, r1
 	/* fallthrough */
 
 /*
@@ -180,7 +181,7 @@ ENTRY(v4wt_cache_fns)
 	.long	v4wt_flush_user_cache_range
 	.long	v4wt_coherent_kern_range
 	.long	v4wt_coherent_user_range
-	.long	v4wt_flush_kern_dcache_page
+	.long	v4wt_flush_kern_dcache_area
 	.long	v4wt_dma_inv_range
 	.long	v4wt_dma_clean_range
 	.long	v4wt_dma_flush_range
diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S
index 295e25dd638..4ba0a24ce6f 100644
--- a/arch/arm/mm/cache-v6.S
+++ b/arch/arm/mm/cache-v6.S
@@ -159,15 +159,16 @@ ENDPROC(v6_coherent_user_range)
 ENDPROC(v6_coherent_kern_range)
 
 /*
- *	v6_flush_kern_dcache_page(kaddr)
+ *	v6_flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure that the data held in the page kaddr is written back
  *	to the page in question.
  *
- *	- kaddr   - kernel address (guaranteed to be page aligned)
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(v6_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(v6_flush_kern_dcache_area)
+	add	r1, r0, r1
 1:
 #ifdef HARVARD_CACHE
 	mcr	p15, 0, r0, c7, c14, 1		@ clean & invalidate D line
@@ -271,7 +272,7 @@ ENTRY(v6_cache_fns)
 	.long	v6_flush_user_cache_range
 	.long	v6_coherent_kern_range
 	.long	v6_coherent_user_range
-	.long	v6_flush_kern_dcache_page
+	.long	v6_flush_kern_dcache_area
 	.long	v6_dma_inv_range
 	.long	v6_dma_clean_range
 	.long	v6_dma_flush_range
diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S
index e1bd9759617..9073db849fb 100644
--- a/arch/arm/mm/cache-v7.S
+++ b/arch/arm/mm/cache-v7.S
@@ -186,16 +186,17 @@ ENDPROC(v7_coherent_kern_range)
 ENDPROC(v7_coherent_user_range)
 
 /*
- *	v7_flush_kern_dcache_page(kaddr)
+ *	v7_flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure that the data held in the page kaddr is written back
  *	to the page in question.
  *
- *	- kaddr   - kernel address (guaranteed to be page aligned)
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(v7_flush_kern_dcache_page)
+ENTRY(v7_flush_kern_dcache_area)
 	dcache_line_size r2, r3
-	add	r1, r0, #PAGE_SZ
+	add	r1, r0, r1
 1:
 	mcr	p15, 0, r0, c7, c14, 1		@ clean & invalidate D line / unified line
 	add	r0, r0, r2
@@ -203,7 +204,7 @@ ENTRY(v7_flush_kern_dcache_page)
 	blo	1b
 	dsb
 	mov	pc, lr
-ENDPROC(v7_flush_kern_dcache_page)
+ENDPROC(v7_flush_kern_dcache_area)
 
 /*
  *	v7_dma_inv_range(start,end)
@@ -279,7 +280,7 @@ ENTRY(v7_cache_fns)
 	.long	v7_flush_user_cache_range
 	.long	v7_coherent_kern_range
 	.long	v7_coherent_user_range
-	.long	v7_flush_kern_dcache_page
+	.long	v7_flush_kern_dcache_area
 	.long	v7_dma_inv_range
 	.long	v7_dma_clean_range
 	.long	v7_dma_flush_range
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index 329594e760c..6f3a4b7a3b8 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -131,7 +131,7 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page)
 	 */
 	if (addr)
 #endif
-		__cpuc_flush_dcache_page(addr);
+		__cpuc_flush_dcache_area(addr, PAGE_SIZE);
 
 	/*
 	 * If this is a page cache page, and we have an aliasing VIPT cache,
@@ -258,5 +258,5 @@ void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned l
 	 * in this mapping of the page.  FIXME: this is overkill
 	 * since we actually ask for a write-back and invalidate.
 	 */
-	__cpuc_flush_dcache_page(page_address(page));
+	__cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
 }
diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c
index 30f82fb5918..2be1ec7c1b4 100644
--- a/arch/arm/mm/highmem.c
+++ b/arch/arm/mm/highmem.c
@@ -79,7 +79,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type)
 	unsigned int idx = type + KM_TYPE_NR * smp_processor_id();
 
 	if (kvaddr >= (void *)FIXADDR_START) {
-		__cpuc_flush_dcache_page((void *)vaddr);
+		__cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE);
 #ifdef CONFIG_DEBUG_HIGHMEM
 		BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx));
 		set_pte_ext(TOP_PTE(vaddr), __pte(0), 0);
diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c
index 900811cc913..374a8311bc8 100644
--- a/arch/arm/mm/nommu.c
+++ b/arch/arm/mm/nommu.c
@@ -61,7 +61,7 @@ void setup_mm_for_reboot(char mode)
 
 void flush_dcache_page(struct page *page)
 {
-	__cpuc_flush_dcache_page(page_address(page));
+	__cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
 }
 EXPORT_SYMBOL(flush_dcache_page);
 
diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S
index d9fb4b98c49..8012e24282b 100644
--- a/arch/arm/mm/proc-arm1020.S
+++ b/arch/arm/mm/proc-arm1020.S
@@ -231,17 +231,18 @@ ENTRY(arm1020_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- page	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(arm1020_flush_kern_dcache_page)
+ENTRY(arm1020_flush_kern_dcache_area)
 	mov	ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
-	add	r1, r0, #PAGE_SZ
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean+invalidate D entry
 	mcr	p15, 0, ip, c7, c10, 4		@ drain WB
 	add	r0, r0, #CACHE_DLINESIZE
@@ -335,7 +336,7 @@ ENTRY(arm1020_cache_fns)
 	.long	arm1020_flush_user_cache_range
 	.long	arm1020_coherent_kern_range
 	.long	arm1020_coherent_user_range
-	.long	arm1020_flush_kern_dcache_page
+	.long	arm1020_flush_kern_dcache_area
 	.long	arm1020_dma_inv_range
 	.long	arm1020_dma_clean_range
 	.long	arm1020_dma_flush_range
diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S
index 7453b75dcea..41fe25d234f 100644
--- a/arch/arm/mm/proc-arm1020e.S
+++ b/arch/arm/mm/proc-arm1020e.S
@@ -225,17 +225,18 @@ ENTRY(arm1020e_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- page	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(arm1020e_flush_kern_dcache_page)
+ENTRY(arm1020e_flush_kern_dcache_area)
 	mov	ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
-	add	r1, r0, #PAGE_SZ
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean+invalidate D entry
 	add	r0, r0, #CACHE_DLINESIZE
 	cmp	r0, r1
@@ -321,7 +322,7 @@ ENTRY(arm1020e_cache_fns)
 	.long	arm1020e_flush_user_cache_range
 	.long	arm1020e_coherent_kern_range
 	.long	arm1020e_coherent_user_range
-	.long	arm1020e_flush_kern_dcache_page
+	.long	arm1020e_flush_kern_dcache_area
 	.long	arm1020e_dma_inv_range
 	.long	arm1020e_dma_clean_range
 	.long	arm1020e_dma_flush_range
diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S
index 8eb72d75a8b..20a5b1b31a7 100644
--- a/arch/arm/mm/proc-arm1022.S
+++ b/arch/arm/mm/proc-arm1022.S
@@ -214,17 +214,18 @@ ENTRY(arm1022_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- page	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(arm1022_flush_kern_dcache_page)
+ENTRY(arm1022_flush_kern_dcache_area)
 	mov	ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
-	add	r1, r0, #PAGE_SZ
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean+invalidate D entry
 	add	r0, r0, #CACHE_DLINESIZE
 	cmp	r0, r1
@@ -310,7 +311,7 @@ ENTRY(arm1022_cache_fns)
 	.long	arm1022_flush_user_cache_range
 	.long	arm1022_coherent_kern_range
 	.long	arm1022_coherent_user_range
-	.long	arm1022_flush_kern_dcache_page
+	.long	arm1022_flush_kern_dcache_area
 	.long	arm1022_dma_inv_range
 	.long	arm1022_dma_clean_range
 	.long	arm1022_dma_flush_range
diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S
index 3b59f0d6713..96aedb10fcc 100644
--- a/arch/arm/mm/proc-arm1026.S
+++ b/arch/arm/mm/proc-arm1026.S
@@ -208,17 +208,18 @@ ENTRY(arm1026_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- page	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(arm1026_flush_kern_dcache_page)
+ENTRY(arm1026_flush_kern_dcache_area)
 	mov	ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
-	add	r1, r0, #PAGE_SZ
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean+invalidate D entry
 	add	r0, r0, #CACHE_DLINESIZE
 	cmp	r0, r1
@@ -304,7 +305,7 @@ ENTRY(arm1026_cache_fns)
 	.long	arm1026_flush_user_cache_range
 	.long	arm1026_coherent_kern_range
 	.long	arm1026_coherent_user_range
-	.long	arm1026_flush_kern_dcache_page
+	.long	arm1026_flush_kern_dcache_area
 	.long	arm1026_dma_inv_range
 	.long	arm1026_dma_clean_range
 	.long	arm1026_dma_flush_range
diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S
index 2b7c197cc58..471669e2d7c 100644
--- a/arch/arm/mm/proc-arm920.S
+++ b/arch/arm/mm/proc-arm920.S
@@ -207,15 +207,16 @@ ENTRY(arm920_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(arm920_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(arm920_flush_kern_dcache_area)
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean+invalidate D entry
 	add	r0, r0, #CACHE_DLINESIZE
 	cmp	r0, r1
@@ -293,7 +294,7 @@ ENTRY(arm920_cache_fns)
 	.long	arm920_flush_user_cache_range
 	.long	arm920_coherent_kern_range
 	.long	arm920_coherent_user_range
-	.long	arm920_flush_kern_dcache_page
+	.long	arm920_flush_kern_dcache_area
 	.long	arm920_dma_inv_range
 	.long	arm920_dma_clean_range
 	.long	arm920_dma_flush_range
diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S
index 06a1aa4e339..ee111b00fa4 100644
--- a/arch/arm/mm/proc-arm922.S
+++ b/arch/arm/mm/proc-arm922.S
@@ -209,15 +209,16 @@ ENTRY(arm922_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(arm922_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(arm922_flush_kern_dcache_area)
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean+invalidate D entry
 	add	r0, r0, #CACHE_DLINESIZE
 	cmp	r0, r1
@@ -295,7 +296,7 @@ ENTRY(arm922_cache_fns)
 	.long	arm922_flush_user_cache_range
 	.long	arm922_coherent_kern_range
 	.long	arm922_coherent_user_range
-	.long	arm922_flush_kern_dcache_page
+	.long	arm922_flush_kern_dcache_area
 	.long	arm922_dma_inv_range
 	.long	arm922_dma_clean_range
 	.long	arm922_dma_flush_range
diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S
index cb53435a85a..8deb5bde58e 100644
--- a/arch/arm/mm/proc-arm925.S
+++ b/arch/arm/mm/proc-arm925.S
@@ -251,15 +251,16 @@ ENTRY(arm925_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(arm925_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(arm925_flush_kern_dcache_area)
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean+invalidate D entry
 	add	r0, r0, #CACHE_DLINESIZE
 	cmp	r0, r1
@@ -346,7 +347,7 @@ ENTRY(arm925_cache_fns)
 	.long	arm925_flush_user_cache_range
 	.long	arm925_coherent_kern_range
 	.long	arm925_coherent_user_range
-	.long	arm925_flush_kern_dcache_page
+	.long	arm925_flush_kern_dcache_area
 	.long	arm925_dma_inv_range
 	.long	arm925_dma_clean_range
 	.long	arm925_dma_flush_range
diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S
index 1c4848704bb..64db6e275a4 100644
--- a/arch/arm/mm/proc-arm926.S
+++ b/arch/arm/mm/proc-arm926.S
@@ -214,15 +214,16 @@ ENTRY(arm926_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(arm926_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(arm926_flush_kern_dcache_area)
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean+invalidate D entry
 	add	r0, r0, #CACHE_DLINESIZE
 	cmp	r0, r1
@@ -309,7 +310,7 @@ ENTRY(arm926_cache_fns)
 	.long	arm926_flush_user_cache_range
 	.long	arm926_coherent_kern_range
 	.long	arm926_coherent_user_range
-	.long	arm926_flush_kern_dcache_page
+	.long	arm926_flush_kern_dcache_area
 	.long	arm926_dma_inv_range
 	.long	arm926_dma_clean_range
 	.long	arm926_dma_flush_range
diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S
index 5b0f8464c8f..8196b9f401f 100644
--- a/arch/arm/mm/proc-arm940.S
+++ b/arch/arm/mm/proc-arm940.S
@@ -141,14 +141,15 @@ ENTRY(arm940_coherent_user_range)
 	/* FALLTHROUGH */
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(arm940_flush_kern_dcache_page)
+ENTRY(arm940_flush_kern_dcache_area)
 	mov	ip, #0
 	mov	r1, #(CACHE_DSEGMENTS - 1) << 4	@ 4 segments
 1:	orr	r3, r1, #(CACHE_DENTRIES - 1) << 26 @ 64 entries
@@ -238,7 +239,7 @@ ENTRY(arm940_cache_fns)
 	.long	arm940_flush_user_cache_range
 	.long	arm940_coherent_kern_range
 	.long	arm940_coherent_user_range
-	.long	arm940_flush_kern_dcache_page
+	.long	arm940_flush_kern_dcache_area
 	.long	arm940_dma_inv_range
 	.long	arm940_dma_clean_range
 	.long	arm940_dma_flush_range
diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S
index 40c0449a139..9a951239c86 100644
--- a/arch/arm/mm/proc-arm946.S
+++ b/arch/arm/mm/proc-arm946.S
@@ -183,16 +183,17 @@ ENTRY(arm946_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  * (same as arm926)
  */
-ENTRY(arm946_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(arm946_flush_kern_dcache_area)
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean+invalidate D entry
 	add	r0, r0, #CACHE_DLINESIZE
 	cmp	r0, r1
@@ -280,7 +281,7 @@ ENTRY(arm946_cache_fns)
 	.long	arm946_flush_user_cache_range
 	.long	arm946_coherent_kern_range
 	.long	arm946_coherent_user_range
-	.long	arm946_flush_kern_dcache_page
+	.long	arm946_flush_kern_dcache_area
 	.long	arm946_dma_inv_range
 	.long	arm946_dma_clean_range
 	.long	arm946_dma_flush_range
diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S
index d0d7795200f..dbc39383e66 100644
--- a/arch/arm/mm/proc-feroceon.S
+++ b/arch/arm/mm/proc-feroceon.S
@@ -226,16 +226,17 @@ ENTRY(feroceon_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
 	.align	5
-ENTRY(feroceon_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(feroceon_flush_kern_dcache_area)
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean+invalidate D entry
 	add	r0, r0, #CACHE_DLINESIZE
 	cmp	r0, r1
@@ -246,7 +247,7 @@ ENTRY(feroceon_flush_kern_dcache_page)
 	mov	pc, lr
 
 	.align	5
-ENTRY(feroceon_range_flush_kern_dcache_page)
+ENTRY(feroceon_range_flush_kern_dcache_area)
 	mrs	r2, cpsr
 	add	r1, r0, #PAGE_SZ - CACHE_DLINESIZE	@ top addr is inclusive
 	orr	r3, r2, #PSR_I_BIT
@@ -372,7 +373,7 @@ ENTRY(feroceon_cache_fns)
 	.long	feroceon_flush_user_cache_range
 	.long	feroceon_coherent_kern_range
 	.long	feroceon_coherent_user_range
-	.long	feroceon_flush_kern_dcache_page
+	.long	feroceon_flush_kern_dcache_area
 	.long	feroceon_dma_inv_range
 	.long	feroceon_dma_clean_range
 	.long	feroceon_dma_flush_range
@@ -383,7 +384,7 @@ ENTRY(feroceon_range_cache_fns)
 	.long	feroceon_flush_user_cache_range
 	.long	feroceon_coherent_kern_range
 	.long	feroceon_coherent_user_range
-	.long	feroceon_range_flush_kern_dcache_page
+	.long	feroceon_range_flush_kern_dcache_area
 	.long	feroceon_range_dma_inv_range
 	.long	feroceon_range_dma_clean_range
 	.long	feroceon_range_dma_flush_range
diff --git a/arch/arm/mm/proc-mohawk.S b/arch/arm/mm/proc-mohawk.S
index 52b5fd74fbb..9674d36cc97 100644
--- a/arch/arm/mm/proc-mohawk.S
+++ b/arch/arm/mm/proc-mohawk.S
@@ -186,15 +186,16 @@ ENTRY(mohawk_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(mohawk_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(mohawk_flush_kern_dcache_area)
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean+invalidate D entry
 	add	r0, r0, #CACHE_DLINESIZE
 	cmp	r0, r1
@@ -273,7 +274,7 @@ ENTRY(mohawk_cache_fns)
 	.long	mohawk_flush_user_cache_range
 	.long	mohawk_coherent_kern_range
 	.long	mohawk_coherent_user_range
-	.long	mohawk_flush_kern_dcache_page
+	.long	mohawk_flush_kern_dcache_area
 	.long	mohawk_dma_inv_range
 	.long	mohawk_dma_clean_range
 	.long	mohawk_dma_flush_range
diff --git a/arch/arm/mm/proc-syms.c b/arch/arm/mm/proc-syms.c
index f604aa8acae..3e6210b4d6d 100644
--- a/arch/arm/mm/proc-syms.c
+++ b/arch/arm/mm/proc-syms.c
@@ -27,7 +27,7 @@ EXPORT_SYMBOL(__cpuc_flush_kern_all);
 EXPORT_SYMBOL(__cpuc_flush_user_all);
 EXPORT_SYMBOL(__cpuc_flush_user_range);
 EXPORT_SYMBOL(__cpuc_coherent_kern_range);
-EXPORT_SYMBOL(__cpuc_flush_dcache_page);
+EXPORT_SYMBOL(__cpuc_flush_dcache_area);
 #else
 EXPORT_SYMBOL(cpu_cache);
 #endif
diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S
index fab134e2982..96456f54879 100644
--- a/arch/arm/mm/proc-xsc3.S
+++ b/arch/arm/mm/proc-xsc3.S
@@ -226,15 +226,16 @@ ENTRY(xsc3_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache.
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(xsc3_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(xsc3_flush_kern_dcache_area)
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c14, 1		@ clean/invalidate L1 D line
 	add	r0, r0, #CACHELINESIZE
 	cmp	r0, r1
@@ -309,7 +310,7 @@ ENTRY(xsc3_cache_fns)
 	.long	xsc3_flush_user_cache_range
 	.long	xsc3_coherent_kern_range
 	.long	xsc3_coherent_user_range
-	.long	xsc3_flush_kern_dcache_page
+	.long	xsc3_flush_kern_dcache_area
 	.long	xsc3_dma_inv_range
 	.long	xsc3_dma_clean_range
 	.long	xsc3_dma_flush_range
diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S
index f056c283682..93df47265f2 100644
--- a/arch/arm/mm/proc-xscale.S
+++ b/arch/arm/mm/proc-xscale.S
@@ -284,15 +284,16 @@ ENTRY(xscale_coherent_user_range)
 	mov	pc, lr
 
 /*
- *	flush_kern_dcache_page(void *page)
+ *	flush_kern_dcache_area(void *addr, size_t size)
  *
  *	Ensure no D cache aliasing occurs, either with itself or
  *	the I cache
  *
- *	- addr	- page aligned address
+ *	- addr	- kernel address
+ *	- size	- region size
  */
-ENTRY(xscale_flush_kern_dcache_page)
-	add	r1, r0, #PAGE_SZ
+ENTRY(xscale_flush_kern_dcache_area)
+	add	r1, r0, r1
 1:	mcr	p15, 0, r0, c7, c10, 1		@ clean D entry
 	mcr	p15, 0, r0, c7, c6, 1		@ invalidate D entry
 	add	r0, r0, #CACHELINESIZE
@@ -368,7 +369,7 @@ ENTRY(xscale_cache_fns)
 	.long	xscale_flush_user_cache_range
 	.long	xscale_coherent_kern_range
 	.long	xscale_coherent_user_range
-	.long	xscale_flush_kern_dcache_page
+	.long	xscale_flush_kern_dcache_area
 	.long	xscale_dma_inv_range
 	.long	xscale_dma_clean_range
 	.long	xscale_dma_flush_range
@@ -392,7 +393,7 @@ ENTRY(xscale_80200_A0_A1_cache_fns)
 	.long	xscale_flush_user_cache_range
 	.long	xscale_coherent_kern_range
 	.long	xscale_coherent_user_range
-	.long	xscale_flush_kern_dcache_page
+	.long	xscale_flush_kern_dcache_area
 	.long	xscale_dma_flush_range
 	.long	xscale_dma_clean_range
 	.long	xscale_dma_flush_range
-- 
cgit v1.2.3


From f74f7e57ae9fa12b2951ae62ce3557799b318399 Mon Sep 17 00:00:00 2001
From: Russell King <rmk+kernel@arm.linux.org.uk>
Date: Thu, 26 Nov 2009 12:31:15 +0000
Subject: ARM: use flush_kernel_dcache_area() for dmabounce

After copying data from the bounce buffer to the real buffer, use
flush_kernel_dcache_page() to ensure that data is written back in
manner coherent with future userspace mappings.

Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
---
 arch/arm/common/dmabounce.c | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

(limited to 'arch/arm')

diff --git a/arch/arm/common/dmabounce.c b/arch/arm/common/dmabounce.c
index 5a375e5fef2..bc90364a96c 100644
--- a/arch/arm/common/dmabounce.c
+++ b/arch/arm/common/dmabounce.c
@@ -308,15 +308,11 @@ static inline void unmap_single(struct device *dev, dma_addr_t dma_addr,
 			memcpy(ptr, buf->safe, size);
 
 			/*
-			 * DMA buffers must have the same cache properties
-			 * as if they were really used for DMA - which means
-			 * data must be written back to RAM.  Note that
-			 * we don't use dmac_flush_range() here for the
-			 * bidirectional case because we know the cache
-			 * lines will be coherent with the data written.
+			 * Since we may have written to a page cache page,
+			 * we need to ensure that the data will be coherent
+			 * with user mappings.
 			 */
-			dmac_clean_range(ptr, ptr + size);
-			outer_clean_range(__pa(ptr), __pa(ptr) + size);
+			__cpuc_flush_kernel_dcache_area(ptr, size);
 		}
 		free_safe_buffer(dev->archdata.dmabounce, buf);
 	}
-- 
cgit v1.2.3