aboutsummaryrefslogtreecommitdiff
path: root/include/linux
diff options
context:
space:
mode:
authorChristoph Lameter <clameter@sgi.com>2007-05-06 14:49:40 -0700
committerLinus Torvalds <torvalds@woody.linux-foundation.org>2007-05-07 12:12:53 -0700
commit6d7779538f765963ced45a3fa4bed7ba8d2c277d (patch)
tree07d47e6ff1ab30309004e2ba0674dcabd83945c1 /include/linux
parentd85f33855c303acfa87fa457157cef755b6087df (diff)
mm: optimize compound_head() by avoiding a shared page flag
The patch adds PageTail(page) and PageHead(page) to check if a page is the head or the tail of a compound page. This is done by masking the two bits describing the state of a compound page and then comparing them. So one comparision and a branch instead of two bit checks and two branches. Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/mm.h11
-rw-r--r--include/linux/page-flags.h37
2 files changed, 28 insertions, 20 deletions
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 8c149fa4491..695b90437bb 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -269,14 +269,7 @@ static inline int get_page_unless_zero(struct page *page)
static inline struct page *compound_head(struct page *page)
{
- /*
- * We could avoid the PageCompound(page) check if
- * we would not overload PageTail().
- *
- * This check has to be done in several performance critical
- * paths of the slab etc. IMHO PageTail deserves its own flag.
- */
- if (unlikely(PageCompound(page) && PageTail(page)))
+ if (unlikely(PageTail(page)))
return page->first_page;
return page;
}
@@ -327,7 +320,7 @@ static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
static inline int compound_order(struct page *page)
{
- if (!PageCompound(page) || PageTail(page))
+ if (!PageHead(page))
return 0;
return (unsigned long)page[1].lru.prev;
}
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index a1e14363494..a3c8b60a9c3 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -6,6 +6,7 @@
#define PAGE_FLAGS_H
#include <linux/types.h>
+#include <linux/mm_types.h>
/*
* Various page->flags bits:
@@ -94,12 +95,6 @@
/* PG_owner_priv_1 users should have descriptive aliases */
#define PG_checked PG_owner_priv_1 /* Used by some filesystems */
-/*
- * Marks tail portion of a compound page. We currently do not reclaim
- * compound pages so we can reuse a flag only used for reclaim here.
- */
-#define PG_tail PG_reclaim
-
#if (BITS_PER_LONG > 32)
/*
* 64-bit-only flags build down from bit 31
@@ -248,12 +243,32 @@ static inline void SetPageUptodate(struct page *page)
#define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags)
/*
- * Note: PG_tail is an alias of another page flag. The result of PageTail()
- * is only valid if PageCompound(page) is true.
+ * PG_reclaim is used in combination with PG_compound to mark the
+ * head and tail of a compound page
+ *
+ * PG_compound & PG_reclaim => Tail page
+ * PG_compound & ~PG_reclaim => Head page
*/
-#define PageTail(page) test_bit(PG_tail, &(page)->flags)
-#define __SetPageTail(page) __set_bit(PG_tail, &(page)->flags)
-#define __ClearPageTail(page) __clear_bit(PG_tail, &(page)->flags)
+
+#define PG_head_tail_mask ((1L << PG_compound) | (1L << PG_reclaim))
+
+#define PageTail(page) ((page->flags & PG_head_tail_mask) \
+ == PG_head_tail_mask)
+
+static inline void __SetPageTail(struct page *page)
+{
+ page->flags |= PG_head_tail_mask;
+}
+
+static inline void __ClearPageTail(struct page *page)
+{
+ page->flags &= ~PG_head_tail_mask;
+}
+
+#define PageHead(page) ((page->flags & PG_head_tail_mask) \
+ == (1L << PG_compound))
+#define __SetPageHead(page) __SetPageCompound(page)
+#define __ClearPageHead(page) __ClearPageCompound(page)
#ifdef CONFIG_SWAP
#define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags)