diff options
| -rw-r--r-- | arch/ia64/Kconfig | 5 | ||||
| -rw-r--r-- | include/linux/mmzone.h | 9 | ||||
| -rw-r--r-- | include/linux/vmstat.h | 129 | ||||
| -rw-r--r-- | mm/page_alloc.c | 2 | ||||
| -rw-r--r-- | mm/slab.c | 1 | ||||
| -rw-r--r-- | mm/vmstat.c | 218 |
6 files changed, 359 insertions, 5 deletions
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index b487e227a1f..47de9ee6bcd 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -70,6 +70,11 @@ config DMA_IS_DMA32 bool default y +config DMA_IS_NORMAL + bool + depends on IA64_SGI_SN2 + default y + choice prompt "System type" default IA64_GENERIC diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index d6120fa6911..543f9e41156 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -46,6 +46,9 @@ struct zone_padding { #define ZONE_PADDING(name) #endif +enum zone_stat_item { + NR_VM_ZONE_STAT_ITEMS }; + struct per_cpu_pages { int count; /* number of pages in the list */ int high; /* high watermark, emptying needed */ @@ -55,6 +58,10 @@ struct per_cpu_pages { struct per_cpu_pageset { struct per_cpu_pages pcp[2]; /* 0: hot. 1: cold */ +#ifdef CONFIG_SMP + s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS]; +#endif + #ifdef CONFIG_NUMA unsigned long numa_hit; /* allocated in intended node */ unsigned long numa_miss; /* allocated in non intended node */ @@ -165,6 +172,8 @@ struct zone { /* A count of how many reclaimers are scanning this zone */ atomic_t reclaim_in_progress; + /* Zone statistics */ + atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; /* * timestamp (in jiffies) of the last zone reclaim that did not * result in freeing of pages. This is used to avoid repeated scans diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index 3ca0c1989fc..3fd5c11e544 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -3,6 +3,9 @@ #include <linux/types.h> #include <linux/percpu.h> +#include <linux/config.h> +#include <linux/mmzone.h> +#include <asm/atomic.h> /* * Global page accounting. One instance per CPU. Only unsigned longs are @@ -134,5 +137,129 @@ extern void __mod_page_state_offset(unsigned long offset, unsigned long delta); DECLARE_PER_CPU(struct page_state, page_states); -#endif /* _LINUX_VMSTAT_H */ +/* + * Zone based page accounting with per cpu differentials. + */ +extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; + +static inline void zone_page_state_add(long x, struct zone *zone, + enum zone_stat_item item) +{ + atomic_long_add(x, &zone->vm_stat[item]); + atomic_long_add(x, &vm_stat[item]); +} + +static inline unsigned long global_page_state(enum zone_stat_item item) +{ + long x = atomic_long_read(&vm_stat[item]); +#ifdef CONFIG_SMP + if (x < 0) + x = 0; +#endif + return x; +} + +static inline unsigned long zone_page_state(struct zone *zone, + enum zone_stat_item item) +{ + long x = atomic_long_read(&zone->vm_stat[item]); +#ifdef CONFIG_SMP + if (x < 0) + x = 0; +#endif + return x; +} + +#ifdef CONFIG_NUMA +/* + * Determine the per node value of a stat item. This function + * is called frequently in a NUMA machine, so try to be as + * frugal as possible. + */ +static inline unsigned long node_page_state(int node, + enum zone_stat_item item) +{ + struct zone *zones = NODE_DATA(node)->node_zones; + + return +#ifndef CONFIG_DMA_IS_NORMAL +#if !defined(CONFIG_DMA_IS_DMA32) && BITS_PER_LONG >= 64 + zone_page_state(&zones[ZONE_DMA32], item) + +#endif + zone_page_state(&zones[ZONE_NORMAL], item) + +#endif +#ifdef CONFIG_HIGHMEM + zone_page_state(&zones[ZONE_HIGHMEM], item) + +#endif + zone_page_state(&zones[ZONE_DMA], item); +} +#else +#define node_page_state(node, item) global_page_state(item) +#endif + +#define __add_zone_page_state(__z, __i, __d) \ + __mod_zone_page_state(__z, __i, __d) +#define __sub_zone_page_state(__z, __i, __d) \ + __mod_zone_page_state(__z, __i,-(__d)) + +#define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d) +#define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d)) + +static inline void zap_zone_vm_stats(struct zone *zone) +{ + memset(zone->vm_stat, 0, sizeof(zone->vm_stat)); +} + +#ifdef CONFIG_SMP +void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int); +void __inc_zone_page_state(struct page *, enum zone_stat_item); +void __dec_zone_page_state(struct page *, enum zone_stat_item); +void mod_zone_page_state(struct zone *, enum zone_stat_item, int); +void inc_zone_page_state(struct page *, enum zone_stat_item); +void dec_zone_page_state(struct page *, enum zone_stat_item); + +extern void inc_zone_state(struct zone *, enum zone_stat_item); + +void refresh_cpu_vm_stats(int); +void refresh_vm_stats(void); + +#else /* CONFIG_SMP */ + +/* + * We do not maintain differentials in a single processor configuration. + * The functions directly modify the zone and global counters. + */ +static inline void __mod_zone_page_state(struct zone *zone, + enum zone_stat_item item, int delta) +{ + zone_page_state_add(delta, zone, item); +} + +static inline void __inc_zone_page_state(struct page *page, + enum zone_stat_item item) +{ + atomic_long_inc(&page_zone(page)->vm_stat[item]); + atomic_long_inc(&vm_stat[item]); +} + +static inline void __dec_zone_page_state(struct page *page, + enum zone_stat_item item) +{ + atomic_long_dec(&page_zone(page)->vm_stat[item]); + atomic_long_dec(&vm_stat[item]); +} + +/* + * We only use atomic operations to update counters. So there is no need to + * disable interrupts. + */ +#define inc_zone_page_state __inc_zone_page_state +#define dec_zone_page_state __dec_zone_page_state +#define mod_zone_page_state __mod_zone_page_state + +static inline void refresh_cpu_vm_stats(int cpu) { } +static inline void refresh_vm_stats(void) { } +#endif + +#endif /* _LINUX_VMSTAT_H */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 87dc1297fe3..3a877fecc30 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2045,6 +2045,7 @@ static void __meminit free_area_init_core(struct pglist_data *pgdat, zone->nr_scan_inactive = 0; zone->nr_active = 0; zone->nr_inactive = 0; + zap_zone_vm_stats(zone); atomic_set(&zone->reclaim_in_progress, 0); if (!size) continue; @@ -2147,6 +2148,7 @@ static int page_alloc_cpu_notify(struct notifier_block *self, } local_irq_enable(); + refresh_cpu_vm_stats(cpu); } return NOTIFY_OK; } diff --git a/mm/slab.c b/mm/slab.c index 233e39d14ca..0c33820038c 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3763,6 +3763,7 @@ next: check_irq_on(); mutex_unlock(&cache_chain_mutex); next_reap_node(); + refresh_cpu_vm_stats(smp_processor_id()); /* Set up the next iteration */ schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC); } diff --git a/mm/vmstat.c b/mm/vmstat.c index ad456202ff1..210f9bbbb04 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -3,10 +3,15 @@ * * Manages VM statistics * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds + * + * zoned VM statistics + * Copyright (C) 2006 Silicon Graphics, Inc., + * Christoph Lameter <christoph@lameter.com> */ #include <linux/config.h> #include <linux/mm.h> +#include <linux/module.h> /* * Accumulate the page_state information across all CPUs. @@ -143,6 +148,197 @@ void get_zone_counts(unsigned long *active, } } +/* + * Manage combined zone based / global counters + * + * vm_stat contains the global counters + */ +atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; +EXPORT_SYMBOL(vm_stat); + +#ifdef CONFIG_SMP + +#define STAT_THRESHOLD 32 + +/* + * Determine pointer to currently valid differential byte given a zone and + * the item number. + * + * Preemption must be off + */ +static inline s8 *diff_pointer(struct zone *zone, enum zone_stat_item item) +{ + return &zone_pcp(zone, smp_processor_id())->vm_stat_diff[item]; +} + +/* + * For use when we know that interrupts are disabled. + */ +void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, + int delta) +{ + s8 *p; + long x; + + p = diff_pointer(zone, item); + x = delta + *p; + + if (unlikely(x > STAT_THRESHOLD || x < -STAT_THRESHOLD)) { + zone_page_state_add(x, zone, item); + x = 0; + } + + *p = x; +} +EXPORT_SYMBOL(__mod_zone_page_state); + +/* + * For an unknown interrupt state + */ +void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, + int delta) +{ + unsigned long flags; + + local_irq_save(flags); + __mod_zone_page_state(zone, item, delta); + local_irq_restore(flags); +} +EXPORT_SYMBOL(mod_zone_page_state); + +/* + * Optimized increment and decrement functions. + * + * These are only for a single page and therefore can take a struct page * + * argument instead of struct zone *. This allows the inclusion of the code + * generated for page_zone(page) into the optimized functions. + * + * No overflow check is necessary and therefore the differential can be + * incremented or decremented in place which may allow the compilers to + * generate better code. + * + * The increment or decrement is known and therefore one boundary check can + * be omitted. + * + * Some processors have inc/dec instructions that are atomic vs an interrupt. + * However, the code must first determine the differential location in a zone + * based on the processor number and then inc/dec the counter. There is no + * guarantee without disabling preemption that the processor will not change + * in between and therefore the atomicity vs. interrupt cannot be exploited + * in a useful way here. + */ +void __inc_zone_page_state(struct page *page, enum zone_stat_item item) +{ + struct zone *zone = page_zone(page); + s8 *p = diff_pointer(zone, item); + + (*p)++; + + if (unlikely(*p > STAT_THRESHOLD)) { + zone_page_state_add(*p, zone, item); + *p = 0; + } +} +EXPORT_SYMBOL(__inc_zone_page_state); + +void __dec_zone_page_state(struct page *page, enum zone_stat_item item) +{ + struct zone *zone = page_zone(page); + s8 *p = diff_pointer(zone, item); + + (*p)--; + + if (unlikely(*p < -STAT_THRESHOLD)) { + zone_page_state_add(*p, zone, item); + *p = 0; + } +} +EXPORT_SYMBOL(__dec_zone_page_state); + +void inc_zone_page_state(struct page *page, enum zone_stat_item item) +{ + unsigned long flags; + struct zone *zone; + s8 *p; + + zone = page_zone(page); + local_irq_save(flags); + p = diff_pointer(zone, item); + + (*p)++; + + if (unlikely(*p > STAT_THRESHOLD)) { + zone_page_state_add(*p, zone, item); + *p = 0; + } + local_irq_restore(flags); +} +EXPORT_SYMBOL(inc_zone_page_state); + +void dec_zone_page_state(struct page *page, enum zone_stat_item item) +{ + unsigned long flags; + struct zone *zone; + s8 *p; + + zone = page_zone(page); + local_irq_save(flags); + p = diff_pointer(zone, item); + + (*p)--; + + if (unlikely(*p < -STAT_THRESHOLD)) { + zone_page_state_add(*p, zone, item); + *p = 0; + } + local_irq_restore(flags); +} +EXPORT_SYMBOL(dec_zone_page_state); + +/* + * Update the zone counters for one cpu. + */ +void refresh_cpu_vm_stats(int cpu) +{ + struct zone *zone; + int i; + unsigned long flags; + + for_each_zone(zone) { + struct per_cpu_pageset *pcp; + + pcp = zone_pcp(zone, cpu); + + for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) + if (pcp->vm_stat_diff[i]) { + local_irq_save(flags); + zone_page_state_add(pcp->vm_stat_diff[i], + zone, i); + pcp->vm_stat_diff[i] = 0; + local_irq_restore(flags); + } + } +} + +static void __refresh_cpu_vm_stats(void *dummy) +{ + refresh_cpu_vm_stats(smp_processor_id()); +} + +/* + * Consolidate all counters. + * + * Note that the result is less inaccurate but still inaccurate + * if concurrent processes are allowed to run. + */ +void refresh_vm_stats(void) +{ + on_each_cpu(__refresh_cpu_vm_stats, NULL, 0, 1); +} +EXPORT_SYMBOL(refresh_vm_stats); + +#endif + #ifdef CONFIG_PROC_FS #include <linux/seq_file.h> @@ -204,6 +400,9 @@ struct seq_operations fragmentation_op = { }; static char *vmstat_text[] = { + /* Zoned VM counters */ + + /* Page state */ "nr_dirty", "nr_writeback", "nr_unstable", @@ -297,6 +496,11 @@ static int zoneinfo_show(struct seq_file *m, void *arg) zone->nr_scan_active, zone->nr_scan_inactive, zone->spanned_pages, zone->present_pages); + + for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) + seq_printf(m, "\n %-12s %lu", vmstat_text[i], + zone_page_state(zone, i)); + seq_printf(m, "\n protection: (%lu", zone->lowmem_reserve[0]); @@ -368,19 +572,25 @@ struct seq_operations zoneinfo_op = { static void *vmstat_start(struct seq_file *m, loff_t *pos) { + unsigned long *v; struct page_state *ps; + int i; if (*pos >= ARRAY_SIZE(vmstat_text)) return NULL; - ps = kmalloc(sizeof(*ps), GFP_KERNEL); - m->private = ps; - if (!ps) + v = kmalloc(NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long) + + sizeof(*ps), GFP_KERNEL); + m->private = v; + if (!v) return ERR_PTR(-ENOMEM); + for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) + v[i] = global_page_state(i); + ps = (struct page_state *)(v + NR_VM_ZONE_STAT_ITEMS); get_full_page_state(ps); ps->pgpgin /= 2; /* sectors -> kbytes */ ps->pgpgout /= 2; - return (unsigned long *)ps + *pos; + return v + *pos; } static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) |