From fe5bb6b00c3a9374841d651e01694fe4190a677e Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 2 Apr 2009 16:59:04 -0700 Subject: sgi-gru: misc GRU cleanup Misc trivial GRU drivers fixes: - fix long lines - eliminate extra whitespace - eliminate compiler warning - better validation of invalidate user parameters - bug fix for GRU TLB flush (not the cpu TLB flush) These changes are all internal to the SGI GRU driver and have no effect on the base kernel. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/gru_instructions.h | 22 ++++++++++++---------- drivers/misc/sgi-gru/grufault.c | 19 ++++++++++++------- drivers/misc/sgi-gru/grufile.c | 25 +++++++++++++++---------- drivers/misc/sgi-gru/gruhandles.h | 2 +- drivers/misc/sgi-gru/grukservices.c | 4 ++-- drivers/misc/sgi-gru/grumain.c | 6 +++--- drivers/misc/sgi-gru/grutables.h | 15 ++++++++------- drivers/misc/sgi-gru/grutlbpurge.c | 5 ++--- 8 files changed, 55 insertions(+), 43 deletions(-) (limited to 'drivers/misc/sgi-gru') diff --git a/drivers/misc/sgi-gru/gru_instructions.h b/drivers/misc/sgi-gru/gru_instructions.h index 48762e7b98b..3fde33c1e8f 100644 --- a/drivers/misc/sgi-gru/gru_instructions.h +++ b/drivers/misc/sgi-gru/gru_instructions.h @@ -19,8 +19,11 @@ #ifndef __GRU_INSTRUCTIONS_H__ #define __GRU_INSTRUCTIONS_H__ -#define gru_flush_cache_hook(p) -#define gru_emulator_wait_hook(p, w) +extern int gru_check_status_proc(void *cb); +extern int gru_wait_proc(void *cb); +extern void gru_wait_abort_proc(void *cb); + + /* * Architecture dependent functions @@ -29,16 +32,16 @@ #if defined(CONFIG_IA64) #include #include -#define __flush_cache(p) ia64_fc(p) +#define __flush_cache(p) ia64_fc((unsigned long)p) /* Use volatile on IA64 to ensure ordering via st4.rel */ -#define gru_ordered_store_int(p,v) \ +#define gru_ordered_store_int(p, v) \ do { \ barrier(); \ *((volatile int *)(p)) = v; /* force st.rel */ \ } while (0) #elif defined(CONFIG_X86_64) #define __flush_cache(p) clflush(p) -#define gru_ordered_store_int(p,v) \ +#define gru_ordered_store_int(p, v) \ do { \ barrier(); \ *(int *)p = v; \ @@ -558,20 +561,19 @@ extern int gru_get_cb_exception_detail(void *cb, #define GRU_EXC_STR_SIZE 256 -extern int gru_check_status_proc(void *cb); -extern int gru_wait_proc(void *cb); -extern void gru_wait_abort_proc(void *cb); /* * Control block definition for checking status */ struct gru_control_block_status { unsigned int icmd :1; - unsigned int unused1 :31; + unsigned int ima :3; + unsigned int reserved0 :4; + unsigned int unused1 :24; unsigned int unused2 :24; unsigned int istatus :2; unsigned int isubstatus :4; - unsigned int inused3 :2; + unsigned int unused3 :2; }; /* Get CB status */ diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 3ee698ad859..f85d2730678 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -368,6 +368,7 @@ failupm: failfmm: /* FMM state on UPM call */ + gru_flush_cache(tfh); STAT(tlb_dropin_fail_fmm); gru_dbg(grudev, "FAILED fmm tfh: 0x%p, state %d\n", tfh, tfh->state); return 0; @@ -497,10 +498,8 @@ int gru_handle_user_call_os(unsigned long cb) if (!gts) return -EINVAL; - if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) { - ret = -EINVAL; + if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) goto exit; - } /* * If force_unload is set, the UPM TLB fault is phony. The task @@ -508,6 +507,10 @@ int gru_handle_user_call_os(unsigned long cb) * unload the context. The task will page fault and assign a new * context. */ + if (gts->ts_tgid_owner == current->tgid && gts->ts_blade >= 0 && + gts->ts_blade != uv_numa_blade_id()) + gts->ts_force_unload = 1; + ret = -EAGAIN; cbrnum = thread_cbr_number(gts, ucbnum); if (gts->ts_force_unload) { @@ -541,11 +544,13 @@ int gru_get_exception_detail(unsigned long arg) if (!gts) return -EINVAL; - if (gts->ts_gru) { - ucbnum = get_cb_number((void *)excdet.cb); + ucbnum = get_cb_number((void *)excdet.cb); + if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) { + ret = -EINVAL; + } else if (gts->ts_gru) { cbrnum = thread_cbr_number(gts, ucbnum); cbe = get_cbe_by_index(gts->ts_gru, cbrnum); - prefetchw(cbe); /* Harmless on hardware, required for emulator */ + prefetchw(cbe);/* Harmless on hardware, required for emulator */ excdet.opc = cbe->opccpy; excdet.exopc = cbe->exopccpy; excdet.ecause = cbe->ecause; @@ -609,7 +614,7 @@ int gru_user_flush_tlb(unsigned long arg) if (!gts) return -EINVAL; - gru_flush_tlb_range(gts->ts_gms, req.vaddr, req.vaddr + req.len); + gru_flush_tlb_range(gts->ts_gms, req.vaddr, req.len); gru_unlock_gts(gts); return 0; diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c index c67e4e8bd62..15292e5f74a 100644 --- a/drivers/misc/sgi-gru/grufile.c +++ b/drivers/misc/sgi-gru/grufile.c @@ -45,7 +45,8 @@ #include struct gru_blade_state *gru_base[GRU_MAX_BLADES] __read_mostly; -unsigned long gru_start_paddr, gru_end_paddr __read_mostly; +unsigned long gru_start_paddr __read_mostly; +unsigned long gru_end_paddr __read_mostly; struct gru_stats_s gru_stats; /* Guaranteed user available resources on each node */ @@ -101,7 +102,7 @@ static int gru_file_mmap(struct file *file, struct vm_area_struct *vma) return -EPERM; if (vma->vm_start & (GRU_GSEG_PAGESIZE - 1) || - vma->vm_end & (GRU_GSEG_PAGESIZE - 1)) + vma->vm_end & (GRU_GSEG_PAGESIZE - 1)) return -EINVAL; vma->vm_flags |= @@ -295,7 +296,7 @@ static int gru_init_tables(unsigned long gru_base_paddr, void *gru_base_vaddr) for_each_online_node(nid) { bid = uv_node_to_blade_id(nid); pnode = uv_node_to_pnode(nid); - if (gru_base[bid]) + if (bid < 0 || gru_base[bid]) continue; page = alloc_pages_node(nid, GFP_KERNEL, order); if (!page) @@ -308,11 +309,11 @@ static int gru_init_tables(unsigned long gru_base_paddr, void *gru_base_vaddr) dsrbytes = 0; cbrs = 0; for (gru = gru_base[bid]->bs_grus, chip = 0; - chip < GRU_CHIPLETS_PER_BLADE; + chip < GRU_CHIPLETS_PER_BLADE; chip++, gru++) { paddr = gru_chiplet_paddr(gru_base_paddr, pnode, chip); vaddr = gru_chiplet_vaddr(gru_base_vaddr, pnode, chip); - gru_init_chiplet(gru, paddr, vaddr, bid, nid, chip); + gru_init_chiplet(gru, paddr, vaddr, nid, bid, chip); n = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE; cbrs = max(cbrs, n); n = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES; @@ -370,26 +371,26 @@ static int __init gru_init(void) void *gru_start_vaddr; if (!is_uv_system()) - return 0; + return -ENODEV; #if defined CONFIG_IA64 gru_start_paddr = 0xd000000000UL; /* ZZZZZZZZZZZZZZZZZZZ fixme */ #else gru_start_paddr = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR) & 0x7fffffffffffUL; - #endif gru_start_vaddr = __va(gru_start_paddr); - gru_end_paddr = gru_start_paddr + MAX_NUMNODES * GRU_SIZE; + gru_end_paddr = gru_start_paddr + GRU_MAX_BLADES * GRU_SIZE; printk(KERN_INFO "GRU space: 0x%lx - 0x%lx\n", gru_start_paddr, gru_end_paddr); irq = get_base_irq(); for (chip = 0; chip < GRU_CHIPLETS_PER_BLADE; chip++) { ret = request_irq(irq + chip, gru_intr, 0, id, NULL); - /* TODO: fix irq handling on x86. For now ignore failures because + /* TODO: fix irq handling on x86. For now ignore failure because * interrupts are not required & not yet fully supported */ if (ret) { - printk("!!!WARNING: GRU ignoring request failure!!!\n"); + printk(KERN_WARNING + "!!!WARNING: GRU ignoring request failure!!!\n"); ret = 0; } if (ret) { @@ -469,7 +470,11 @@ struct vm_operations_struct gru_vm_ops = { .fault = gru_fault, }; +#ifndef MODULE fs_initcall(gru_init); +#else +module_init(gru_init); +#endif module_exit(gru_exit); module_param(gru_options, ulong, 0644); diff --git a/drivers/misc/sgi-gru/gruhandles.h b/drivers/misc/sgi-gru/gruhandles.h index b63018d60fe..fb72a52a34a 100644 --- a/drivers/misc/sgi-gru/gruhandles.h +++ b/drivers/misc/sgi-gru/gruhandles.h @@ -489,7 +489,7 @@ enum gru_cbr_state { * 64m 26 8 * ... */ -#define GRU_PAGESIZE(sh) ((((sh) > 20 ? (sh) + 2: (sh)) >> 1) - 6) +#define GRU_PAGESIZE(sh) ((((sh) > 20 ? (sh) + 2 : (sh)) >> 1) - 6) #define GRU_SIZEAVAIL(sh) (1UL << GRU_PAGESIZE(sh)) /* minimum TLB purge count to ensure a full purge */ diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c index 880c55dfb66..3e36b7b6e1c 100644 --- a/drivers/misc/sgi-gru/grukservices.c +++ b/drivers/misc/sgi-gru/grukservices.c @@ -122,7 +122,7 @@ int gru_get_cb_exception_detail(void *cb, struct gru_control_block_extended *cbe; cbe = get_cbe(GRUBASE(cb), get_cb_number(cb)); - prefetchw(cbe); /* Harmless on hardware, required for emulator */ + prefetchw(cbe); /* Harmless on hardware, required for emulator */ excdet->opc = cbe->opccpy; excdet->exopc = cbe->exopccpy; excdet->ecause = cbe->ecause; @@ -437,7 +437,7 @@ static int send_message_failure(void *cb, break; case CBSS_PUT_NACKED: STAT(mesq_send_put_nacked); - m =mq + (gru_get_amo_value_head(cb) << 6); + m = mq + (gru_get_amo_value_head(cb) << 6); gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA); if (gru_wait(cb) == CBS_IDLE) ret = MQE_OK; diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index 3d2fc216bae..1ce32bcd025 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -432,8 +432,8 @@ static inline long gru_copy_handle(void *d, void *s) return GRU_HANDLE_BYTES; } -static void gru_prefetch_context(void *gseg, void *cb, void *cbe, unsigned long cbrmap, - unsigned long length) +static void gru_prefetch_context(void *gseg, void *cb, void *cbe, + unsigned long cbrmap, unsigned long length) { int i, scr; @@ -773,8 +773,8 @@ int gru_fault(struct vm_area_struct *vma, struct vm_fault *vmf) return VM_FAULT_SIGBUS; again: - preempt_disable(); mutex_lock(>s->ts_ctxlock); + preempt_disable(); if (gts->ts_gru) { if (gts->ts_gru->gs_blade_id != uv_numa_blade_id()) { STAT(migrated_nopfn_unload); diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index a78f70deeb5..db3fe08bf79 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -278,13 +278,12 @@ struct gru_stats_s { /* Generate a GRU asid value from a GRU base asid & a virtual address. */ #if defined CONFIG_IA64 #define VADDR_HI_BIT 64 -#define GRUREGION(addr) ((addr) >> (VADDR_HI_BIT - 3) & 3) #elif defined CONFIG_X86_64 #define VADDR_HI_BIT 48 -#define GRUREGION(addr) (0) /* ZZZ could do better */ #else #error "Unsupported architecture" #endif +#define GRUREGION(addr) ((addr) >> (VADDR_HI_BIT - 3) & 3) #define GRUASID(asid, addr) ((asid) + GRUREGION(addr)) /*------------------------------------------------------------------------------ @@ -297,12 +296,12 @@ struct gru_state; * This structure is pointed to from the mmstruct via the notifier pointer. * There is one of these per address space. */ -struct gru_mm_tracker { - unsigned int mt_asid_gen; /* ASID wrap count */ - int mt_asid; /* current base ASID for gru */ - unsigned short mt_ctxbitmap; /* bitmap of contexts using +struct gru_mm_tracker { /* pack to reduce size */ + unsigned int mt_asid_gen:24; /* ASID wrap count */ + unsigned int mt_asid:24; /* current base ASID for gru */ + unsigned short mt_ctxbitmap:16;/* bitmap of contexts using asid */ -}; +} __attribute__ ((packed)); struct gru_mm_struct { struct mmu_notifier ms_notifier; @@ -359,6 +358,8 @@ struct gru_thread_state { required for contest */ unsigned char ts_cbr_au_count;/* Number of CBR resources required for contest */ + char ts_blade; /* If >= 0, migrate context if + ref from diferent blade */ char ts_force_unload;/* force context to be unloaded after migration */ char ts_cbr_idx[GRU_CBR_AU];/* CBR numbers of each diff --git a/drivers/misc/sgi-gru/grutlbpurge.c b/drivers/misc/sgi-gru/grutlbpurge.c index c84496a7769..4e438e4dd2a 100644 --- a/drivers/misc/sgi-gru/grutlbpurge.c +++ b/drivers/misc/sgi-gru/grutlbpurge.c @@ -187,7 +187,7 @@ void gru_flush_tlb_range(struct gru_mm_struct *gms, unsigned long start, " FLUSH gruid %d, asid 0x%x, num %ld, cbmap 0x%x\n", gid, asid, num, asids->mt_ctxbitmap); tgh = get_lock_tgh_handle(gru); - tgh_invalidate(tgh, start, 0, asid, grupagesize, 0, + tgh_invalidate(tgh, start, ~0, asid, grupagesize, 0, num - 1, asids->mt_ctxbitmap); get_unlock_tgh_handle(tgh); } else { @@ -212,9 +212,8 @@ void gru_flush_all_tlb(struct gru_state *gru) gru_dbg(grudev, "gru %p, gid %d\n", gru, gru->gs_gid); tgh = get_lock_tgh_handle(gru); - tgh_invalidate(tgh, 0, ~0, 0, 1, 1, GRUMAXINVAL - 1, 0); + tgh_invalidate(tgh, 0, ~0, 0, 1, 1, GRUMAXINVAL - 1, 0xffff); get_unlock_tgh_handle(tgh); - preempt_enable(); } /* -- cgit v1.2.3 From 438846043af39fef286f68b197d42bd75f49898e Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 2 Apr 2009 16:59:05 -0700 Subject: sgi-gru: improvements to GRU debug messages & statistics Improvements to GRU debug messages & statistics. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 5 +++- drivers/misc/sgi-gru/grufile.c | 2 +- drivers/misc/sgi-gru/grukservices.c | 4 ++-- drivers/misc/sgi-gru/grumain.c | 48 ++++++++++++++++++++----------------- drivers/misc/sgi-gru/gruprocfs.c | 2 ++ drivers/misc/sgi-gru/grutables.h | 2 ++ drivers/misc/sgi-gru/grutlbpurge.c | 2 +- 7 files changed, 38 insertions(+), 27 deletions(-) (limited to 'drivers/misc/sgi-gru') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index f85d2730678..61aa80a8411 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -449,6 +449,7 @@ irqreturn_t gru_intr(int irq, void *dev_id) up_read(>s->ts_mm->mmap_sem); } else { tfh_user_polling_mode(tfh); + STAT(intr_mm_lock_failed); } } return IRQ_HANDLED; @@ -508,8 +509,10 @@ int gru_handle_user_call_os(unsigned long cb) * context. */ if (gts->ts_tgid_owner == current->tgid && gts->ts_blade >= 0 && - gts->ts_blade != uv_numa_blade_id()) + gts->ts_blade != uv_numa_blade_id()) { + STAT(call_os_offnode_reference); gts->ts_force_unload = 1; + } ret = -EAGAIN; cbrnum = thread_cbr_number(gts, ucbnum); diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c index 15292e5f74a..812678d8b47 100644 --- a/drivers/misc/sgi-gru/grufile.c +++ b/drivers/misc/sgi-gru/grufile.c @@ -275,7 +275,7 @@ static void gru_init_chiplet(struct gru_state *gru, unsigned long paddr, gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1; gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1; gru_tgh_flush_init(gru); - gru_dbg(grudev, "bid %d, nid %d, gru %x, vaddr %p (0x%lx)\n", + gru_dbg(grudev, "bid %d, nid %d, gid %d, vaddr %p (0x%lx)\n", bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr, gru->gs_gru_base_paddr); gru_kservices_init(gru); diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c index 3e36b7b6e1c..247635d6d6e 100644 --- a/drivers/misc/sgi-gru/grukservices.c +++ b/drivers/misc/sgi-gru/grukservices.c @@ -663,12 +663,12 @@ int gru_kservices_init(struct gru_state *gru) err = cch_allocate(cch, 0, cbr_map, dsr_map); if (err) { gru_dbg(grudev, - "Unable to allocate kernel CCH: gru %d, err %d\n", + "Unable to allocate kernel CCH: gid %d, err %d\n", gru->gs_gid, err); BUG(); } if (cch_start(cch)) { - gru_dbg(grudev, "Unable to start kernel CCH: gru %d, err %d\n", + gru_dbg(grudev, "Unable to start kernel CCH: gid %d, err %d\n", gru->gs_gid, err); BUG(); } diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index 1ce32bcd025..de60458f6d1 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -76,7 +76,7 @@ int gru_cpu_fault_map_id(void) /* Hit the asid limit. Start over */ static int gru_wrap_asid(struct gru_state *gru) { - gru_dbg(grudev, "gru %p\n", gru); + gru_dbg(grudev, "gid %d\n", gru->gs_gid); STAT(asid_wrap); gru->gs_asid_gen++; gru_flush_all_tlb(gru); @@ -88,7 +88,7 @@ static int gru_reset_asid_limit(struct gru_state *gru, int asid) { int i, gid, inuse_asid, limit; - gru_dbg(grudev, "gru %p, asid 0x%x\n", gru, asid); + gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid); STAT(asid_next); limit = MAX_ASID; if (asid >= limit) @@ -99,8 +99,9 @@ again: if (!gru->gs_gts[i]) continue; inuse_asid = gru->gs_gts[i]->ts_gms->ms_asids[gid].mt_asid; - gru_dbg(grudev, "gru %p, inuse_asid 0x%x, cxtnum %d, gts %p\n", - gru, inuse_asid, i, gru->gs_gts[i]); + gru_dbg(grudev, "gid %d, gts %p, gms %p, inuse 0x%x, cxt %d\n", + gru->gs_gid, gru->gs_gts[i], gru->gs_gts[i]->ts_gms, + inuse_asid, i); if (inuse_asid == asid) { asid += ASID_INC; if (asid >= limit) { @@ -120,8 +121,8 @@ again: } gru->gs_asid_limit = limit; gru->gs_asid = asid; - gru_dbg(grudev, "gru %p, new asid 0x%x, new_limit 0x%x\n", gru, asid, - limit); + gru_dbg(grudev, "gid %d, new asid 0x%x, new_limit 0x%x\n", gru->gs_gid, + asid, limit); return asid; } @@ -137,7 +138,7 @@ static int gru_assign_asid(struct gru_state *gru) asid = gru_reset_asid_limit(gru, asid); spin_unlock(&gru->gs_asid_lock); - gru_dbg(grudev, "gru %p, asid 0x%x\n", gru, asid); + gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid); return asid; } @@ -215,11 +216,12 @@ static int check_gru_resources(struct gru_state *gru, int cbr_au_count, * TLB manangment requires tracking all GRU chiplets that have loaded a GSEG * context. */ -static int gru_load_mm_tracker(struct gru_state *gru, struct gru_mm_struct *gms, - int ctxnum) +static int gru_load_mm_tracker(struct gru_state *gru, + struct gru_thread_state *gts) { + struct gru_mm_struct *gms = gts->ts_gms; struct gru_mm_tracker *asids = &gms->ms_asids[gru->gs_gid]; - unsigned short ctxbitmap = (1 << ctxnum); + unsigned short ctxbitmap = (1 << gts->ts_ctxnum); int asid; spin_lock(&gms->ms_asid_lock); @@ -241,24 +243,26 @@ static int gru_load_mm_tracker(struct gru_state *gru, struct gru_mm_struct *gms, spin_unlock(&gms->ms_asid_lock); gru_dbg(grudev, - "gru %x, gms %p, ctxnum 0x%d, asid 0x%x, asidmap 0x%lx\n", - gru->gs_gid, gms, ctxnum, asid, gms->ms_asidmap[0]); + "gid %d, gts %p, gms %p, ctxnum %d, asid 0x%x, asidmap 0x%lx\n", + gru->gs_gid, gts, gms, gts->ts_ctxnum, asid, + gms->ms_asidmap[0]); return asid; } static void gru_unload_mm_tracker(struct gru_state *gru, - struct gru_mm_struct *gms, int ctxnum) + struct gru_thread_state *gts) { + struct gru_mm_struct *gms = gts->ts_gms; struct gru_mm_tracker *asids; unsigned short ctxbitmap; asids = &gms->ms_asids[gru->gs_gid]; - ctxbitmap = (1 << ctxnum); + ctxbitmap = (1 << gts->ts_ctxnum); spin_lock(&gms->ms_asid_lock); BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap); asids->mt_ctxbitmap ^= ctxbitmap; - gru_dbg(grudev, "gru %x, gms %p, ctxnum 0x%d, asidmap 0x%lx\n", - gru->gs_gid, gms, ctxnum, gms->ms_asidmap[0]); + gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n", + gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]); spin_unlock(&gms->ms_asid_lock); } @@ -399,7 +403,7 @@ static void gru_free_gru_context(struct gru_thread_state *gts) struct gru_state *gru; gru = gts->ts_gru; - gru_dbg(grudev, "gts %p, gru %p\n", gts, gru); + gru_dbg(grudev, "gts %p, gid %d\n", gts, gru->gs_gid); spin_lock(&gru->gs_lock); gru->gs_gts[gts->ts_ctxnum] = NULL; @@ -500,12 +504,12 @@ void gru_unload_context(struct gru_thread_state *gts, int savestate) zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE); cch = get_cch(gru->gs_gru_base_vaddr, ctxnum); + gru_dbg(grudev, "gts %p\n", gts); lock_cch_handle(cch); if (cch_interrupt_sync(cch)) BUG(); - gru_dbg(grudev, "gts %p\n", gts); - gru_unload_mm_tracker(gru, gts->ts_gms, gts->ts_ctxnum); + gru_unload_mm_tracker(gru, gts); if (savestate) gru_unload_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr, ctxnum, gts->ts_cbr_map, @@ -534,7 +538,7 @@ static void gru_load_context(struct gru_thread_state *gts) cch = get_cch(gru->gs_gru_base_vaddr, ctxnum); lock_cch_handle(cch); - asid = gru_load_mm_tracker(gru, gts->ts_gms, gts->ts_ctxnum); + asid = gru_load_mm_tracker(gru, gts); cch->tfm_fault_bit_enable = (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR); @@ -688,7 +692,7 @@ static void gru_steal_context(struct gru_thread_state *gts) STAT(steal_context_failed); } gru_dbg(grudev, - "stole gru %x, ctxnum %d from gts %p. Need cb %d, ds %d;" + "stole gid %d, ctxnum %d from gts %p. Need cb %d, ds %d;" " avail cb %ld, ds %ld\n", gru->gs_gid, ctxnum, ngts, cbr, dsr, hweight64(gru->gs_cbr_map), hweight64(gru->gs_dsr_map)); @@ -737,7 +741,7 @@ again: STAT(assign_context); gru_dbg(grudev, - "gseg %p, gts %p, gru %x, ctx %d, cbr %d, dsr %d\n", + "gseg %p, gts %p, gid %d, ctx %d, cbr %d, dsr %d\n", gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), gts, gts->ts_gru->gs_gid, gts->ts_ctxnum, gts->ts_cbr_au_count, gts->ts_dsr_au_count); diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c index 73b0ca061bb..6d63a8d70f8 100644 --- a/drivers/misc/sgi-gru/gruprocfs.c +++ b/drivers/misc/sgi-gru/gruprocfs.c @@ -62,7 +62,9 @@ static int statistics_show(struct seq_file *s, void *p) printstat(s, asid_wrap); printstat(s, asid_reuse); printstat(s, intr); + printstat(s, intr_mm_lock_failed); printstat(s, call_os); + printstat(s, call_os_offnode_reference); printstat(s, call_os_check_for_bug); printstat(s, call_os_wait_queue); printstat(s, user_flush_tlb); diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index db3fe08bf79..cc0fc734405 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -184,7 +184,9 @@ struct gru_stats_s { atomic_long_t asid_wrap; atomic_long_t asid_reuse; atomic_long_t intr; + atomic_long_t intr_mm_lock_failed; atomic_long_t call_os; + atomic_long_t call_os_offnode_reference; atomic_long_t call_os_check_for_bug; atomic_long_t call_os_wait_queue; atomic_long_t user_flush_tlb; diff --git a/drivers/misc/sgi-gru/grutlbpurge.c b/drivers/misc/sgi-gru/grutlbpurge.c index 4e438e4dd2a..1d125091f5e 100644 --- a/drivers/misc/sgi-gru/grutlbpurge.c +++ b/drivers/misc/sgi-gru/grutlbpurge.c @@ -210,7 +210,7 @@ void gru_flush_all_tlb(struct gru_state *gru) { struct gru_tlb_global_handle *tgh; - gru_dbg(grudev, "gru %p, gid %d\n", gru, gru->gs_gid); + gru_dbg(grudev, "gid %d\n", gru->gs_gid); tgh = get_lock_tgh_handle(gru); tgh_invalidate(tgh, 0, ~0, 0, 1, 1, GRUMAXINVAL - 1, 0xffff); get_unlock_tgh_handle(tgh); -- cgit v1.2.3 From a24e5e1caf4875bbe09f328b5f4daef6d25ebcc4 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 2 Apr 2009 16:59:06 -0700 Subject: sgi-gru: change GRU CCH commands from inline functions to outofline functions Change the GRU instructions that manage contexts from inline functions to out-of-line functions. This simplifies adding statistics & error checking to these functions. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/Makefile | 2 +- drivers/misc/sgi-gru/gruhandles.c | 197 ++++++++++++++++++++++++++++++++++++++ drivers/misc/sgi-gru/gruhandles.h | 176 ++++------------------------------ drivers/misc/sgi-gru/grutables.h | 3 + 4 files changed, 218 insertions(+), 160 deletions(-) create mode 100644 drivers/misc/sgi-gru/gruhandles.c (limited to 'drivers/misc/sgi-gru') diff --git a/drivers/misc/sgi-gru/Makefile b/drivers/misc/sgi-gru/Makefile index 9e9170b3599..bcd8136d2f9 100644 --- a/drivers/misc/sgi-gru/Makefile +++ b/drivers/misc/sgi-gru/Makefile @@ -3,5 +3,5 @@ ifdef CONFIG_SGI_GRU_DEBUG endif obj-$(CONFIG_SGI_GRU) := gru.o -gru-y := grufile.o grumain.o grufault.o grutlbpurge.o gruprocfs.o grukservices.o +gru-y := grufile.o grumain.o grufault.o grutlbpurge.o gruprocfs.o grukservices.o gruhandles.o diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c new file mode 100644 index 00000000000..5b82e7def16 --- /dev/null +++ b/drivers/misc/sgi-gru/gruhandles.c @@ -0,0 +1,197 @@ +/* + * GRU KERNEL MCS INSTRUCTIONS + * + * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include "gru.h" +#include "grulib.h" +#include "grutables.h" + +/* 10 sec */ +#ifdef CONFIG_IA64 +#include +#define GRU_OPERATION_TIMEOUT (((cycles_t) local_cpu_data->itc_freq)*10) +#else +#include +#define GRU_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000) +#endif + +/* Extract the status field from a kernel handle */ +#define GET_MSEG_HANDLE_STATUS(h) (((*(unsigned long *)(h)) >> 16) & 3) + +static void start_instruction(void *h) +{ + unsigned long *w0 = h; + + wmb(); /* setting CMD bit must be last */ + *w0 = *w0 | 1; + gru_flush_cache(h); +} + +static int wait_instruction_complete(void *h, enum mcs_op opc) +{ + int status; + cycles_t start_time = get_cycles(); + + while (1) { + cpu_relax(); + status = GET_MSEG_HANDLE_STATUS(h); + if (status != CCHSTATUS_ACTIVE) + break; + if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time)) + panic("GRU %p is malfunctioning\n", h); + } + return status; +} + +#if defined CONFIG_IA64 +static void cch_allocate_set_asids( + struct gru_context_configuration_handle *cch, int asidval) +{ + int i; + + for (i = 0; i < 8; i++) { + cch->asid[i] = (asidval++); +#if 0 + /* ZZZ hugepages not supported yet */ + if (i == RGN_HPAGE) + cch->sizeavail[i] = GRU_SIZEAVAIL(hpage_shift); + else +#endif + cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT); + } +} +#elif defined CONFIG_X86_64 +static void cch_allocate_set_asids( + struct gru_context_configuration_handle *cch, int asidval) +{ + int i; + + for (i = 0; i < 8; i++) { + cch->asid[i] = asidval++; + cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT) | + GRU_SIZEAVAIL(21); + } +} +#endif + +int cch_allocate(struct gru_context_configuration_handle *cch, + int asidval, unsigned long cbrmap, + unsigned long dsrmap) +{ + cch_allocate_set_asids(cch, asidval); + cch->dsr_allocation_map = dsrmap; + cch->cbr_allocation_map = cbrmap; + cch->opc = CCHOP_ALLOCATE; + start_instruction(cch); + return wait_instruction_complete(cch, cchop_allocate); +} + +int cch_start(struct gru_context_configuration_handle *cch) +{ + cch->opc = CCHOP_START; + start_instruction(cch); + return wait_instruction_complete(cch, cchop_start); +} + +int cch_interrupt(struct gru_context_configuration_handle *cch) +{ + cch->opc = CCHOP_INTERRUPT; + start_instruction(cch); + return wait_instruction_complete(cch, cchop_interrupt); +} + +int cch_deallocate(struct gru_context_configuration_handle *cch) +{ + cch->opc = CCHOP_DEALLOCATE; + start_instruction(cch); + return wait_instruction_complete(cch, cchop_deallocate); +} + +int cch_interrupt_sync(struct gru_context_configuration_handle + *cch) +{ + cch->opc = CCHOP_INTERRUPT_SYNC; + start_instruction(cch); + return wait_instruction_complete(cch, cchop_interrupt_sync); +} + +int tgh_invalidate(struct gru_tlb_global_handle *tgh, + unsigned long vaddr, unsigned long vaddrmask, + int asid, int pagesize, int global, int n, + unsigned short ctxbitmap) +{ + tgh->vaddr = vaddr; + tgh->asid = asid; + tgh->pagesize = pagesize; + tgh->n = n; + tgh->global = global; + tgh->vaddrmask = vaddrmask; + tgh->ctxbitmap = ctxbitmap; + tgh->opc = TGHOP_TLBINV; + start_instruction(tgh); + return wait_instruction_complete(tgh, tghop_invalidate); +} + +void tfh_write_only(struct gru_tlb_fault_handle *tfh, + unsigned long pfn, unsigned long vaddr, + int asid, int dirty, int pagesize) +{ + tfh->fillasid = asid; + tfh->fillvaddr = vaddr; + tfh->pfn = pfn; + tfh->dirty = dirty; + tfh->pagesize = pagesize; + tfh->opc = TFHOP_WRITE_ONLY; + start_instruction(tfh); +} + +void tfh_write_restart(struct gru_tlb_fault_handle *tfh, + unsigned long paddr, int gaa, + unsigned long vaddr, int asid, int dirty, + int pagesize) +{ + tfh->fillasid = asid; + tfh->fillvaddr = vaddr; + tfh->pfn = paddr >> GRU_PADDR_SHIFT; + tfh->gaa = gaa; + tfh->dirty = dirty; + tfh->pagesize = pagesize; + tfh->opc = TFHOP_WRITE_RESTART; + start_instruction(tfh); +} + +void tfh_restart(struct gru_tlb_fault_handle *tfh) +{ + tfh->opc = TFHOP_RESTART; + start_instruction(tfh); +} + +void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh) +{ + tfh->opc = TFHOP_USER_POLLING_MODE; + start_instruction(tfh); +} + +void tfh_exception(struct gru_tlb_fault_handle *tfh) +{ + tfh->opc = TFHOP_EXCEPTION; + start_instruction(tfh); +} + diff --git a/drivers/misc/sgi-gru/gruhandles.h b/drivers/misc/sgi-gru/gruhandles.h index fb72a52a34a..dfce8ea5da2 100644 --- a/drivers/misc/sgi-gru/gruhandles.h +++ b/drivers/misc/sgi-gru/gruhandles.h @@ -495,164 +495,22 @@ enum gru_cbr_state { /* minimum TLB purge count to ensure a full purge */ #define GRUMAXINVAL 1024UL - -/* Extract the status field from a kernel handle */ -#define GET_MSEG_HANDLE_STATUS(h) (((*(unsigned long *)(h)) >> 16) & 3) - -static inline void start_instruction(void *h) -{ - unsigned long *w0 = h; - - wmb(); /* setting CMD bit must be last */ - *w0 = *w0 | 1; - gru_flush_cache(h); -} - -static inline int wait_instruction_complete(void *h) -{ - int status; - - do { - cpu_relax(); - barrier(); - status = GET_MSEG_HANDLE_STATUS(h); - } while (status == CCHSTATUS_ACTIVE); - return status; -} - -#if defined CONFIG_IA64 -static inline void cch_allocate_set_asids( - struct gru_context_configuration_handle *cch, int asidval) -{ - int i; - - for (i = 0; i <= RGN_HPAGE; i++) { /* assume HPAGE is last region */ - cch->asid[i] = (asidval++); -#if 0 - /* ZZZ hugepages not supported yet */ - if (i == RGN_HPAGE) - cch->sizeavail[i] = GRU_SIZEAVAIL(hpage_shift); - else -#endif - cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT); - } -} -#elif defined CONFIG_X86_64 -static inline void cch_allocate_set_asids( - struct gru_context_configuration_handle *cch, int asidval) -{ - int i; - - for (i = 0; i < 8; i++) { - cch->asid[i] = asidval++; - cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT) | - GRU_SIZEAVAIL(21); - } -} -#endif - -static inline int cch_allocate(struct gru_context_configuration_handle *cch, - int asidval, unsigned long cbrmap, - unsigned long dsrmap) -{ - cch_allocate_set_asids(cch, asidval); - cch->dsr_allocation_map = dsrmap; - cch->cbr_allocation_map = cbrmap; - cch->opc = CCHOP_ALLOCATE; - start_instruction(cch); - return wait_instruction_complete(cch); -} - -static inline int cch_start(struct gru_context_configuration_handle *cch) -{ - cch->opc = CCHOP_START; - start_instruction(cch); - return wait_instruction_complete(cch); -} - -static inline int cch_interrupt(struct gru_context_configuration_handle *cch) -{ - cch->opc = CCHOP_INTERRUPT; - start_instruction(cch); - return wait_instruction_complete(cch); -} - -static inline int cch_deallocate(struct gru_context_configuration_handle *cch) -{ - cch->opc = CCHOP_DEALLOCATE; - start_instruction(cch); - return wait_instruction_complete(cch); -} - -static inline int cch_interrupt_sync(struct gru_context_configuration_handle - *cch) -{ - cch->opc = CCHOP_INTERRUPT_SYNC; - start_instruction(cch); - return wait_instruction_complete(cch); -} - -static inline int tgh_invalidate(struct gru_tlb_global_handle *tgh, - unsigned long vaddr, unsigned long vaddrmask, - int asid, int pagesize, int global, int n, - unsigned short ctxbitmap) -{ - tgh->vaddr = vaddr; - tgh->asid = asid; - tgh->pagesize = pagesize; - tgh->n = n; - tgh->global = global; - tgh->vaddrmask = vaddrmask; - tgh->ctxbitmap = ctxbitmap; - tgh->opc = TGHOP_TLBINV; - start_instruction(tgh); - return wait_instruction_complete(tgh); -} - -static inline void tfh_write_only(struct gru_tlb_fault_handle *tfh, - unsigned long pfn, unsigned long vaddr, - int asid, int dirty, int pagesize) -{ - tfh->fillasid = asid; - tfh->fillvaddr = vaddr; - tfh->pfn = pfn; - tfh->dirty = dirty; - tfh->pagesize = pagesize; - tfh->opc = TFHOP_WRITE_ONLY; - start_instruction(tfh); -} - -static inline void tfh_write_restart(struct gru_tlb_fault_handle *tfh, - unsigned long paddr, int gaa, - unsigned long vaddr, int asid, int dirty, - int pagesize) -{ - tfh->fillasid = asid; - tfh->fillvaddr = vaddr; - tfh->pfn = paddr >> GRU_PADDR_SHIFT; - tfh->gaa = gaa; - tfh->dirty = dirty; - tfh->pagesize = pagesize; - tfh->opc = TFHOP_WRITE_RESTART; - start_instruction(tfh); -} - -static inline void tfh_restart(struct gru_tlb_fault_handle *tfh) -{ - tfh->opc = TFHOP_RESTART; - start_instruction(tfh); -} - -static inline void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh) -{ - tfh->opc = TFHOP_USER_POLLING_MODE; - start_instruction(tfh); -} - -static inline void tfh_exception(struct gru_tlb_fault_handle *tfh) -{ - tfh->opc = TFHOP_EXCEPTION; - start_instruction(tfh); -} +int cch_allocate(struct gru_context_configuration_handle *cch, + int asidval, unsigned long cbrmap, unsigned long dsrmap); + +int cch_start(struct gru_context_configuration_handle *cch); +int cch_interrupt(struct gru_context_configuration_handle *cch); +int cch_deallocate(struct gru_context_configuration_handle *cch); +int cch_interrupt_sync(struct gru_context_configuration_handle *cch); +int tgh_invalidate(struct gru_tlb_global_handle *tgh, unsigned long vaddr, + unsigned long vaddrmask, int asid, int pagesize, int global, int n, + unsigned short ctxbitmap); +void tfh_write_only(struct gru_tlb_fault_handle *tfh, unsigned long pfn, + unsigned long vaddr, int asid, int dirty, int pagesize); +void tfh_write_restart(struct gru_tlb_fault_handle *tfh, unsigned long paddr, + int gaa, unsigned long vaddr, int asid, int dirty, int pagesize); +void tfh_restart(struct gru_tlb_fault_handle *tfh); +void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh); +void tfh_exception(struct gru_tlb_fault_handle *tfh); #endif /* __GRUHANDLES_H__ */ diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index cc0fc734405..378a0894198 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -239,6 +239,9 @@ struct gru_stats_s { }; +enum mcs_op {cchop_allocate, cchop_start, cchop_interrupt, cchop_interrupt_sync, + cchop_deallocate, tghop_invalidate, mcsop_last}; + #define OPT_DPRINT 1 #define OPT_STATS 2 #define GRU_QUICKLOOK 4 -- cgit v1.2.3 From e56484da3aa550dace96cbf1202446231471d7e1 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 2 Apr 2009 16:59:06 -0700 Subject: sgi-gru: add statistics to the GRU context management functions Add statistics to the GRU context management functions. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/gruhandles.c | 12 ++++++++++++ drivers/misc/sgi-gru/gruprocfs.c | 39 +++++++++++++++++++++++++++++++++++++++ drivers/misc/sgi-gru/grutables.h | 8 ++++++++ 3 files changed, 59 insertions(+) (limited to 'drivers/misc/sgi-gru') diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c index 5b82e7def16..9dcf5d89668 100644 --- a/drivers/misc/sgi-gru/gruhandles.c +++ b/drivers/misc/sgi-gru/gruhandles.c @@ -35,6 +35,16 @@ /* Extract the status field from a kernel handle */ #define GET_MSEG_HANDLE_STATUS(h) (((*(unsigned long *)(h)) >> 16) & 3) +struct mcs_op_statistic mcs_op_statistics[mcsop_last]; + +static void update_mcs_stats(enum mcs_op op, unsigned long clks) +{ + atomic_long_inc(&mcs_op_statistics[op].count); + atomic_long_add(clks, &mcs_op_statistics[op].total); + if (mcs_op_statistics[op].max < clks) + mcs_op_statistics[op].max = clks; +} + static void start_instruction(void *h) { unsigned long *w0 = h; @@ -57,6 +67,8 @@ static int wait_instruction_complete(void *h, enum mcs_op opc) if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time)) panic("GRU %p is malfunctioning\n", h); } + if (gru_options & OPT_STATS) + update_mcs_stats(opc, get_cycles() - start_time); return status; } diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c index 6d63a8d70f8..8724551aa55 100644 --- a/drivers/misc/sgi-gru/gruprocfs.c +++ b/drivers/misc/sgi-gru/gruprocfs.c @@ -122,6 +122,30 @@ static ssize_t statistics_write(struct file *file, const char __user *userbuf, return count; } +static int mcs_statistics_show(struct seq_file *s, void *p) +{ + int op; + unsigned long total, count, max; + static char *id[] = {"cch_allocate", "cch_start", "cch_interrupt", + "cch_interrupt_sync", "cch_deallocate", "tgh_invalidate"}; + + for (op = 0; op < mcsop_last; op++) { + count = atomic_long_read(&mcs_op_statistics[op].count); + total = atomic_long_read(&mcs_op_statistics[op].total); + max = mcs_op_statistics[op].max; + seq_printf(s, "%-20s%12ld%12ld%12ld\n", id[op], count, + count ? total / count : 0, max); + } + return 0; +} + +static ssize_t mcs_statistics_write(struct file *file, + const char __user *userbuf, size_t count, loff_t *data) +{ + memset(mcs_op_statistics, 0, sizeof(mcs_op_statistics)); + return count; +} + static int options_show(struct seq_file *s, void *p) { seq_printf(s, "0x%lx\n", gru_options); @@ -137,6 +161,7 @@ static ssize_t options_write(struct file *file, const char __user *userbuf, if (copy_from_user (buf, userbuf, count < sizeof(buf) ? count : sizeof(buf))) return -EFAULT; + buf[count - 1] = '\0'; if (!strict_strtoul(buf, 10, &val)) gru_options = val; @@ -233,6 +258,11 @@ static int statistics_open(struct inode *inode, struct file *file) return single_open(file, statistics_show, NULL); } +static int mcs_statistics_open(struct inode *inode, struct file *file) +{ + return single_open(file, mcs_statistics_show, NULL); +} + static int options_open(struct inode *inode, struct file *file) { return single_open(file, options_show, NULL); @@ -257,6 +287,14 @@ static const struct file_operations statistics_fops = { .release = single_release, }; +static const struct file_operations mcs_statistics_fops = { + .open = mcs_statistics_open, + .read = seq_read, + .write = mcs_statistics_write, + .llseek = seq_lseek, + .release = single_release, +}; + static const struct file_operations options_fops = { .open = options_open, .read = seq_read, @@ -285,6 +323,7 @@ static struct proc_entry { struct proc_dir_entry *entry; } proc_files[] = { {"statistics", 0644, &statistics_fops}, + {"mcs_statistics", 0644, &mcs_statistics_fops}, {"debug_options", 0644, &options_fops}, {"cch_status", 0444, &cch_fops}, {"gru_status", 0444, &gru_fops}, diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index 378a0894198..685db88cc47 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -242,6 +242,14 @@ struct gru_stats_s { enum mcs_op {cchop_allocate, cchop_start, cchop_interrupt, cchop_interrupt_sync, cchop_deallocate, tghop_invalidate, mcsop_last}; +struct mcs_op_statistic { + atomic_long_t count; + atomic_long_t total; + unsigned long max; +}; + +extern struct mcs_op_statistic mcs_op_statistics[mcsop_last]; + #define OPT_DPRINT 1 #define OPT_STATS 2 #define GRU_QUICKLOOK 4 -- cgit v1.2.3 From bb04aa78ec1393295c30edad154e6f6e906955f3 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 2 Apr 2009 16:59:07 -0700 Subject: sgi-gru: add support for a user to explicitly unload a GRU context Add support for a user to explicitly unload a GRU context. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'drivers/misc/sgi-gru') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 61aa80a8411..8ae426edc85 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include "gru.h" #include "grutables.h" @@ -575,6 +576,38 @@ int gru_get_exception_detail(unsigned long arg) /* * User request to unload a context. Content is saved for possible reload. */ +static int gru_unload_all_contexts(void) +{ + struct gru_thread_state *gts; + struct gru_state *gru; + int maxgid, gid, ctxnum; + int nodesperblade; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (num_online_nodes() > 1 && + (uv_node_to_blade_id(1) == uv_node_to_blade_id(0))) + nodesperblade = 2; + else + nodesperblade = 1; + maxgid = GRU_CHIPLETS_PER_BLADE * num_online_nodes() / nodesperblade; + for (gid = 0; gid < maxgid; gid++) { + gru = GID_TO_GRU(gid); + spin_lock(&gru->gs_lock); + for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) { + gts = gru->gs_gts[ctxnum]; + if (gts && mutex_trylock(>s->ts_ctxlock)) { + spin_unlock(&gru->gs_lock); + gru_unload_context(gts, 1); + gru_unlock_gts(gts); + spin_lock(&gru->gs_lock); + } + } + spin_unlock(&gru->gs_lock); + } + return 0; +} + int gru_user_unload_context(unsigned long arg) { struct gru_thread_state *gts; @@ -586,6 +619,9 @@ int gru_user_unload_context(unsigned long arg) gru_dbg(grudev, "gseg 0x%lx\n", req.gseg); + if (!req.gseg) + return gru_unload_all_contexts(); + gts = gru_find_lock_gts(req.gseg); if (!gts) return -EINVAL; -- cgit v1.2.3 From 874194123718e625aa96632bac457d686ba1378e Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 2 Apr 2009 16:59:08 -0700 Subject: sgi-gru: aSID (context management) bug fixes This patch fixes bugs related to ASID (context id) management in the GRU driver. These changes are all internal to the SGI GRU driver and have no effect on the base kernel. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufile.c | 1 + drivers/misc/sgi-gru/grumain.c | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'drivers/misc/sgi-gru') diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c index 812678d8b47..f8dcdb1a0ec 100644 --- a/drivers/misc/sgi-gru/grufile.c +++ b/drivers/misc/sgi-gru/grufile.c @@ -274,6 +274,7 @@ static void gru_init_chiplet(struct gru_state *gru, unsigned long paddr, gru->gs_blade_id = bid; gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1; gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1; + gru->gs_asid_limit = MAX_ASID; gru_tgh_flush_init(gru); gru_dbg(grudev, "bid %d, nid %d, gid %d, vaddr %p (0x%lx)\n", bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr, diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index de60458f6d1..5fc7b5ecde6 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -79,7 +79,6 @@ static int gru_wrap_asid(struct gru_state *gru) gru_dbg(grudev, "gid %d\n", gru->gs_gid); STAT(asid_wrap); gru->gs_asid_gen++; - gru_flush_all_tlb(gru); return MIN_ASID; } @@ -93,6 +92,7 @@ static int gru_reset_asid_limit(struct gru_state *gru, int asid) limit = MAX_ASID; if (asid >= limit) asid = gru_wrap_asid(gru); + gru_flush_all_tlb(gru); gid = gru->gs_gid; again: for (i = 0; i < GRU_NUM_CCH; i++) { @@ -131,12 +131,10 @@ static int gru_assign_asid(struct gru_state *gru) { int asid; - spin_lock(&gru->gs_asid_lock); gru->gs_asid += ASID_INC; asid = gru->gs_asid; if (asid >= gru->gs_asid_limit) asid = gru_reset_asid_limit(gru, asid); - spin_unlock(&gru->gs_asid_lock); gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid); return asid; @@ -227,7 +225,9 @@ static int gru_load_mm_tracker(struct gru_state *gru, spin_lock(&gms->ms_asid_lock); asid = asids->mt_asid; - if (asid == 0 || asids->mt_asid_gen != gru->gs_asid_gen) { + spin_lock(&gru->gs_asid_lock); + if (asid == 0 || (asids->mt_ctxbitmap == 0 && asids->mt_asid_gen != + gru->gs_asid_gen)) { asid = gru_assign_asid(gru); asids->mt_asid = asid; asids->mt_asid_gen = gru->gs_asid_gen; @@ -235,6 +235,7 @@ static int gru_load_mm_tracker(struct gru_state *gru, } else { STAT(asid_reuse); } + spin_unlock(&gru->gs_asid_lock); BUG_ON(asids->mt_ctxbitmap & ctxbitmap); asids->mt_ctxbitmap |= ctxbitmap; @@ -259,10 +260,12 @@ static void gru_unload_mm_tracker(struct gru_state *gru, asids = &gms->ms_asids[gru->gs_gid]; ctxbitmap = (1 << gts->ts_ctxnum); spin_lock(&gms->ms_asid_lock); + spin_lock(&gru->gs_asid_lock); BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap); asids->mt_ctxbitmap ^= ctxbitmap; gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n", gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]); + spin_unlock(&gru->gs_asid_lock); spin_unlock(&gms->ms_asid_lock); } @@ -412,6 +415,7 @@ static void gru_free_gru_context(struct gru_thread_state *gts) __clear_bit(gts->ts_ctxnum, &gru->gs_context_map); gts->ts_ctxnum = NULLCTX; gts->ts_gru = NULL; + gts->ts_blade = -1; spin_unlock(&gru->gs_lock); gts_drop(gts); @@ -731,6 +735,7 @@ again: } reserve_gru_resources(gru, gts); gts->ts_gru = gru; + gts->ts_blade = gru->gs_blade_id; gts->ts_ctxnum = find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH); BUG_ON(gts->ts_ctxnum == GRU_NUM_CCH); -- cgit v1.2.3 From ecdaf2b55251f718a1fbaf4a3f72bfd6e25c582c Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 2 Apr 2009 16:59:09 -0700 Subject: sgi-gru: restructure the GRU vtop functions Restructure the GRU vtop functions in preparation for future changes. This patch simply moves code around & does not change the algorithm. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 68 ++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 24 deletions(-) (limited to 'drivers/misc/sgi-gru') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 8ae426edc85..bf6e9f7bed5 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -267,6 +267,44 @@ err: return 1; } +static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr, + int write, int atomic, unsigned long *gpa, int *pageshift) +{ + struct mm_struct *mm = gts->ts_mm; + struct vm_area_struct *vma; + unsigned long paddr; + int ret, ps; + + vma = find_vma(mm, vaddr); + if (!vma) + goto inval; + + /* + * Atomic lookup is faster & usually works even if called in non-atomic + * context. + */ + rmb(); /* Must/check ms_range_active before loading PTEs */ + ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &ps); + if (ret) { + if (atomic) + goto upm; + if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps)) + goto inval; + } + if (is_gru_paddr(paddr)) + goto inval; + paddr = paddr & ~((1UL << ps) - 1); + *gpa = uv_soc_phys_ram_to_gpa(paddr); + *pageshift = ps; + return 0; + +inval: + return -1; +upm: + return -2; +} + + /* * Drop a TLB entry into the GRU. The fault is described by info in an TFH. * Input: @@ -281,10 +319,8 @@ static int gru_try_dropin(struct gru_thread_state *gts, struct gru_tlb_fault_handle *tfh, unsigned long __user *cb) { - struct mm_struct *mm = gts->ts_mm; - struct vm_area_struct *vma; - int pageshift, asid, write, ret; - unsigned long paddr, gpa, vaddr; + int pageshift = 0, asid, write, ret, atomic = !cb; + unsigned long gpa = 0, vaddr = 0; /* * NOTE: The GRU contains magic hardware that eliminates races between @@ -318,28 +354,12 @@ static int gru_try_dropin(struct gru_thread_state *gts, if (atomic_read(>s->ts_gms->ms_range_active)) goto failactive; - vma = find_vma(mm, vaddr); - if (!vma) - goto failinval; - - /* - * Atomic lookup is faster & usually works even if called in non-atomic - * context. - */ - rmb(); /* Must/check ms_range_active before loading PTEs */ - ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &pageshift); - if (ret) { - if (!cb) - goto failupm; - if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, - &pageshift)) - goto failinval; - } - if (is_gru_paddr(paddr)) + ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift); + if (ret == -1) goto failinval; + if (ret == -2) + goto failupm; - paddr = paddr & ~((1UL << pageshift) - 1); - gpa = uv_soc_phys_ram_to_gpa(paddr); gru_cb_set_istatus_active(cb); tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write, GRU_PAGESIZE(pageshift)); -- cgit v1.2.3 From 6f2584f47474d29ce829604bfc8b56c10b352fdb Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 2 Apr 2009 16:59:10 -0700 Subject: sgi-gru: add support to the GRU driver for message queue interrupts Add support to the GRU driver for message queue interrupts. Signed-off-by: Jack Steiner Signed-off-by: Dean Nelson Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grukservices.c | 104 ++++++++++++++++++++---------------- drivers/misc/sgi-gru/grukservices.h | 33 ++++++++---- 2 files changed, 82 insertions(+), 55 deletions(-) (limited to 'drivers/misc/sgi-gru') diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c index 247635d6d6e..a3712843115 100644 --- a/drivers/misc/sgi-gru/grukservices.c +++ b/drivers/misc/sgi-gru/grukservices.c @@ -52,8 +52,10 @@ */ /* Blade percpu resources PERMANENTLY reserved for kernel use */ -#define GRU_NUM_KERNEL_CBR 1 +#define GRU_NUM_KERNEL_CBR 1 #define GRU_NUM_KERNEL_DSR_BYTES 256 +#define GRU_NUM_KERNEL_DSR_CL (GRU_NUM_KERNEL_DSR_BYTES / \ + GRU_CACHE_LINE_BYTES) #define KERNEL_CTXNUM 15 /* GRU instruction attributes for all instructions */ @@ -94,7 +96,6 @@ struct message_header { char fill; }; -#define QLINES(mq) ((mq) + offsetof(struct message_queue, qlines)) #define HSTATUS(mq, h) ((mq) + offsetof(struct message_queue, hstatus[h])) static int gru_get_cpu_resources(int dsr_bytes, void **cb, void **dsr) @@ -250,7 +251,8 @@ static inline void restore_present2(void *p, int val) * Create a message queue. * qlines - message queue size in cache lines. Includes 2-line header. */ -int gru_create_message_queue(void *p, unsigned int bytes) +int gru_create_message_queue(struct gru_message_queue_desc *mqd, + void *p, unsigned int bytes, int nasid, int vector, int apicid) { struct message_queue *mq = p; unsigned int qlines; @@ -265,6 +267,12 @@ int gru_create_message_queue(void *p, unsigned int bytes) mq->hstatus[0] = 0; mq->hstatus[1] = 1; mq->head = gru_mesq_head(2, qlines / 2 + 1); + mqd->mq = mq; + mqd->mq_gpa = uv_gpa(mq); + mqd->qlines = qlines; + mqd->interrupt_pnode = UV_NASID_TO_PNODE(nasid); + mqd->interrupt_vector = vector; + mqd->interrupt_apicid = apicid; return 0; } EXPORT_SYMBOL_GPL(gru_create_message_queue); @@ -277,8 +285,8 @@ EXPORT_SYMBOL_GPL(gru_create_message_queue); * -1 - if mesq sent successfully but queue not full * >0 - unexpected error. MQE_xxx returned */ -static int send_noop_message(void *cb, - unsigned long mq, void *mesg) +static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd, + void *mesg) { const struct message_header noop_header = { .present = MQS_NOOP, .lines = 1}; @@ -289,7 +297,7 @@ static int send_noop_message(void *cb, STAT(mesq_noop); save_mhdr = *mhdr; *mhdr = noop_header; - gru_mesq(cb, mq, gru_get_tri(mhdr), 1, IMA); + gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA); ret = gru_wait(cb); if (ret) { @@ -313,7 +321,7 @@ static int send_noop_message(void *cb, break; case CBSS_PUT_NACKED: STAT(mesq_noop_put_nacked); - m = mq + (gru_get_amo_value_head(cb) << 6); + m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1, IMA); if (gru_wait(cb) == CBS_IDLE) @@ -333,30 +341,20 @@ static int send_noop_message(void *cb, /* * Handle a gru_mesq full. */ -static int send_message_queue_full(void *cb, - unsigned long mq, void *mesg, int lines) +static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd, + void *mesg, int lines) { union gru_mesqhead mqh; unsigned int limit, head; unsigned long avalue; - int half, qlines, save; + int half, qlines; /* Determine if switching to first/second half of q */ avalue = gru_get_amo_value(cb); head = gru_get_amo_value_head(cb); limit = gru_get_amo_value_limit(cb); - /* - * Fetch "qlines" from the queue header. Since the queue may be - * in memory that can't be accessed using socket addresses, use - * the GRU to access the data. Use DSR space from the message. - */ - save = *(int *)mesg; - gru_vload(cb, QLINES(mq), gru_get_tri(mesg), XTYPE_W, 1, 1, IMA); - if (gru_wait(cb) != CBS_IDLE) - goto cberr; - qlines = *(int *)mesg; - *(int *)mesg = save; + qlines = mqd->qlines; half = (limit != qlines); if (half) @@ -365,7 +363,7 @@ static int send_message_queue_full(void *cb, mqh = gru_mesq_head(2, qlines / 2 + 1); /* Try to get lock for switching head pointer */ - gru_gamir(cb, EOP_IR_CLR, HSTATUS(mq, half), XTYPE_DW, IMA); + gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; if (!gru_get_amo_value(cb)) { @@ -375,8 +373,8 @@ static int send_message_queue_full(void *cb, /* Got the lock. Send optional NOP if queue not full, */ if (head != limit) { - if (send_noop_message(cb, mq, mesg)) { - gru_gamir(cb, EOP_IR_INC, HSTATUS(mq, half), + if (send_noop_message(cb, mqd, mesg)) { + gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; @@ -387,14 +385,16 @@ static int send_message_queue_full(void *cb, } /* Then flip queuehead to other half of queue. */ - gru_gamer(cb, EOP_ERR_CSWAP, mq, XTYPE_DW, mqh.val, avalue, IMA); + gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue, + IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; /* If not successfully in swapping queue head, clear the hstatus lock */ if (gru_get_amo_value(cb) != avalue) { STAT(mesq_qf_switch_head_failed); - gru_gamir(cb, EOP_IR_INC, HSTATUS(mq, half), XTYPE_DW, IMA); + gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, + IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; } @@ -404,15 +404,25 @@ cberr: return MQE_UNEXPECTED_CB_ERR; } +/* + * Send a cross-partition interrupt to the SSI that contains the target + * message queue. Normally, the interrupt is automatically delivered by hardware + * but some error conditions require explicit delivery. + */ +static void send_message_queue_interrupt(struct gru_message_queue_desc *mqd) +{ + if (mqd->interrupt_vector) + uv_hub_send_ipi(mqd->interrupt_pnode, mqd->interrupt_apicid, + mqd->interrupt_vector); +} + /* * Handle a gru_mesq failure. Some of these failures are software recoverable * or retryable. */ -static int send_message_failure(void *cb, - unsigned long mq, - void *mesg, - int lines) +static int send_message_failure(void *cb, struct gru_message_queue_desc *mqd, + void *mesg, int lines) { int substatus, ret = 0; unsigned long m; @@ -429,7 +439,7 @@ static int send_message_failure(void *cb, break; case CBSS_QLIMIT_REACHED: STAT(mesq_send_qlimit_reached); - ret = send_message_queue_full(cb, mq, mesg, lines); + ret = send_message_queue_full(cb, mqd, mesg, lines); break; case CBSS_AMO_NACKED: STAT(mesq_send_amo_nacked); @@ -437,12 +447,14 @@ static int send_message_failure(void *cb, break; case CBSS_PUT_NACKED: STAT(mesq_send_put_nacked); - m = mq + (gru_get_amo_value_head(cb) << 6); + m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA); - if (gru_wait(cb) == CBS_IDLE) + if (gru_wait(cb) == CBS_IDLE) { ret = MQE_OK; - else + send_message_queue_interrupt(mqd); + } else { ret = MQE_UNEXPECTED_CB_ERR; + } break; default: BUG(); @@ -452,12 +464,12 @@ static int send_message_failure(void *cb, /* * Send a message to a message queue - * cb GRU control block to use to send message - * mq message queue + * mqd message queue descriptor * mesg message. ust be vaddr within a GSEG * bytes message size (<= 2 CL) */ -int gru_send_message_gpa(unsigned long mq, void *mesg, unsigned int bytes) +int gru_send_message_gpa(struct gru_message_queue_desc *mqd, void *mesg, + unsigned int bytes) { struct message_header *mhdr; void *cb; @@ -481,10 +493,10 @@ int gru_send_message_gpa(unsigned long mq, void *mesg, unsigned int bytes) do { ret = MQE_OK; - gru_mesq(cb, mq, gru_get_tri(mhdr), clines, IMA); + gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), clines, IMA); istatus = gru_wait(cb); if (istatus != CBS_IDLE) - ret = send_message_failure(cb, mq, dsr, clines); + ret = send_message_failure(cb, mqd, dsr, clines); } while (ret == MQIE_AGAIN); gru_free_cpu_resources(cb, dsr); @@ -497,9 +509,9 @@ EXPORT_SYMBOL_GPL(gru_send_message_gpa); /* * Advance the receive pointer for the queue to the next message. */ -void gru_free_message(void *rmq, void *mesg) +void gru_free_message(struct gru_message_queue_desc *mqd, void *mesg) { - struct message_queue *mq = rmq; + struct message_queue *mq = mqd->mq; struct message_header *mhdr = mq->next; void *next, *pnext; int half = -1; @@ -529,16 +541,16 @@ EXPORT_SYMBOL_GPL(gru_free_message); * present. User must call next_message() to move to next message. * rmq message queue */ -void *gru_get_next_message(void *rmq) +void *gru_get_next_message(struct gru_message_queue_desc *mqd) { - struct message_queue *mq = rmq; + struct message_queue *mq = mqd->mq; struct message_header *mhdr = mq->next; int present = mhdr->present; /* skip NOOP messages */ STAT(mesq_receive); while (present == MQS_NOOP) { - gru_free_message(rmq, mhdr); + gru_free_message(mqd, mhdr); mhdr = mq->next; present = mhdr->present; } @@ -576,7 +588,7 @@ int gru_copy_gpa(unsigned long dest_gpa, unsigned long src_gpa, if (gru_get_cpu_resources(GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr)) return MQE_BUG_NO_RESOURCES; gru_bcopy(cb, src_gpa, dest_gpa, gru_get_tri(dsr), - XTYPE_B, bytes, GRU_NUM_KERNEL_DSR_BYTES, IMA); + XTYPE_B, bytes, GRU_NUM_KERNEL_DSR_CL, IMA); ret = gru_wait(cb); gru_free_cpu_resources(cb, dsr); return ret; @@ -611,7 +623,7 @@ static int quicktest(struct gru_state *gru) if (word0 != word1 || word0 != MAGIC) { printk - ("GRU quicktest err: gru %d, found 0x%lx, expected 0x%lx\n", + ("GRU quicktest err: gid %d, found 0x%lx, expected 0x%lx\n", gru->gs_gid, word1, MAGIC); BUG(); /* ZZZ should not be fatal */ } diff --git a/drivers/misc/sgi-gru/grukservices.h b/drivers/misc/sgi-gru/grukservices.h index eb17e0a3ac6..747ed315d56 100644 --- a/drivers/misc/sgi-gru/grukservices.h +++ b/drivers/misc/sgi-gru/grukservices.h @@ -41,6 +41,15 @@ * - gru_create_message_queue() needs interrupt vector info */ +struct gru_message_queue_desc { + void *mq; /* message queue vaddress */ + unsigned long mq_gpa; /* global address of mq */ + int qlines; /* queue size in CL */ + int interrupt_vector; /* interrupt vector */ + int interrupt_pnode; /* pnode for interrupt */ + int interrupt_apicid; /* lapicid for interrupt */ +}; + /* * Initialize a user allocated chunk of memory to be used as * a message queue. The caller must ensure that the queue is @@ -51,14 +60,19 @@ * to manage the queue. * * Input: - * p pointer to user allocated memory. + * mqd pointer to message queue descriptor + * p pointer to user allocated mesq memory. * bytes size of message queue in bytes + * vector interrupt vector (zero if no interrupts) + * nasid nasid of blade where interrupt is delivered + * apicid apicid of cpu for interrupt * * Errors: * 0 OK * >0 error */ -extern int gru_create_message_queue(void *p, unsigned int bytes); +extern int gru_create_message_queue(struct gru_message_queue_desc *mqd, + void *p, unsigned int bytes, int nasid, int vector, int apicid); /* * Send a message to a message queue. @@ -68,7 +82,7 @@ extern int gru_create_message_queue(void *p, unsigned int bytes); * * * Input: - * xmq message queue - must be a UV global physical address + * mqd pointer to message queue descriptor * mesg pointer to message. Must be 64-bit aligned * bytes size of message in bytes * @@ -77,8 +91,8 @@ extern int gru_create_message_queue(void *p, unsigned int bytes); * >0 Send failure - see error codes below * */ -extern int gru_send_message_gpa(unsigned long mq_gpa, void *mesg, - unsigned int bytes); +extern int gru_send_message_gpa(struct gru_message_queue_desc *mqd, + void *mesg, unsigned int bytes); /* Status values for gru_send_message() */ #define MQE_OK 0 /* message sent successfully */ @@ -94,10 +108,11 @@ extern int gru_send_message_gpa(unsigned long mq_gpa, void *mesg, * API extensions may allow for out-of-order freeing. * * Input - * mq message queue + * mqd pointer to message queue descriptor * mesq message being freed */ -extern void gru_free_message(void *mq, void *mesq); +extern void gru_free_message(struct gru_message_queue_desc *mqd, + void *mesq); /* * Get next message from message queue. Returns pointer to @@ -106,13 +121,13 @@ extern void gru_free_message(void *mq, void *mesq); * in order to move the queue pointers to next message. * * Input - * mq message queue + * mqd pointer to message queue descriptor * * Output: * p pointer to message * NULL no message available */ -extern void *gru_get_next_message(void *mq); +extern void *gru_get_next_message(struct gru_message_queue_desc *mqd); /* -- cgit v1.2.3 From e1c3219d06447dab8a87b96df1211d158e21071e Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 2 Apr 2009 16:59:10 -0700 Subject: sgi-gru: macro for scanning all gru chiplets Add macro for scanning all active GRU chiplets. Maximum chiplet id is saved during GRU initialization. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 11 ++--------- drivers/misc/sgi-gru/grufile.c | 3 +++ drivers/misc/sgi-gru/gruprocfs.c | 4 ++-- drivers/misc/sgi-gru/grutables.h | 9 +++++++-- 4 files changed, 14 insertions(+), 13 deletions(-) (limited to 'drivers/misc/sgi-gru') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index bf6e9f7bed5..15507d31e62 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -600,18 +600,11 @@ static int gru_unload_all_contexts(void) { struct gru_thread_state *gts; struct gru_state *gru; - int maxgid, gid, ctxnum; - int nodesperblade; + int gid, ctxnum; if (!capable(CAP_SYS_ADMIN)) return -EPERM; - if (num_online_nodes() > 1 && - (uv_node_to_blade_id(1) == uv_node_to_blade_id(0))) - nodesperblade = 2; - else - nodesperblade = 1; - maxgid = GRU_CHIPLETS_PER_BLADE * num_online_nodes() / nodesperblade; - for (gid = 0; gid < maxgid; gid++) { + foreach_gid(gid) { gru = GID_TO_GRU(gid); spin_lock(&gru->gs_lock); for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) { diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c index f8dcdb1a0ec..b7a21ce9149 100644 --- a/drivers/misc/sgi-gru/grufile.c +++ b/drivers/misc/sgi-gru/grufile.c @@ -47,6 +47,7 @@ struct gru_blade_state *gru_base[GRU_MAX_BLADES] __read_mostly; unsigned long gru_start_paddr __read_mostly; unsigned long gru_end_paddr __read_mostly; +unsigned int gru_max_gids __read_mostly; struct gru_stats_s gru_stats; /* Guaranteed user available resources on each node */ @@ -276,6 +277,8 @@ static void gru_init_chiplet(struct gru_state *gru, unsigned long paddr, gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1; gru->gs_asid_limit = MAX_ASID; gru_tgh_flush_init(gru); + if (gru->gs_gid >= gru_max_gids) + gru_max_gids = gru->gs_gid + 1; gru_dbg(grudev, "bid %d, nid %d, gid %d, vaddr %p (0x%lx)\n", bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr, gru->gs_gru_base_paddr); diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c index 8724551aa55..ee74821b171 100644 --- a/drivers/misc/sgi-gru/gruprocfs.c +++ b/drivers/misc/sgi-gru/gruprocfs.c @@ -226,7 +226,7 @@ static void seq_stop(struct seq_file *file, void *data) static void *seq_start(struct seq_file *file, loff_t *gid) { - if (*gid < GRU_MAX_GRUS) + if (*gid < gru_max_gids) return gid; return NULL; } @@ -234,7 +234,7 @@ static void *seq_start(struct seq_file *file, loff_t *gid) static void *seq_next(struct seq_file *file, void *data, loff_t *gid) { (*gid)++; - if (*gid < GRU_MAX_GRUS) + if (*gid < gru_max_gids) return gid; return NULL; } diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index 685db88cc47..c6e0e3de07f 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -153,6 +153,7 @@ extern struct gru_stats_s gru_stats; extern struct gru_blade_state *gru_base[]; extern unsigned long gru_start_paddr, gru_end_paddr; +extern unsigned int gru_max_gids; #define GRU_MAX_BLADES MAX_NUMNODES #define GRU_MAX_GRUS (GRU_MAX_BLADES * GRU_CHIPLETS_PER_BLADE) @@ -406,12 +407,12 @@ struct gru_state { gru segments (64) */ void *gs_gru_base_vaddr; /* Virtual address of gru segments (64) */ - unsigned char gs_gid; /* unique GRU number */ + unsigned short gs_gid; /* unique GRU number */ + unsigned short gs_blade_id; /* blade of GRU */ unsigned char gs_tgh_local_shift; /* used to pick TGH for local flush */ unsigned char gs_tgh_first_remote; /* starting TGH# for remote flush */ - unsigned short gs_blade_id; /* blade of GRU */ spinlock_t gs_asid_lock; /* lock used for assigning asids */ spinlock_t gs_lock; /* lock used for @@ -506,6 +507,10 @@ struct gru_blade_state { (i) < GRU_CHIPLETS_PER_BLADE; \ (i)++, (gru)++) +/* Scan all GRUs */ +#define foreach_gid(gid) \ + for ((gid) = 0; (gid) < gru_max_gids; (gid)++) + /* Scan all active GTSs on a gru. Note: must hold ss_lock to use this macro. */ #define for_each_gts_on_gru(gts, gru, ctxnum) \ for ((ctxnum) = 0; (ctxnum) < GRU_NUM_CCH; (ctxnum)++) \ -- cgit v1.2.3 From 27ca8a7b2bdfb3e22e67fbd5df58e6b6f0bbcd48 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 2 Apr 2009 16:59:11 -0700 Subject: sgi-gru: fix bugs related to module unload of the GRU driver Fix bugs related to module unload of the GRU driver. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufile.c | 5 ++++- drivers/misc/sgi-gru/grukservices.c | 19 +++++++++++++++++++ drivers/misc/sgi-gru/grutables.h | 1 + 3 files changed, 24 insertions(+), 1 deletion(-) (limited to 'drivers/misc/sgi-gru') diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c index b7a21ce9149..3e6e42d2f01 100644 --- a/drivers/misc/sgi-gru/grufile.c +++ b/drivers/misc/sgi-gru/grufile.c @@ -440,7 +440,7 @@ exit1: static void __exit gru_exit(void) { - int i, bid; + int i, bid, gid; int order = get_order(sizeof(struct gru_state) * GRU_CHIPLETS_PER_BLADE); @@ -450,6 +450,9 @@ static void __exit gru_exit(void) for (i = 0; i < GRU_CHIPLETS_PER_BLADE; i++) free_irq(IRQ_GRU + i, NULL); + foreach_gid(gid) + gru_kservices_exit(GID_TO_GRU(gid)); + for (bid = 0; bid < GRU_MAX_BLADES; bid++) free_pages((unsigned long)gru_base[bid], order); diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c index a3712843115..6b1efe44727 100644 --- a/drivers/misc/sgi-gru/grukservices.c +++ b/drivers/misc/sgi-gru/grukservices.c @@ -690,3 +690,22 @@ int gru_kservices_init(struct gru_state *gru) quicktest(gru); return 0; } + +void gru_kservices_exit(struct gru_state *gru) +{ + struct gru_context_configuration_handle *cch; + struct gru_blade_state *bs; + + bs = gru->gs_blade; + if (gru != &bs->bs_grus[1]) + return; + + cch = get_cch(gru->gs_gru_base_vaddr, KERNEL_CTXNUM); + lock_cch_handle(cch); + if (cch_interrupt_sync(cch)) + BUG(); + if (cch_deallocate(cch)) + BUG(); + unlock_cch_handle(cch); +} + diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index c6e0e3de07f..8e5c8e5bb53 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -600,6 +600,7 @@ extern void gru_unload_context(struct gru_thread_state *gts, int savestate); extern void gts_drop(struct gru_thread_state *gts); extern void gru_tgh_flush_init(struct gru_state *gru); extern int gru_kservices_init(struct gru_state *gru); +extern void gru_kservices_exit(struct gru_state *gru); extern irqreturn_t gru_intr(int irq, void *dev_id); extern int gru_handle_user_call_os(unsigned long address); extern int gru_user_flush_tlb(unsigned long arg); -- cgit v1.2.3 From 7b8274e93fbabc7534aa51f95551c30aecdd0066 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 2 Apr 2009 16:59:12 -0700 Subject: sgi-gru: support multiple pagesizes in GRU Add multiple pagesize support to the GRU driver. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 15 +++++++++++++++ drivers/misc/sgi-gru/gruhandles.c | 34 ++++------------------------------ drivers/misc/sgi-gru/gruhandles.h | 2 +- drivers/misc/sgi-gru/grukservices.c | 2 +- drivers/misc/sgi-gru/grumain.c | 17 +++++++++++------ drivers/misc/sgi-gru/grutables.h | 3 +++ 6 files changed, 35 insertions(+), 38 deletions(-) (limited to 'drivers/misc/sgi-gru') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 15507d31e62..ab118558552 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -360,6 +360,13 @@ static int gru_try_dropin(struct gru_thread_state *gts, if (ret == -2) goto failupm; + if (!(gts->ts_sizeavail & GRU_SIZEAVAIL(pageshift))) { + gts->ts_sizeavail |= GRU_SIZEAVAIL(pageshift); + if (atomic || !gru_update_cch(gts, 0)) { + gts->ts_force_cch_reload = 1; + goto failupm; + } + } gru_cb_set_istatus_active(cb); tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write, GRU_PAGESIZE(pageshift)); @@ -535,6 +542,14 @@ int gru_handle_user_call_os(unsigned long cb) gts->ts_force_unload = 1; } + /* + * CCH may contain stale data if ts_force_cch_reload is set. + */ + if (gts->ts_gru && gts->ts_force_cch_reload) { + gru_update_cch(gts, 0); + gts->ts_force_cch_reload = 0; + } + ret = -EAGAIN; cbrnum = thread_cbr_number(gts, ucbnum); if (gts->ts_force_unload) { diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c index 9dcf5d89668..9b7ccb32869 100644 --- a/drivers/misc/sgi-gru/gruhandles.c +++ b/drivers/misc/sgi-gru/gruhandles.c @@ -72,42 +72,16 @@ static int wait_instruction_complete(void *h, enum mcs_op opc) return status; } -#if defined CONFIG_IA64 -static void cch_allocate_set_asids( - struct gru_context_configuration_handle *cch, int asidval) +int cch_allocate(struct gru_context_configuration_handle *cch, + int asidval, int sizeavail, unsigned long cbrmap, + unsigned long dsrmap) { int i; for (i = 0; i < 8; i++) { cch->asid[i] = (asidval++); -#if 0 - /* ZZZ hugepages not supported yet */ - if (i == RGN_HPAGE) - cch->sizeavail[i] = GRU_SIZEAVAIL(hpage_shift); - else -#endif - cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT); + cch->sizeavail[i] = sizeavail; } -} -#elif defined CONFIG_X86_64 -static void cch_allocate_set_asids( - struct gru_context_configuration_handle *cch, int asidval) -{ - int i; - - for (i = 0; i < 8; i++) { - cch->asid[i] = asidval++; - cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT) | - GRU_SIZEAVAIL(21); - } -} -#endif - -int cch_allocate(struct gru_context_configuration_handle *cch, - int asidval, unsigned long cbrmap, - unsigned long dsrmap) -{ - cch_allocate_set_asids(cch, asidval); cch->dsr_allocation_map = dsrmap; cch->cbr_allocation_map = cbrmap; cch->opc = CCHOP_ALLOCATE; diff --git a/drivers/misc/sgi-gru/gruhandles.h b/drivers/misc/sgi-gru/gruhandles.h index dfce8ea5da2..1ed74d7508c 100644 --- a/drivers/misc/sgi-gru/gruhandles.h +++ b/drivers/misc/sgi-gru/gruhandles.h @@ -496,7 +496,7 @@ enum gru_cbr_state { #define GRUMAXINVAL 1024UL int cch_allocate(struct gru_context_configuration_handle *cch, - int asidval, unsigned long cbrmap, unsigned long dsrmap); + int asidval, int sizeavail, unsigned long cbrmap, unsigned long dsrmap); int cch_start(struct gru_context_configuration_handle *cch); int cch_interrupt(struct gru_context_configuration_handle *cch); diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c index 6b1efe44727..d8bd7d84a7c 100644 --- a/drivers/misc/sgi-gru/grukservices.c +++ b/drivers/misc/sgi-gru/grukservices.c @@ -672,7 +672,7 @@ int gru_kservices_init(struct gru_state *gru) cch->tlb_int_enable = 0; cch->tfm_done_bit_enable = 0; cch->unmap_enable = 1; - err = cch_allocate(cch, 0, cbr_map, dsr_map); + err = cch_allocate(cch, 0, 0, cbr_map, dsr_map); if (err) { gru_dbg(grudev, "Unable to allocate kernel CCH: gid %d, err %d\n", diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index 5fc7b5ecde6..ec3f7a17d22 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -326,6 +326,7 @@ static struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma, gts->ts_vma = vma; gts->ts_tlb_int_select = -1; gts->ts_gms = gru_register_mmu_notifier(); + gts->ts_sizeavail = GRU_SIZEAVAIL(PAGE_SHIFT); if (!gts->ts_gms) goto err; @@ -552,7 +553,8 @@ static void gru_load_context(struct gru_thread_state *gts) cch->tlb_int_select = gts->ts_tlb_int_select; } cch->tfm_done_bit_enable = 0; - err = cch_allocate(cch, asid, gts->ts_cbr_map, gts->ts_dsr_map); + err = cch_allocate(cch, asid, gts->ts_sizeavail, gts->ts_cbr_map, + gts->ts_dsr_map); if (err) { gru_dbg(grudev, "err %d: cch %p, gts %p, cbr 0x%lx, dsr 0x%lx\n", @@ -573,11 +575,12 @@ static void gru_load_context(struct gru_thread_state *gts) /* * Update fields in an active CCH: * - retarget interrupts on local blade + * - update sizeavail mask * - force a delayed context unload by clearing the CCH asids. This * forces TLB misses for new GRU instructions. The context is unloaded * when the next TLB miss occurs. */ -static int gru_update_cch(struct gru_thread_state *gts, int int_select) +int gru_update_cch(struct gru_thread_state *gts, int force_unload) { struct gru_context_configuration_handle *cch; struct gru_state *gru = gts->ts_gru; @@ -591,9 +594,11 @@ static int gru_update_cch(struct gru_thread_state *gts, int int_select) goto exit; if (cch_interrupt(cch)) BUG(); - if (int_select >= 0) { - gts->ts_tlb_int_select = int_select; - cch->tlb_int_select = int_select; + if (!force_unload) { + for (i = 0; i < 8; i++) + cch->sizeavail[i] = gts->ts_sizeavail; + gts->ts_tlb_int_select = gru_cpu_fault_map_id(); + cch->tlb_int_select = gru_cpu_fault_map_id(); } else { for (i = 0; i < 8; i++) cch->asid[i] = 0; @@ -625,7 +630,7 @@ static int gru_retarget_intr(struct gru_thread_state *gts) gru_dbg(grudev, "retarget from %d to %d\n", gts->ts_tlb_int_select, gru_cpu_fault_map_id()); - return gru_update_cch(gts, gru_cpu_fault_map_id()); + return gru_update_cch(gts, 0); } diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index 8e5c8e5bb53..bf1eeb7553e 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -361,6 +361,7 @@ struct gru_thread_state { long ts_user_options;/* misc user option flags */ pid_t ts_tgid_owner; /* task that is using the context - for migration */ + unsigned short ts_sizeavail; /* Pagesizes in use */ int ts_tsid; /* thread that owns the structure */ int ts_tlb_int_select;/* target cpu if interrupts @@ -374,6 +375,7 @@ struct gru_thread_state { required for contest */ char ts_blade; /* If >= 0, migrate context if ref from diferent blade */ + char ts_force_cch_reload; char ts_force_unload;/* force context to be unloaded after migration */ char ts_cbr_idx[GRU_CBR_AU];/* CBR numbers of each @@ -597,6 +599,7 @@ extern struct gru_thread_state *gru_find_thread_state(struct vm_area_struct extern struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct *vma, int tsid); extern void gru_unload_context(struct gru_thread_state *gts, int savestate); +extern int gru_update_cch(struct gru_thread_state *gts, int force_unload); extern void gts_drop(struct gru_thread_state *gts); extern void gru_tgh_flush_init(struct gru_state *gru); extern int gru_kservices_init(struct gru_state *gru); -- cgit v1.2.3