diff options
author | Linus Torvalds <torvalds@woody.linux-foundation.org> | 2007-07-20 12:02:20 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@woody.linux-foundation.org> | 2007-07-20 12:02:20 -0700 |
commit | efa7e8673c78cc6de2d6c367eb6f50449c57ed90 (patch) | |
tree | 2759535bd50e6d6cc4801a02106ee26f27aa610f /arch/ia64/kernel/iosapic.c | |
parent | 02d6112cd7d75bf30685ee4b2e9dc0291b4d3133 (diff) | |
parent | bd807f9c5b90c236c3d173a5815d620bc1b1316d (diff) |
Merge branch 'release' of git://git.kernel.org/pub/scm/linux/kernel/git/aegl/linux-2.6
* 'release' of git://git.kernel.org/pub/scm/linux/kernel/git/aegl/linux-2.6:
[IA64] Prevent people from directly including <asm/rwsem.h>.
[IA64] remove time interpolator
[IA64] Convert to generic timekeeping/clocksource
[IA64] refresh some config files for 64K pagesize
[IA64] Delete iosapic_free_rte()
[IA64] fallocate system call
[IA64] Enable percpu vector domain for IA64_DIG
[IA64] Enable percpu vector domain for IA64_GENERIC
[IA64] Support irq migration across domain
[IA64] Add support for vector domain
[IA64] Add mapping table between irq and vector
[IA64] Check if irq is sharable
[IA64] Fix invalid irq vector assumption for iosapic
[IA64] Use dynamic irq for iosapic interrupts
[IA64] Use per iosapic lock for indirect iosapic register access
[IA64] Cleanup lock order in iosapic_register_intr
[IA64] Remove duplicated members in iosapic_rte_info
[IA64] Remove block structure for locking in iosapic.c
Diffstat (limited to 'arch/ia64/kernel/iosapic.c')
-rw-r--r-- | arch/ia64/kernel/iosapic.c | 652 |
1 files changed, 298 insertions, 354 deletions
diff --git a/arch/ia64/kernel/iosapic.c b/arch/ia64/kernel/iosapic.c index 37f46527d23..91e6dc1e7ba 100644 --- a/arch/ia64/kernel/iosapic.c +++ b/arch/ia64/kernel/iosapic.c @@ -118,15 +118,25 @@ static DEFINE_SPINLOCK(iosapic_lock); * vector. */ -struct iosapic_rte_info { - struct list_head rte_list; /* node in list of RTEs sharing the - * same vector */ +#define NO_REF_RTE 0 + +static struct iosapic { char __iomem *addr; /* base address of IOSAPIC */ - unsigned int gsi_base; /* first GSI assigned to this - * IOSAPIC */ + unsigned int gsi_base; /* GSI base */ + unsigned short num_rte; /* # of RTEs on this IOSAPIC */ + int rtes_inuse; /* # of RTEs in use on this IOSAPIC */ +#ifdef CONFIG_NUMA + unsigned short node; /* numa node association via pxm */ +#endif + spinlock_t lock; /* lock for indirect reg access */ +} iosapic_lists[NR_IOSAPICS]; + +struct iosapic_rte_info { + struct list_head rte_list; /* RTEs sharing the same vector */ char rte_index; /* IOSAPIC RTE index */ int refcnt; /* reference counter */ unsigned int flags; /* flags */ + struct iosapic *iosapic; } ____cacheline_aligned; static struct iosapic_intr_info { @@ -140,24 +150,23 @@ static struct iosapic_intr_info { unsigned char polarity: 1; /* interrupt polarity * (see iosapic.h) */ unsigned char trigger : 1; /* trigger mode (see iosapic.h) */ -} iosapic_intr_info[IA64_NUM_VECTORS]; - -static struct iosapic { - char __iomem *addr; /* base address of IOSAPIC */ - unsigned int gsi_base; /* first GSI assigned to this - * IOSAPIC */ - unsigned short num_rte; /* # of RTEs on this IOSAPIC */ - int rtes_inuse; /* # of RTEs in use on this IOSAPIC */ -#ifdef CONFIG_NUMA - unsigned short node; /* numa node association via pxm */ -#endif -} iosapic_lists[NR_IOSAPICS]; +} iosapic_intr_info[NR_IRQS]; static unsigned char pcat_compat __devinitdata; /* 8259 compatibility flag */ static int iosapic_kmalloc_ok; static LIST_HEAD(free_rte_list); +static inline void +iosapic_write(struct iosapic *iosapic, unsigned int reg, u32 val) +{ + unsigned long flags; + + spin_lock_irqsave(&iosapic->lock, flags); + __iosapic_write(iosapic->addr, reg, val); + spin_unlock_irqrestore(&iosapic->lock, flags); +} + /* * Find an IOSAPIC associated with a GSI */ @@ -175,17 +184,18 @@ find_iosapic (unsigned int gsi) return -1; } -static inline int -_gsi_to_vector (unsigned int gsi) +static inline int __gsi_to_irq(unsigned int gsi) { + int irq; struct iosapic_intr_info *info; struct iosapic_rte_info *rte; - for (info = iosapic_intr_info; info < - iosapic_intr_info + IA64_NUM_VECTORS; ++info) + for (irq = 0; irq < NR_IRQS; irq++) { + info = &iosapic_intr_info[irq]; list_for_each_entry(rte, &info->rtes, rte_list) - if (rte->gsi_base + rte->rte_index == gsi) - return info - iosapic_intr_info; + if (rte->iosapic->gsi_base + rte->rte_index == gsi) + return irq; + } return -1; } @@ -196,7 +206,10 @@ _gsi_to_vector (unsigned int gsi) inline int gsi_to_vector (unsigned int gsi) { - return _gsi_to_vector(gsi); + int irq = __gsi_to_irq(gsi); + if (check_irq_used(irq) < 0) + return -1; + return irq_to_vector(irq); } int @@ -204,66 +217,48 @@ gsi_to_irq (unsigned int gsi) { unsigned long flags; int irq; - /* - * XXX fix me: this assumes an identity mapping between IA-64 vector - * and Linux irq numbers... - */ + spin_lock_irqsave(&iosapic_lock, flags); - { - irq = _gsi_to_vector(gsi); - } + irq = __gsi_to_irq(gsi); spin_unlock_irqrestore(&iosapic_lock, flags); - return irq; } -static struct iosapic_rte_info *gsi_vector_to_rte(unsigned int gsi, - unsigned int vec) +static struct iosapic_rte_info *find_rte(unsigned int irq, unsigned int gsi) { struct iosapic_rte_info *rte; - list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) - if (rte->gsi_base + rte->rte_index == gsi) + list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) + if (rte->iosapic->gsi_base + rte->rte_index == gsi) return rte; return NULL; } static void -set_rte (unsigned int gsi, unsigned int vector, unsigned int dest, int mask) +set_rte (unsigned int gsi, unsigned int irq, unsigned int dest, int mask) { unsigned long pol, trigger, dmode; u32 low32, high32; - char __iomem *addr; int rte_index; char redir; struct iosapic_rte_info *rte; + ia64_vector vector = irq_to_vector(irq); DBG(KERN_DEBUG"IOSAPIC: routing vector %d to 0x%x\n", vector, dest); - rte = gsi_vector_to_rte(gsi, vector); + rte = find_rte(irq, gsi); if (!rte) return; /* not an IOSAPIC interrupt */ rte_index = rte->rte_index; - addr = rte->addr; - pol = iosapic_intr_info[vector].polarity; - trigger = iosapic_intr_info[vector].trigger; - dmode = iosapic_intr_info[vector].dmode; + pol = iosapic_intr_info[irq].polarity; + trigger = iosapic_intr_info[irq].trigger; + dmode = iosapic_intr_info[irq].dmode; redir = (dmode == IOSAPIC_LOWEST_PRIORITY) ? 1 : 0; #ifdef CONFIG_SMP - { - unsigned int irq; - - for (irq = 0; irq < NR_IRQS; ++irq) - if (irq_to_vector(irq) == vector) { - set_irq_affinity_info(irq, - (int)(dest & 0xffff), - redir); - break; - } - } + set_irq_affinity_info(irq, (int)(dest & 0xffff), redir); #endif low32 = ((pol << IOSAPIC_POLARITY_SHIFT) | @@ -275,10 +270,10 @@ set_rte (unsigned int gsi, unsigned int vector, unsigned int dest, int mask) /* dest contains both id and eid */ high32 = (dest << IOSAPIC_DEST_SHIFT); - iosapic_write(addr, IOSAPIC_RTE_HIGH(rte_index), high32); - iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32); - iosapic_intr_info[vector].low32 = low32; - iosapic_intr_info[vector].dest = dest; + iosapic_write(rte->iosapic, IOSAPIC_RTE_HIGH(rte_index), high32); + iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32); + iosapic_intr_info[irq].low32 = low32; + iosapic_intr_info[irq].dest = dest; } static void @@ -294,15 +289,18 @@ kexec_disable_iosapic(void) { struct iosapic_intr_info *info; struct iosapic_rte_info *rte; - u8 vec = 0; - for (info = iosapic_intr_info; info < - iosapic_intr_info + IA64_NUM_VECTORS; ++info, ++vec) { + ia64_vector vec; + int irq; + + for (irq = 0; irq < NR_IRQS; irq++) { + info = &iosapic_intr_info[irq]; + vec = irq_to_vector(irq); list_for_each_entry(rte, &info->rtes, rte_list) { - iosapic_write(rte->addr, + iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte->rte_index), IOSAPIC_MASK|vec); - iosapic_eoi(rte->addr, vec); + iosapic_eoi(rte->iosapic->addr, vec); } } } @@ -311,54 +309,36 @@ kexec_disable_iosapic(void) static void mask_irq (unsigned int irq) { - unsigned long flags; - char __iomem *addr; u32 low32; int rte_index; - ia64_vector vec = irq_to_vector(irq); struct iosapic_rte_info *rte; - if (list_empty(&iosapic_intr_info[vec].rtes)) + if (list_empty(&iosapic_intr_info[irq].rtes)) return; /* not an IOSAPIC interrupt! */ - spin_lock_irqsave(&iosapic_lock, flags); - { - /* set only the mask bit */ - low32 = iosapic_intr_info[vec].low32 |= IOSAPIC_MASK; - list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, - rte_list) { - addr = rte->addr; - rte_index = rte->rte_index; - iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32); - } + /* set only the mask bit */ + low32 = iosapic_intr_info[irq].low32 |= IOSAPIC_MASK; + list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) { + rte_index = rte->rte_index; + iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32); } - spin_unlock_irqrestore(&iosapic_lock, flags); } static void unmask_irq (unsigned int irq) { - unsigned long flags; - char __iomem *addr; u32 low32; int rte_index; - ia64_vector vec = irq_to_vector(irq); struct iosapic_rte_info *rte; - if (list_empty(&iosapic_intr_info[vec].rtes)) + if (list_empty(&iosapic_intr_info[irq].rtes)) return; /* not an IOSAPIC interrupt! */ - spin_lock_irqsave(&iosapic_lock, flags); - { - low32 = iosapic_intr_info[vec].low32 &= ~IOSAPIC_MASK; - list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, - rte_list) { - addr = rte->addr; - rte_index = rte->rte_index; - iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32); - } + low32 = iosapic_intr_info[irq].low32 &= ~IOSAPIC_MASK; + list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) { + rte_index = rte->rte_index; + iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32); } - spin_unlock_irqrestore(&iosapic_lock, flags); } @@ -366,23 +346,24 @@ static void iosapic_set_affinity (unsigned int irq, cpumask_t mask) { #ifdef CONFIG_SMP - unsigned long flags; u32 high32, low32; int dest, rte_index; - char __iomem *addr; int redir = (irq & IA64_IRQ_REDIRECTED) ? 1 : 0; - ia64_vector vec; struct iosapic_rte_info *rte; + struct iosapic *iosapic; irq &= (~IA64_IRQ_REDIRECTED); - vec = irq_to_vector(irq); + cpus_and(mask, mask, cpu_online_map); if (cpus_empty(mask)) return; + if (reassign_irq_vector(irq, first_cpu(mask))) + return; + dest = cpu_physical_id(first_cpu(mask)); - if (list_empty(&iosapic_intr_info[vec].rtes)) + if (list_empty(&iosapic_intr_info[irq].rtes)) return; /* not an IOSAPIC interrupt */ set_irq_affinity_info(irq, dest, redir); @@ -390,31 +371,24 @@ iosapic_set_affinity (unsigned int irq, cpumask_t mask) /* dest contains both id and eid */ high32 = dest << IOSAPIC_DEST_SHIFT; - spin_lock_irqsave(&iosapic_lock, flags); - { - low32 = iosapic_intr_info[vec].low32 & - ~(7 << IOSAPIC_DELIVERY_SHIFT); - - if (redir) - /* change delivery mode to lowest priority */ - low32 |= (IOSAPIC_LOWEST_PRIORITY << - IOSAPIC_DELIVERY_SHIFT); - else - /* change delivery mode to fixed */ - low32 |= (IOSAPIC_FIXED << IOSAPIC_DELIVERY_SHIFT); - - iosapic_intr_info[vec].low32 = low32; - iosapic_intr_info[vec].dest = dest; - list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, - rte_list) { - addr = rte->addr; - rte_index = rte->rte_index; - iosapic_write(addr, IOSAPIC_RTE_HIGH(rte_index), - high32); - iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32); - } + low32 = iosapic_intr_info[irq].low32 & ~(7 << IOSAPIC_DELIVERY_SHIFT); + if (redir) + /* change delivery mode to lowest priority */ + low32 |= (IOSAPIC_LOWEST_PRIORITY << IOSAPIC_DELIVERY_SHIFT); + else + /* change delivery mode to fixed */ + low32 |= (IOSAPIC_FIXED << IOSAPIC_DELIVERY_SHIFT); + low32 &= IOSAPIC_VECTOR_MASK; + low32 |= irq_to_vector(irq); + + iosapic_intr_info[irq].low32 = low32; + iosapic_intr_info[irq].dest = dest; + list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) { + iosapic = rte->iosapic; + rte_index = rte->rte_index; + iosapic_write(iosapic, IOSAPIC_RTE_HIGH(rte_index), high32); + iosapic_write(iosapic, IOSAPIC_RTE_LOW(rte_index), low32); } - spin_unlock_irqrestore(&iosapic_lock, flags); #endif } @@ -434,10 +408,20 @@ iosapic_end_level_irq (unsigned int irq) { ia64_vector vec = irq_to_vector(irq); struct iosapic_rte_info *rte; + int do_unmask_irq = 0; - move_native_irq(irq); - list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) - iosapic_eoi(rte->addr, vec); + if (unlikely(irq_desc[irq].status & IRQ_MOVE_PENDING)) { + do_unmask_irq = 1; + mask_irq(irq); + } + + list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) + iosapic_eoi(rte->iosapic->addr, vec); + + if (unlikely(do_unmask_irq)) { + move_masked_irq(irq); + unmask_irq(irq); + } } #define iosapic_shutdown_level_irq mask_irq @@ -519,13 +503,12 @@ iosapic_version (char __iomem *addr) * unsigned int reserved2 : 8; * } */ - return iosapic_read(addr, IOSAPIC_VERSION); + return __iosapic_read(addr, IOSAPIC_VERSION); } -static int iosapic_find_sharable_vector (unsigned long trigger, - unsigned long pol) +static int iosapic_find_sharable_irq(unsigned long trigger, unsigned long pol) { - int i, vector = -1, min_count = -1; + int i, irq = -ENOSPC, min_count = -1; struct iosapic_intr_info *info; /* @@ -533,21 +516,21 @@ static int iosapic_find_sharable_vector (unsigned long trigger, * supported yet */ if (trigger == IOSAPIC_EDGE) - return -1; + return -EINVAL; - for (i = IA64_FIRST_DEVICE_VECTOR; i <= IA64_LAST_DEVICE_VECTOR; i++) { + for (i = 0; i <= NR_IRQS; i++) { info = &iosapic_intr_info[i]; if (info->trigger == trigger && info->polarity == pol && - (info->dmode == IOSAPIC_FIXED || info->dmode == - IOSAPIC_LOWEST_PRIORITY)) { + (info->dmode == IOSAPIC_FIXED || + info->dmode == IOSAPIC_LOWEST_PRIORITY) && + can_request_irq(i, IRQF_SHARED)) { if (min_count == -1 || info->count < min_count) { - vector = i; + irq = i; min_count = info->count; } } } - - return vector; + return irq; } /* @@ -555,25 +538,25 @@ static int iosapic_find_sharable_vector (unsigned long trigger, * assign a new vector for the other and make the vector available */ static void __init -iosapic_reassign_vector (int vector) +iosapic_reassign_vector (int irq) { - int new_vector; + int new_irq; - if (!list_empty(&iosapic_intr_info[vector].rtes)) { - new_vector = assign_irq_vector(AUTO_ASSIGN); - if (new_vector < 0) + if (!list_empty(&iosapic_intr_info[irq].rtes)) { + new_irq = create_irq(); + if (new_irq < 0) panic("%s: out of interrupt vectors!\n", __FUNCTION__); printk(KERN_INFO "Reassigning vector %d to %d\n", - vector, new_vector); - memcpy(&iosapic_intr_info[new_vector], &iosapic_intr_info[vector], + irq_to_vector(irq), irq_to_vector(new_irq)); + memcpy(&iosapic_intr_info[new_irq], &iosapic_intr_info[irq], sizeof(struct iosapic_intr_info)); - INIT_LIST_HEAD(&iosapic_intr_info[new_vector].rtes); - list_move(iosapic_intr_info[vector].rtes.next, - &iosapic_intr_info[new_vector].rtes); - memset(&iosapic_intr_info[vector], 0, + INIT_LIST_HEAD(&iosapic_intr_info[new_irq].rtes); + list_move(iosapic_intr_info[irq].rtes.next, + &iosapic_intr_info[new_irq].rtes); + memset(&iosapic_intr_info[irq], 0, sizeof(struct iosapic_intr_info)); - iosapic_intr_info[vector].low32 = IOSAPIC_MASK; - INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes); + iosapic_intr_info[irq].low32 = IOSAPIC_MASK; + INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes); } } @@ -610,29 +593,18 @@ static struct iosapic_rte_info *iosapic_alloc_rte (void) return rte; } -static void iosapic_free_rte (struct iosapic_rte_info *rte) +static inline int irq_is_shared (int irq) { - if (rte->flags & RTE_PREALLOCATED) - list_add_tail(&rte->rte_list, &free_rte_list); - else - kfree(rte); -} - -static inline int vector_is_shared (int vector) -{ - return (iosapic_intr_info[vector].count > 1); + return (iosapic_intr_info[irq].count > 1); } static int -register_intr (unsigned int gsi, int vector, unsigned char delivery, +register_intr (unsigned int gsi, int irq, unsigned char delivery, unsigned long polarity, unsigned long trigger) { irq_desc_t *idesc; struct hw_interrupt_type *irq_type; - int rte_index; int index; - unsigned long gsi_base; - void __iomem *iosapic_address; struct iosapic_rte_info *rte; index = find_iosapic(gsi); @@ -642,10 +614,7 @@ register_intr (unsigned int gsi, int vector, unsigned char delivery, return -ENODEV; } - iosapic_address = iosapic_lists[index].addr; - gsi_base = iosapic_lists[index].gsi_base; - - rte = gsi_vector_to_rte(gsi, vector); + rte = find_rte(irq, gsi); if (!rte) { rte = iosapic_alloc_rte(); if (!rte) { @@ -654,40 +623,42 @@ register_intr (unsigned int gsi, int vector, unsigned char delivery, return -ENOMEM; } - rte_index = gsi - gsi_base; - rte->rte_index = rte_index; - rte->addr = iosapic_address; - rte->gsi_base = gsi_base; + rte->iosapic = &iosapic_lists[index]; + rte->rte_index = gsi - rte->iosapic->gsi_base; rte->refcnt++; - list_add_tail(&rte->rte_list, &iosapic_intr_info[vector].rtes); - iosapic_intr_info[vector].count++; + list_add_tail(&rte->rte_list, &iosapic_intr_info[irq].rtes); + iosapic_intr_info[irq].count++; iosapic_lists[index].rtes_inuse++; } - else if (vector_is_shared(vector)) { - struct iosapic_intr_info *info = &iosapic_intr_info[vector]; - if (info->trigger != trigger || info->polarity != polarity) { + else if (rte->refcnt == NO_REF_RTE) { + struct iosapic_intr_info *info = &iosapic_intr_info[irq]; + if (info->count > 0 && + (info->trigger != trigger || info->polarity != polarity)){ printk (KERN_WARNING "%s: cannot override the interrupt\n", __FUNCTION__); return -EINVAL; } + rte->refcnt++; + iosapic_intr_info[irq].count++; + iosapic_lists[index].rtes_inuse++; } - iosapic_intr_info[vector].polarity = polarity; - iosapic_intr_info[vector].dmode = delivery; - iosapic_intr_info[vector].trigger = trigger; + iosapic_intr_info[irq].polarity = polarity; + iosapic_intr_info[irq].dmode = delivery; + iosapic_intr_info[irq].trigger = trigger; if (trigger == IOSAPIC_EDGE) irq_type = &irq_type_iosapic_edge; else irq_type = &irq_type_iosapic_level; - idesc = irq_desc + vector; + idesc = irq_desc + irq; if (idesc->chip != irq_type) { if (idesc->chip != &no_irq_type) printk(KERN_WARNING "%s: changing vector %d from %s to %s\n", - __FUNCTION__, vector, + __FUNCTION__, irq_to_vector(irq), idesc->chip->name, irq_type->name); idesc->chip = irq_type; } @@ -695,18 +666,19 @@ register_intr (unsigned int gsi, int vector, unsigned char delivery, } static unsigned int -get_target_cpu (unsigned int gsi, int vector) +get_target_cpu (unsigned int gsi, int irq) { #ifdef CONFIG_SMP static int cpu = -1; extern int cpe_vector; + cpumask_t domain = irq_to_domain(irq); /* * In case of vector shared by multiple RTEs, all RTEs that * share the vector need to use the same destination CPU. */ - if (!list_empty(&iosapic_intr_info[vector].rtes)) - return iosapic_intr_info[vector].dest; + if (!list_empty(&iosapic_intr_info[irq].rtes)) + return iosapic_intr_info[irq].dest; /* * If the platform supports redirection via XTP, let it @@ -723,7 +695,7 @@ get_target_cpu (unsigned int gsi, int vector) return cpu_physical_id(smp_processor_id()); #ifdef CONFIG_ACPI - if (cpe_vector > 0 && vector == IA64_CPEP_VECTOR) + if (cpe_vector > 0 && irq_to_vector(irq) == IA64_CPEP_VECTOR) return get_cpei_target_cpu(); #endif @@ -738,7 +710,7 @@ get_target_cpu (unsigned int gsi, int vector) goto skip_numa_setup; cpu_mask = node_to_cpumask(iosapic_lists[iosapic_index].node); - + cpus_and(cpu_mask, cpu_mask, domain); for_each_cpu_mask(numa_cpu, cpu_mask) { if (!cpu_online(numa_cpu)) cpu_clear(numa_cpu, cpu_mask); @@ -749,8 +721,8 @@ get_target_cpu (unsigned int gsi, int vector) if (!num_cpus) goto skip_numa_setup; - /* Use vector assignment to distribute across cpus in node */ - cpu_index = vector % num_cpus; + /* Use irq assignment to distribute across cpus in node */ + cpu_index = irq % num_cpus; for (numa_cpu = first_cpu(cpu_mask) ; i < cpu_index ; i++) numa_cpu = next_cpu(numa_cpu, cpu_mask); @@ -768,7 +740,7 @@ skip_numa_setup: do { if (++cpu >= NR_CPUS) cpu = 0; - } while (!cpu_online(cpu)); + } while (!cpu_online(cpu) || !cpu_isset(cpu, domain)); return cpu_physical_id(cpu); #else /* CONFIG_SMP */ @@ -785,84 +757,72 @@ int iosapic_register_intr (unsigned int gsi, unsigned long polarity, unsigned long trigger) { - int vector, mask = 1, err; + int irq, mask = 1, err; unsigned int dest; unsigned long flags; struct iosapic_rte_info *rte; u32 low32; -again: + /* * If this GSI has already been registered (i.e., it's a * shared interrupt, or we lost a race to register it), * don't touch the RTE. */ spin_lock_irqsave(&iosapic_lock, flags); - { - vector = gsi_to_vector(gsi); - if (vector > 0) { - rte = gsi_vector_to_rte(gsi, vector); + irq = __gsi_to_irq(gsi); + if (irq > 0) { + rte = find_rte(irq, gsi); + if(iosapic_intr_info[irq].count == 0) { + assign_irq_vector(irq); + dynamic_irq_init(irq); + } else if (rte->refcnt != NO_REF_RTE) { rte->refcnt++; - spin_unlock_irqrestore(&iosapic_lock, flags); - return vector; + goto unlock_iosapic_lock; } - } - spin_unlock_irqrestore(&iosapic_lock, flags); + } else + irq = create_irq(); /* If vector is running out, we try to find a sharable vector */ - vector = assign_irq_vector(AUTO_ASSIGN); - if (vector < 0) { - vector = iosapic_find_sharable_vector(trigger, polarity); - if (vector < 0) - return -ENOSPC; + if (irq < 0) { + irq = iosapic_find_sharable_irq(trigger, polarity); + if (irq < 0) + goto unlock_iosapic_lock; } - spin_lock_irqsave(&irq_desc[vector].lock, flags); - spin_lock(&iosapic_lock); - { - if (gsi_to_vector(gsi) > 0) { - if (list_empty(&iosapic_intr_info[vector].rtes)) - free_irq_vector(vector); - spin_unlock(&iosapic_lock); - spin_unlock_irqrestore(&irq_desc[vector].lock, - flags); - goto again; - } - - dest = get_target_cpu(gsi, vector); - err = register_intr(gsi, vector, IOSAPIC_LOWEST_PRIORITY, - polarity, trigger); - if (err < 0) { - spin_unlock(&iosapic_lock); - spin_unlock_irqrestore(&irq_desc[vector].lock, - flags); - return err; - } - - /* - * If the vector is shared and already unmasked for - * other interrupt sources, don't mask it. - */ - low32 = iosapic_intr_info[vector].low32; - if (vector_is_shared(vector) && !(low32 & IOSAPIC_MASK)) - mask = 0; - set_rte(gsi, vector, dest, mask); + spin_lock(&irq_desc[irq].lock); + dest = get_target_cpu(gsi, irq); + err = register_intr(gsi, irq, IOSAPIC_LOWEST_PRIORITY, + polarity, trigger); + if (err < 0) { + irq = err; + goto unlock_all; } - spin_unlock(&iosapic_lock); - spin_unlock_irqrestore(&irq_desc[vector].lock, flags); + + /* + * If the vector is shared and already unmasked for other + * interrupt sources, don't mask it. + */ + low32 = iosapic_intr_info[irq].low32; + if (irq_is_shared(irq) && !(low32 & IOSAPIC_MASK)) + mask = 0; + set_rte(gsi, irq, dest, mask); printk(KERN_INFO "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d\n", gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"), (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), - cpu_logical_id(dest), dest, vector); - - return vector; + cpu_logical_id(dest), dest, irq_to_vector(irq)); + unlock_all: + spin_unlock(&irq_desc[irq].lock); + unlock_iosapic_lock: + spin_unlock_irqrestore(&iosapic_lock, flags); + return irq; } void iosapic_unregister_intr (unsigned int gsi) { unsigned long flags; - int irq, vector, index; + int irq, index; irq_desc_t *idesc; u32 low32; unsigned long trigger, polarity; @@ -881,78 +841,56 @@ iosapic_unregister_intr (unsigned int gsi) WARN_ON(1); return; } - vector = irq_to_vector(irq); - idesc = irq_desc + irq; - spin_lock_irqsave(&idesc->lock, flags); - spin_lock(&iosapic_lock); - { - if ((rte = gsi_vector_to_rte(gsi, vector)) == NULL) { - printk(KERN_ERR - "iosapic_unregister_intr(%u) unbalanced\n", - gsi); - WARN_ON(1); - goto out; - } + spin_lock_irqsave(&iosapic_lock, flags); + if ((rte = find_rte(irq, gsi)) == NULL) { + printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n", + gsi); + WARN_ON(1); + goto out; + } - if (--rte->refcnt > 0) - goto out; + if (--rte->refcnt > 0) + goto out; - /* Mask the interrupt */ - low32 = iosapic_intr_info[vector].low32 | IOSAPIC_MASK; - iosapic_write(rte->addr, IOSAPIC_RTE_LOW(rte->rte_index), - low32); + idesc = irq_desc + irq; + rte->refcnt = NO_REF_RTE; - /* Remove the rte entry from the list */ - list_del(&rte->rte_list); - iosapic_intr_info[vector].count--; - iosapic_free_rte(rte); - index = find_iosapic(gsi); - iosapic_lists[index].rtes_inuse--; - WARN_ON(iosapic_lists[index].rtes_inuse < 0); - - trigger = iosapic_intr_info[vector].trigger; - polarity = iosapic_intr_info[vector].polarity; - dest = iosapic_intr_info[vector].dest; - printk(KERN_INFO - "GSI %u (%s, %s) -> CPU %d (0x%04x)" - " vector %d unregistered\n", - gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"), - (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), - cpu_logical_id(dest), dest, vector); + /* Mask the interrupt */ + low32 = iosapic_intr_info[irq].low32 | IOSAPIC_MASK; + iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte->rte_index), low32); - if (list_empty(&iosapic_intr_info[vector].rtes)) { - /* Sanity check */ - BUG_ON(iosapic_intr_info[vector].count); + iosapic_intr_info[irq].count--; + index = find_iosapic(gsi); + iosapic_lists[index].rtes_inuse--; + WARN_ON(iosapic_lists[index].rtes_inuse < 0); - /* Clear the interrupt controller descriptor */ - idesc->chip = &no_irq_type; + trigger = iosapic_intr_info[irq].trigger; + polarity = iosapic_intr_info[irq].polarity; + dest = iosapic_intr_info[irq].dest; + printk(KERN_INFO + "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d unregistered\n", + gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"), + (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), + cpu_logical_id(dest), dest, irq_to_vector(irq)); + if (iosapic_intr_info[irq].count == 0) { #ifdef CONFIG_SMP - /* Clear affinity */ - cpus_setall(idesc->affinity); + /* Clear affinity */ + cpus_setall(idesc->affinity); #endif - - /* Clear the interrupt information */ - memset(&iosapic_intr_info[vector], 0, - sizeof(struct iosapic_intr_info)); - iosapic_intr_info[vector].low32 |= IOSAPIC_MASK; - INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes); - - if (idesc->action) { - printk(KERN_ERR - "interrupt handlers still exist on" - "IRQ %u\n", irq); - WARN_ON(1); - } - - /* Free the interrupt vector */ - free_irq_vector(vector); - } + /* Clear the interrupt information */ + iosapic_intr_info[irq].dest = 0; + iosapic_intr_info[irq].dmode = 0; + iosapic_intr_info[irq].polarity = 0; + iosapic_intr_info[irq].trigger = 0; + iosapic_intr_info[irq].low32 |= IOSAPIC_MASK; + + /* Destroy and reserve IRQ */ + destroy_and_reserve_irq(irq); } out: - spin_unlock(&iosapic_lock); - spin_unlock_irqrestore(&idesc->lock, flags); + spin_unlock_irqrestore(&iosapic_lock, flags); } /* @@ -965,27 +903,30 @@ iosapic_register_platform_intr (u32 int_type, unsigned int gsi, { static const char * const name[] = {"unknown", "PMI", "INIT", "CPEI"}; unsigned char delivery; - int vector, mask = 0; + int irq, vector, mask = 0; unsigned int dest = ((id << 8) | eid) & 0xffff; switch (int_type) { case ACPI_INTERRUPT_PMI: - vector = iosapic_vector; + irq = vector = iosapic_vector; + bind_irq_vector(irq, vector, CPU_MASK_ALL); /* * since PMI vector is alloc'd by FW(ACPI) not by kernel, * we need to make sure the vector is available */ - iosapic_reassign_vector(vector); + iosapic_reassign_vector(irq); delivery = IOSAPIC_PMI; break; case ACPI_INTERRUPT_INIT: - vector = assign_irq_vector(AUTO_ASSIGN); - if (vector < 0) + irq = create_irq(); + if (irq < 0) panic("%s: out of interrupt vectors!\n", __FUNCTION__); + vector = irq_to_vector(irq); delivery = IOSAPIC_INIT; break; case ACPI_INTERRUPT_CPEI: - vector = IA64_CPE_VECTOR; + irq = vector = IA64_CPE_VECTOR; + BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL)); delivery = IOSAPIC_LOWEST_PRIORITY; mask = 1; break; @@ -995,7 +936,7 @@ iosapic_register_platform_intr (u32 int_type, unsigned int gsi, return -1; } - register_intr(gsi, vector, delivery, polarity, trigger); + register_intr(gsi, irq, delivery, polarity, trigger); printk(KERN_INFO "PLATFORM int %s (0x%x): GSI %u (%s, %s) -> CPU %d (0x%04x)" @@ -1005,7 +946,7 @@ iosapic_register_platform_intr (u32 int_type, unsigned int gsi, (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), cpu_logical_id(dest), dest, vector); - set_rte(gsi, vector, dest, mask); + set_rte(gsi, irq, dest, mask); return vector; } @@ -1017,30 +958,32 @@ iosapic_override_isa_irq (unsigned int isa_irq, unsigned int gsi, unsigned long polarity, unsigned long trigger) { - int vector; + int vector, irq; unsigned int dest = cpu_physical_id(smp_processor_id()); - vector = isa_irq_to_vector(isa_irq); - - register_intr(gsi, vector, IOSAPIC_LOWEST_PRIORITY, polarity, trigger); + irq = vector = isa_irq_to_vector(isa_irq); + BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL)); + register_intr(gsi, irq, IOSAPIC_LOWEST_PRIORITY, polarity, trigger); DBG("ISA: IRQ %u -> GSI %u (%s,%s) -> CPU %d (0x%04x) vector %d\n", isa_irq, gsi, trigger == IOSAPIC_EDGE ? "edge" : "level", polarity == IOSAPIC_POL_HIGH ? "high" : "low", cpu_logical_id(dest), dest, vector); - set_rte(gsi, vector, dest, 1); + set_rte(gsi, irq, dest, 1); } void __init iosapic_system_init (int system_pcat_compat) { - int vector; + int irq; - for (vector = 0; vector < IA64_NUM_VECTORS; ++vector) { - iosapic_intr_info[vector].low32 = IOSAPIC_MASK; + for (irq = 0; irq < NR_IRQS; ++irq) { + iosapic_intr_info[irq].low32 = IOSAPIC_MASK; /* mark as unused */ - INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes); + INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes); + + iosapic_intr_info[irq].count = 0; } pcat_compat = system_pcat_compat; @@ -1108,31 +1051,35 @@ iosapic_init (unsigned long phys_addr, unsigned int gsi_base) unsigned long flags; spin_lock_irqsave(&iosapic_lock, flags); - { - addr = ioremap(phys_addr, 0); - ver = iosapic_version(addr); + index = find_iosapic(gsi_base); + if (index >= 0) { + spin_unlock_irqrestore(&iosapic_lock, flags); + return -EBUSY; + } - if ((err = iosapic_check_gsi_range(gsi_base, ver))) { - iounmap(addr); - spin_unlock_irqrestore(&iosapic_lock, flags); - return err; - } + addr = ioremap(phys_addr, 0); + ver = iosapic_version(addr); + if ((err = iosapic_check_gsi_range(gsi_base, ver))) { + iounmap(addr); + spin_unlock_irqrestore(&iosapic_lock, flags); + return err; + } - /* - * The MAX_REDIR register holds the highest input pin - * number (starting from 0). - * We add 1 so that we can use it for number of pins (= RTEs) - */ - num_rte = ((ver >> 16) & 0xff) + 1; + /* + * The MAX_REDIR register holds the highest input pin number + * (starting from 0). We add 1 so that we can use it for + * number of pins (= RTEs) + */ + num_rte = ((ver >> 16) & 0xff) + 1; - index = iosapic_alloc(); - iosapic_lists[index].addr = addr; - iosapic_lists[index].gsi_base = gsi_base; - iosapic_lists[index].num_rte = num_rte; + index = iosapic_alloc(); + iosapic_lists[index].addr = addr; + iosapic_lists[index].gsi_base = gsi_base; + iosapic_lists[index].num_rte = num_rte; #ifdef CONFIG_NUMA - iosapic_lists[index].node = MAX_NUMNODES; + iosapic_lists[index].node = MAX_NUMNODES; #endif - } + spin_lock_init(&iosapic_lists[index].lock); spin_unlock_irqrestore(&iosapic_lock, flags); if ((gsi_base == 0) && pcat_compat) { @@ -1157,25 +1104,22 @@ iosapic_remove (unsigned int gsi_base) unsigned long flags; spin_lock_irqsave(&iosapic_lock, flags); - { - index = find_iosapic(gsi_base); - if (index < 0) { - printk(KERN_WARNING "%s: No IOSAPIC for GSI base %u\n", - __FUNCTION__, gsi_base); - goto out; - } - - if (iosapic_lists[index].rtes_inuse) { - err = -EBUSY; - printk(KERN_WARNING - "%s: IOSAPIC for GSI base %u is busy\n", - __FUNCTION__, gsi_base); - goto out; - } + index = find_iosapic(gsi_base); + if (index < 0) { + printk(KERN_WARNING "%s: No IOSAPIC for GSI base %u\n", + __FUNCTION__, gsi_base); + goto out; + } - iounmap(iosapic_lists[index].addr); - iosapic_free(index); + if (iosapic_lists[index].rtes_inuse) { + err = -EBUSY; + printk(KERN_WARNING "%s: IOSAPIC for GSI base %u is busy\n", + __FUNCTION__, gsi_base); + goto out; } + + iounmap(iosapic_lists[index].addr); + iosapic_free(index); out: spin_unlock_irqrestore(&iosapic_lock, flags); return err; |