aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--arch/sparc64/kernel/entry.S8
-rw-r--r--arch/sparc64/kernel/irq.c41
-rw-r--r--arch/sparc64/kernel/sun4v_ivec.S8
-rw-r--r--include/asm-sparc64/cpudata.h11
-rw-r--r--include/asm-sparc64/irq.h2
5 files changed, 38 insertions, 32 deletions
diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S
index 8059531bf0a..9a785293203 100644
--- a/arch/sparc64/kernel/entry.S
+++ b/arch/sparc64/kernel/entry.S
@@ -430,15 +430,15 @@ do_ivec:
membar #Sync
sethi %hi(ivector_table), %g2
- sllx %g3, 3, %g3
+ sllx %g3, 4, %g3
or %g2, %lo(ivector_table), %g2
add %g2, %g3, %g3
TRAP_LOAD_IRQ_WORK(%g6, %g1)
- lduw [%g6], %g5 /* g5 = irq_work(cpu) */
- stw %g5, [%g3 + 0x00] /* bucket->irq_chain = g5 */
- stw %g3, [%g6] /* irq_work(cpu) = bucket */
+ ldx [%g6], %g5 /* g5 = irq_work(cpu) */
+ stx %g5, [%g3 + 0x00] /* bucket->irq_chain = g5 */
+ stx %g3, [%g6] /* irq_work(cpu) = bucket */
wr %g0, 1 << PIL_DEVICE_IRQ, %set_softint
retry
do_ivec_xcall:
diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c
index 045ab27d427..4db4dd57621 100644
--- a/arch/sparc64/kernel/irq.c
+++ b/arch/sparc64/kernel/irq.c
@@ -59,23 +59,20 @@
* of the vectored interrupt trap handler(s) in entry.S and sun4v_ivec.S
*/
struct ino_bucket {
- /* Next handler in per-CPU IRQ worklist. We know that
- * bucket pointers have the high 32-bits clear, so to
- * save space we only store the bits we need.
- */
-/*0x00*/unsigned int irq_chain;
+/*0x00*/unsigned long irq_chain;
/* Virtual interrupt number assigned to this INO. */
-/*0x04*/unsigned int virt_irq;
+/*0x08*/unsigned int virt_irq;
+/*0x0c*/unsigned int __pad;
};
#define NUM_IVECS (IMAP_INR + 1)
struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES)));
#define __irq_ino(irq) \
- (((struct ino_bucket *)(unsigned long)(irq)) - &ivector_table[0])
-#define __bucket(irq) ((struct ino_bucket *)(unsigned long)(irq))
-#define __irq(bucket) ((unsigned int)(unsigned long)(bucket))
+ (((struct ino_bucket *)(irq)) - &ivector_table[0])
+#define __bucket(irq) ((struct ino_bucket *)(irq))
+#define __irq(bucket) ((unsigned long)(bucket))
/* This has to be in the main kernel image, it cannot be
* turned into per-cpu data. The reason is that the main
@@ -87,13 +84,13 @@ struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BY
#define irq_work(__cpu) &(trap_block[(__cpu)].irq_worklist)
static struct {
- unsigned int irq;
+ unsigned long irq;
unsigned int dev_handle;
unsigned int dev_ino;
} virt_to_real_irq_table[NR_IRQS];
static DEFINE_SPINLOCK(virt_irq_alloc_lock);
-unsigned char virt_irq_alloc(unsigned int real_irq)
+unsigned char virt_irq_alloc(unsigned long real_irq)
{
unsigned long flags;
unsigned char ent;
@@ -134,7 +131,7 @@ void virt_irq_free(unsigned int virt_irq)
}
#endif
-static unsigned int virt_to_real_irq(unsigned char virt_irq)
+static unsigned long virt_to_real_irq(unsigned char virt_irq)
{
return virt_to_real_irq_table[virt_irq].irq;
}
@@ -227,7 +224,7 @@ struct irq_handler_data {
static inline struct ino_bucket *virt_irq_to_bucket(unsigned int virt_irq)
{
- unsigned int real_irq = virt_to_real_irq(virt_irq);
+ unsigned long real_irq = virt_to_real_irq(virt_irq);
struct ino_bucket *bucket = NULL;
if (likely(real_irq))
@@ -694,18 +691,28 @@ void handler_irq(int irq, struct pt_regs *regs)
{
struct ino_bucket *bucket;
struct pt_regs *old_regs;
+ unsigned long pstate;
clear_softint(1 << irq);
old_regs = set_irq_regs(regs);
irq_enter();
- /* Sliiiick... */
- bucket = __bucket(xchg32(irq_work(smp_processor_id()), 0));
+ /* Grab an atomic snapshot of the pending IVECs. */
+ __asm__ __volatile__("rdpr %%pstate, %0\n\t"
+ "wrpr %0, %3, %%pstate\n\t"
+ "ldx [%2], %1\n\t"
+ "stx %%g0, [%2]\n\t"
+ "wrpr %0, 0x0, %%pstate\n\t"
+ : "=&r" (pstate), "=&r" (bucket)
+ : "r" (irq_work(smp_processor_id())),
+ "i" (PSTATE_IE)
+ : "memory");
+
while (bucket) {
struct ino_bucket *next = __bucket(bucket->irq_chain);
- bucket->irq_chain = 0;
+ bucket->irq_chain = 0UL;
__do_IRQ(bucket->virt_irq);
bucket = next;
@@ -808,7 +815,7 @@ void init_irqwork_curcpu(void)
{
int cpu = hard_smp_processor_id();
- trap_block[cpu].irq_worklist = 0;
+ trap_block[cpu].irq_worklist = 0UL;
}
/* Please be very careful with register_one_mondo() and
diff --git a/arch/sparc64/kernel/sun4v_ivec.S b/arch/sparc64/kernel/sun4v_ivec.S
index 574bc248bca..e3e9d4c1574 100644
--- a/arch/sparc64/kernel/sun4v_ivec.S
+++ b/arch/sparc64/kernel/sun4v_ivec.S
@@ -101,14 +101,14 @@ sun4v_dev_mondo:
/* Get &ivector_table[IVEC] into %g4. */
sethi %hi(ivector_table), %g4
- sllx %g3, 3, %g3
+ sllx %g3, 4, %g3
or %g4, %lo(ivector_table), %g4
add %g4, %g3, %g4
/* Insert ivector_table[] entry into __irq_work[] queue. */
- lduw [%g1], %g2 /* g2 = irq_work(cpu) */
- stw %g2, [%g4 + 0x00] /* bucket->irq_chain = g2 */
- stw %g4, [%g1] /* irq_work(cpu) = bucket */
+ ldx [%g1], %g2 /* g2 = irq_work(cpu) */
+ stx %g2, [%g4 + 0x00] /* bucket->irq_chain = g2 */
+ stx %g4, [%g1] /* irq_work(cpu) = bucket */
/* Signal the interrupt by setting (1 << pil) in %softint. */
wr %g0, 1 << PIL_DEVICE_IRQ, %set_softint
diff --git a/include/asm-sparc64/cpudata.h b/include/asm-sparc64/cpudata.h
index 98a6e609163..379c219c3b7 100644
--- a/include/asm-sparc64/cpudata.h
+++ b/include/asm-sparc64/cpudata.h
@@ -75,12 +75,11 @@ struct trap_per_cpu {
unsigned long tsb_huge_temp;
/* Dcache line 8: IRQ work list, and keep trap_block a power-of-2 in size. */
- unsigned int irq_worklist;
+ unsigned long irq_worklist;
unsigned int cpu_mondo_qmask;
unsigned int dev_mondo_qmask;
unsigned int resum_qmask;
unsigned int nonresum_qmask;
- unsigned int __pad2[1];
void *hdesc;
} __attribute__((aligned(64)));
extern struct trap_per_cpu trap_block[NR_CPUS];
@@ -129,10 +128,10 @@ extern struct sun4v_2insn_patch_entry __sun4v_2insn_patch,
#define TRAP_PER_CPU_TSB_HUGE 0xd0
#define TRAP_PER_CPU_TSB_HUGE_TEMP 0xd8
#define TRAP_PER_CPU_IRQ_WORKLIST 0xe0
-#define TRAP_PER_CPU_CPU_MONDO_QMASK 0xe4
-#define TRAP_PER_CPU_DEV_MONDO_QMASK 0xe8
-#define TRAP_PER_CPU_RESUM_QMASK 0xec
-#define TRAP_PER_CPU_NONRESUM_QMASK 0xf0
+#define TRAP_PER_CPU_CPU_MONDO_QMASK 0xe8
+#define TRAP_PER_CPU_DEV_MONDO_QMASK 0xec
+#define TRAP_PER_CPU_RESUM_QMASK 0xf0
+#define TRAP_PER_CPU_NONRESUM_QMASK 0xf4
#define TRAP_BLOCK_SZ_SHIFT 8
diff --git a/include/asm-sparc64/irq.h b/include/asm-sparc64/irq.h
index bad3c28cdee..24841c22e81 100644
--- a/include/asm-sparc64/irq.h
+++ b/include/asm-sparc64/irq.h
@@ -59,7 +59,7 @@ extern unsigned int sun4u_build_msi(u32 portid, unsigned int *virt_irq_p,
extern void sun4u_destroy_msi(unsigned int virt_irq);
extern unsigned int sbus_build_irq(void *sbus, unsigned int ino);
-extern unsigned char virt_irq_alloc(unsigned int real_irq);
+extern unsigned char virt_irq_alloc(unsigned long real_irq);
#ifdef CONFIG_PCI_MSI
extern void virt_irq_free(unsigned int virt_irq);
#endif