/* * Memory arbiter functions. Allocates bandwidth through the * arbiter and sets up arbiter breakpoints. * * The algorithm first assigns slots to the clients that has specified * bandwith (e.g. ethernet) and then the remaining slots are divided * on all the active clients. * * Copyright (c) 2004-2007 Axis Communications AB. * * The artpec-3 has two arbiters. The memory hierarchy looks like this: * * * CPU DMAs * | | * | | * -------------- ------------------ * | foo arbiter|----| Internal memory| * -------------- ------------------ * | * -------------- * | L2 cache | * -------------- * | * h264 etc | * | | * | | * -------------- * | bar arbiter| * -------------- * | * --------- * | SDRAM | * --------- * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #define D(x) struct crisv32_watch_entry { unsigned long instance; watch_callback *cb; unsigned long start; unsigned long end; int used; }; #define NUMBER_OF_BP 4 #define SDRAM_BANDWIDTH 400000000 #define INTMEM_BANDWIDTH 400000000 #define NBR_OF_SLOTS 64 #define NBR_OF_REGIONS 2 #define NBR_OF_CLIENTS 15 #define ARBITERS 2 #define UNASSIGNED 100 struct arbiter { unsigned long instance; int nbr_regions; int nbr_clients; int requested_slots[NBR_OF_REGIONS][NBR_OF_CLIENTS]; int active_clients[NBR_OF_REGIONS][NBR_OF_CLIENTS]; }; static struct crisv32_watch_entry watches[ARBITERS][NUMBER_OF_BP] = { { {regi_marb_foo_bp0}, {regi_marb_foo_bp1}, {regi_marb_foo_bp2}, {regi_marb_foo_bp3} }, { {regi_marb_bar_bp0}, {regi_marb_bar_bp1}, {regi_marb_bar_bp2}, {regi_marb_bar_bp3} } }; struct arbiter arbiters[ARBITERS] = { { /* L2 cache arbiter */ .instance = regi_marb_foo, .nbr_regions = 2, .nbr_clients = 15 }, { /* DDR2 arbiter */ .instance = regi_marb_bar, .nbr_regions = 1, .nbr_clients = 9 } }; static int max_bandwidth[NBR_OF_REGIONS] = {SDRAM_BANDWIDTH, INTMEM_BANDWIDTH}; DEFINE_SPINLOCK(arbiter_lock); static irqreturn_t crisv32_foo_arbiter_irq(int irq, void *dev_id); static irqreturn_t crisv32_bar_arbiter_irq(int irq, void *dev_id); /* * "I'm the arbiter, I know the score. * From square one I'll be watching all 64." * (memory arbiter slots, that is) * * Or in other words: * Program the memory arbiter slots for "region" according to what's * in requested_slots[] and active_clients[], while minimizing * latency. A caller may pass a non-zero positive amount for * "unused_slots", which must then be the unallocated, remaining * number of slots, free to hand out to any client. */ static void crisv32_arbiter_config(int arbiter, int region, int unused_slots) { int slot; int client; int interval = 0; /* * This vector corresponds to the hardware arbiter slots (see * the hardware documentation for semantics). We initialize * each slot with a suitable sentinel value outside the valid * range {0 .. NBR_OF_CLIENTS - 1} and replace them with * client indexes. Then it's fed to the hardware. */ s8 val[NBR_OF_SLOTS]; for (slot = 0; slot < NBR_OF_SLOTS; slot++) val[slot] = -1; for (client = 0; client < arbiters[arbiter].nbr_clients; client++) { int pos; /* Allocate the requested non-zero number of slots, but * also give clients with zero-requests one slot each * while stocks last. We do the latter here, in client * order. This makes sure zero-request clients are the * first to get to any spare slots, else those slots * could, when bandwidth is allocated close to the limit, * all be allocated to low-index non-zero-request clients * in the default-fill loop below. Another positive but * secondary effect is a somewhat better spread of the * zero-bandwidth clients in the vector, avoiding some of * the latency that could otherwise be caused by the * partitioning of non-zero-bandwidth clients at low * indexes and zero-bandwidth clients at high * indexes. (Note that this spreading can only affect the * unallocated bandwidth.) All the above only matters for * memory-intensive situations, of course. */ if (!arbiters[arbiter].requested_slots[region][client]) { /* * Skip inactive clients. Also skip zero-slot * allocations in this pass when there are no known * free slots. */ if (!arbiters[arbiter].active_clients[region][client] || unused_slots <= 0) continue; unused_slots--; /* Only allocate one slot for this client. */ interval = NBR_OF_SLOTS; } else interval = NBR_OF_SLOTS / arbiters[arbiter].requested_slots[region][client]; pos = 0; while (pos < NBR_OF_SLOTS) { if (val[pos] >= 0) pos++; else { val[pos] = client; pos += interval; } } } client = 0; for (slot = 0; slot < NBR_OF_SLOTS; slot++) { /* * Allocate remaining slots in round-robin * client-number order for active clients. For this * pass, we ignore requested bandwidth and previous * allocations. */ if (val[slot] < 0) { int first = client; while (!arbiters[arbiter].active_clients[region][client]) { client = (client + 1) % arbiters[arbiter].nbr_clients; if (client == first) break; } val[slot] = client; client = (client + 1) % arbiters[arbiter].nbr_clients; } if (arbiter == 0) { if (region == EXT_REGION) REG_WR_INT_VECT(marb_foo, regi_marb_foo, rw_l2_slots, slot, val[slot]); else if (region == INT_REGION) REG_WR_INT_VECT(marb_foo, regi_marb_foo, rw_intm_slots, slot, val[slot]); } else { REG_WR_INT_VECT(marb_bar, regi_marb_bar, rw_ddr2_slots, slot, val[slot]); } } } extern char _stext, _etext; static void crisv32_arbiter_init(void) { static int initialized; if (initialized) return; initialized = 1; /* * CPU caches are always set to active, but with zero * bandwidth allocated. It should be ok to allocate zero * bandwidth for the caches, because DMA for other channels * will supposedly finish, once their programmed amount is * done, and then the caches will get access according to the * "fixed scheme" for unclaimed slots. Though, if for some * use-case somewhere, there's a maximum CPU latency for * e.g. some interrupt, we have to start allocating specific * bandwidth for the CPU caches too. */ arbiters[0].active_clients[EXT_REGION][11] = 1; arbiters[0].active_clients[EXT_REGION][12] = 1; crisv32_arbiter_config(0, EXT_REGION, 0); crisv32_arbiter_config(0, INT_REGION, 0); crisv32_arbiter_config(1, EXT_REGION, 0); if (request_irq(MEMARB_FOO_INTR_VECT, crisv32_foo_arbiter_irq, IRQF_DISABLED, "arbiter", NULL)) printk(KERN_ERR "Couldn't allocate arbiter IRQ\n"); if (request_irq(MEMARB_BAR_INTR_VECT, crisv32_bar_arbiter_irq, IRQF_DISABLED, "arbiter", NULL)) printk(KERN_ERR "Couldn't allocate arbiter IRQ\n"); #ifndef CONFIG_ETRAX_KGDB /* Global watch for writes to kernel text segment. */ crisv32_arbiter_watch(virt_to_phys(&_stext), &_etext - &_stext, MARB_CLIENTS(arbiter_all_clients, arbiter_bar_all_clients), arbiter_all_write, NULL); #endif /* Set up max burst sizes by default */ REG_WR_INT(marb_bar, regi_marb_bar, rw_h264_rd_burst, 3); REG_WR_INT(marb_bar, regi_marb_bar, rw_h264_wr_burst, 3); REG_WR_INT(marb_bar, regi_marb_bar, rw_ccd_burst, 3); REG_WR_INT(marb_bar, regi_marb_bar, rw_vin_wr_burst, 3); REG_WR_INT(marb_bar, regi_marb_bar, rw_vin_rd_burst, 3); REG_WR_INT(marb_bar, regi_marb_bar, rw_sclr_rd_burst, 3); REG_WR_INT(marb_bar, regi_marb_bar, rw_vout_burst, 3); REG_WR_INT(marb_bar, regi_marb_bar, rw_sclr_fifo_burst, 3); REG_WR_INT(marb_bar, regi_marb_bar, rw_l2cache_burst, 3); } int crisv32_arbiter_allocate_bandwith(int client, int region, unsigned long bandwidth) { int i; int total_assigned = 0; int total_clients = 0; int req; int arbiter = 0; crisv32_arbiter_init(); if (client & 0xffff0000) { arbiter = 1; client >>= 16; } for (i = 0; i < arbiters[arbiter].nbr_clients; i++) { total_assigned += arbiters[arbiter].requested_slots[region][i]; total_clients += arbiters[arbiter].active_clients[region][i]; } /* Avoid division by 0 for 0-bandwidth requests. */ req = bandwidth == 0 ? 0 : NBR_OF_SLOTS / (max_bandwidth[region] / bandwidth); /* * We make sure that there are enough slots only for non-zero * requests. Requesting 0 bandwidth *may* allocate slots, * though if all bandwidth is allocated, such a client won't * get any and will have to rely on getting memory access * according to the fixed scheme that's the default when one * of the slot-allocated clients doesn't claim their slot. */ if (total_assigned + req > NBR_OF_SLOTS) return -ENOMEM; arbiters[arbiter].active_clients[region][client] = 1; arbiters[arbiter].requested_slots[region][client] = req; crisv32_arbiter_config(arbiter, region, NBR_OF_SLOTS - total_assigned); /* Propagate allocation from foo to bar */ if (arbiter == 0) crisv32_arbiter_allocate_bandwith(8 << 16, EXT_REGION, bandwidth); return 0; } /* * Main entry for bandwidth deallocation. * * Strictly speaking, for a somewhat constant set of clients where * each client gets a constant bandwidth and is just enabled or * disabled (somewhat dynamically), no action is necessary here to * avoid starvation for non-zero-allocation clients, as the allocated * slots will just be unused. However, handing out those unused slots * to active clients avoids needless latency if the "fixed scheme" * would give unclaimed slots to an eager low-index client. */ void crisv32_arbiter_deallocate_bandwidth(int client, int region) { int i; int total_assigned = 0; int arbiter = 0; if (client & 0xffff0000) arbiter = 1; arbiters[arbiter].requested_slots[region][client] = 0; arbiters[arbiter].active_clients[region][client] = 0; for (i = 0; i < arbiters[arbiter].nbr_clients; i++) total_assigned += arbiters[arbiter].requested_slots[region][i]; crisv32_arbiter_config(arbiter, region, NBR_OF_SLOTS - total_assigned); } int crisv32_arbiter_watch(unsigned long start, unsigned long size, unsigned long clients, unsigned long accesses, watch_callback *cb) { int i; int arbiter; int used[2]; int ret = 0; crisv32_arbiter_init(); if (start > 0x80000000) { printk(KERN_ERR "Arbiter: %lX doesn't look like a " "physical address", start); return -EFAULT; } spin_lock(&arbiter_lock); if (clients & 0xffff) used[0] = 1; if (clients & 0xffff0000) used[1] = 1; for (arbiter = 0; arbiter < ARBITERS; arbiter++) { if (!used[arbiter]) continue; for (i = 0; i < NUMBER_OF_BP; i++) { if (!watches[arbiter][i].used) { unsigned intr_mask; if (arbiter) intr_mask = REG_RD_INT(marb_bar, regi_marb_bar, rw_intr_mask); else intr_mask = REG_RD_INT(marb_foo, regi_marb_foo, rw_intr_mask); watches[arbiter][i].used = 1; watches[arbiter][i].start = start; watches[arbiter][i].end = start + size; watches[arbiter][i].cb = cb; ret |= (i + 1) << (arbiter + 8); if (arbiter) { REG_WR_INT(marb_bar_bp, watches[arbiter][i].instance, rw_first_addr, watches[arbiter][i].start); REG_WR_INT(marb_bar_bp, watches[arbiter][i].instance, rw_last_addr, watches[arbiter][i].end); REG_WR_INT(marb_bar_bp, watches[arbiter][i].instance, rw_op, accesses); REG_WR_INT(marb_bar_bp, watches[arbiter][i].instance, rw_clients, clients & 0xffff); } else { REG_WR_INT(marb_foo_bp, watches[arbiter][i].instance, rw_first_addr, watches[arbiter][i].start); REG_WR_INT(marb_foo_bp, watches[arbiter][i].instance, rw_last_addr, watches[arbiter][i].end); REG_WR_INT(marb_foo_bp, watches[arbiter][i].instance, rw_op, accesses); REG_WR_INT(marb_foo_bp, watches[arbiter][i].instance, rw_clients, clients >> 16); } if (i == 0) intr_mask |= 1; else if (i == 1) intr_mask |= 2; else if (i == 2) intr_mask |= 4; else if (i == 3) intr_mask |= 8; if (arbiter) REG_WR_INT(marb_bar, regi_marb_bar, rw_intr_mask, intr_mask); else REG_WR_INT(marb_foo, regi_marb_foo, rw_intr_mask, intr_mask); spin_unlock(&arbiter_lock); break; } } } spin_unlock(&arbiter_lock); if (ret) return ret; else return -ENOMEM; } int crisv32_arbiter_unwatch(int id) { int arbiter; int intr_mask; crisv32_arbiter_init(); spin_lock(&arbiter_lock); for (arbiter = 0; arbiter < ARBITERS; arbiter++) { int id2; if (arbiter) intr_mask = REG_RD_INT(marb_bar, regi_marb_bar, rw_intr_mask); else intr_mask = REG_RD_INT(marb_foo, regi_marb_foo, rw_intr_mask); id2 = (id & (0xff << (arbiter + 8))) >> (arbiter + 8); if (id2 == 0) continue; id2--; if ((id2 >= NUMBER_OF_BP) || (!watches[arbiter][id2].used)) { spin_unlock(&arbiter_lock); return -EINVAL; } memset(&watches[arbiter][id2], 0, sizeof(struct crisv32_watch_entry)); if (id2 == 0) intr_mask &= ~1; else if (id2 == 1) intr_mask &= ~2; else if (id2 == 2) intr_mask &= ~4; else if (id2 == 3) intr_mask &= ~8; if (arbiter) REG_WR_INT(marb_bar, regi_marb_bar, rw_intr_mask, intr_mask); else REG_WR_INT(marb_foo, regi_marb_foo, rw_intr_mask, intr_mask); } spin_unlock(&arbiter_lock); return 0; } extern void show_registers(struct pt_regs *regs); static irqreturn_t crisv32_foo_arbiter_irq(int irq, void *dev_id) { reg_marb_foo_r_masked_intr masked_intr = REG_RD(marb_foo, regi_marb_foo, r_masked_intr); reg_marb_foo_bp_r_brk_clients r_clients; reg_marb_foo_bp_r_brk_addr r_addr; reg_marb_foo_bp_r_brk_op r_op; reg_marb_foo_bp_r_brk_first_client r_first; reg_marb_foo_bp_r_brk_size r_size; reg_marb_foo_bp_rw_ack ack = {0}; reg_marb_foo_rw_ack_intr ack_intr = { .bp0 = 1, .bp1 = 1, .bp2 = 1, .bp3 = 1 }; struct crisv32_watch_entry *watch; unsigned arbiter = (unsigned)dev_id; masked_intr = REG_RD(marb_foo, regi_marb_foo, r_masked_intr); if (masked_intr.bp0) watch = &watches[arbiter][0]; else if (masked_intr.bp1) watch = &watches[arbiter][1]; else if (masked_intr.bp2) watch = &watches[arbiter][2]; else if (masked_intr.bp3) watch = &watches[arbiter][3]; else return IRQ_NONE; /* Retrieve all useful information and print it. */ r_clients = REG_RD(marb_foo_bp, watch->instance, r_brk_clients); r_addr = REG_RD(marb_foo_bp, watch->instance, r_brk_addr); r_op = REG_RD(marb_foo_bp, watch->instance, r_brk_op); r_first = REG_RD(marb_foo_bp, watch->instance, r_brk_first_client); r_size = REG_RD(marb_foo_bp, watch->instance, r_brk_size); printk(KERN_DEBUG "Arbiter IRQ\n"); printk(KERN_DEBUG "Clients %X addr %X op %X first %X size %X\n", REG_TYPE_CONV(int, reg_marb_foo_bp_r_brk_clients, r_clients), REG_TYPE_CONV(int, reg_marb_foo_bp_r_brk_addr, r_addr), REG_TYPE_CONV(int, reg_marb_foo_bp_r_brk_op, r_op), REG_TYPE_CONV(int, reg_marb_foo_bp_r_brk_first_client, r_first), REG_TYPE_CONV(int, reg_marb_foo_bp_r_brk_size, r_size)); REG_WR(marb_foo_bp, watch->instance, rw_ack, ack); REG_WR(marb_foo, regi_marb_foo, rw_ack_intr, ack_intr); printk(KERN_DEBUG "IRQ occured at %X\n", (unsigned)get_irq_regs()); if (watch->cb) watch->cb(); return IRQ_HANDLED; } static irqreturn_t crisv32_bar_arbiter_irq(int irq, void *dev_id) { reg_marb_bar_r_masked_intr masked_intr = REG_RD(marb_bar, regi_marb_bar, r_masked_intr); reg_marb_bar_bp_r_brk_clients r_clients; reg_marb_bar_bp_r_brk_addr r_addr; reg_marb_bar_bp_r_brk_op r_op; reg_marb_bar_bp_r_brk_first_client r_first; reg_marb_bar_bp_r_brk_size r_size; reg_marb_bar_bp_rw_ack ack = {0}; reg_marb_bar_rw_ack_intr ack_intr = { .bp0 = 1, .bp1 = 1, .bp2 = 1, .bp3 = 1 }; struct crisv32_watch_entry *watch; unsigned arbiter = (unsigned)dev_id; masked_intr = REG_RD(marb_bar, regi_marb_bar, r_masked_intr); if (masked_intr.bp0) watch = &watches[arbiter][0]; else if (masked_intr.bp1) watch = &watches[arbiter][1]; else if (masked_intr.bp2) watch = &watches[arbiter][2]; else if (masked_intr.bp3) watch = &watches[arbiter][3]; else return IRQ_NONE; /* Retrieve all useful information and print it. */ r_clients = REG_RD(marb_bar_bp, watch->instance, r_brk_clients); r_addr = REG_RD(marb_bar_bp, watch->instance, r_brk_addr); r_op = REG_RD(marb_bar_bp, watch->instance, r_brk_op); r_first = REG_RD(marb_bar_bp, watch->instance, r_brk_first_client); r_size = REG_RD(marb_bar_bp, watch->instance, r_brk_size); printk(KERN_DEBUG "Arbiter IRQ\n"); printk(KERN_DEBUG "Clients %X addr %X op %X first %X size %X\n", REG_TYPE_CONV(int, reg_marb_bar_bp_r_brk_clients, r_clients), REG_TYPE_CONV(int, reg_marb_bar_bp_r_brk_addr, r_addr), REG_TYPE_CONV(int, reg_marb_bar_bp_r_brk_op, r_op), REG_TYPE_CONV(int, reg_marb_bar_bp_r_brk_first_client, r_first), REG_TYPE_CONV(int, reg_marb_bar_bp_r_brk_size, r_size)); REG_WR(marb_bar_bp, watch->instance, rw_ack, ack); REG_WR(marb_bar, regi_marb_bar, rw_ack_intr, ack_intr); printk(KERN_DEBUG "IRQ occured at %X\n", (unsigned)get_irq_regs()->erp); if (watch->cb) watch->cb(); return IRQ_HANDLED; }