diff options
Diffstat (limited to 'drivers/dma')
| -rw-r--r-- | drivers/dma/Kconfig | 60 | ||||
| -rw-r--r-- | drivers/dma/Makefile | 1 | ||||
| -rw-r--r-- | drivers/dma/ioat.c | 211 | ||||
| -rw-r--r-- | drivers/dma/ioat_dca.c | 263 | ||||
| -rw-r--r-- | drivers/dma/ioat_dma.c (renamed from drivers/dma/ioatdma.c) | 653 | ||||
| -rw-r--r-- | drivers/dma/ioatdma.h | 35 | ||||
| -rw-r--r-- | drivers/dma/ioatdma_hw.h | 2 | ||||
| -rw-r--r-- | drivers/dma/ioatdma_registers.h | 6 |
8 files changed, 943 insertions, 288 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 8f670dae53b..9c91b0fd134 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -2,42 +2,52 @@ # DMA engine configuration # -menu "DMA Engine support" - depends on HAS_DMA +menuconfig DMADEVICES + bool "DMA Offload Engine support" + depends on (PCI && X86) || ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX + help + Intel(R) offload engines enable offloading memory copies in the + network stack and RAID operations in the MD driver. + +if DMADEVICES + +comment "DMA Devices" + +config INTEL_IOATDMA + tristate "Intel I/OAT DMA support" + depends on PCI && X86 + select DMA_ENGINE + select DCA + help + Enable support for the Intel(R) I/OAT DMA engine present + in recent Intel Xeon chipsets. + + Say Y here if you have such a chipset. + + If unsure, say N. + +config INTEL_IOP_ADMA + tristate "Intel IOP ADMA support" + depends on ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX + select ASYNC_CORE + select DMA_ENGINE + help + Enable support for the Intel(R) IOP Series RAID engines. config DMA_ENGINE - bool "Support for DMA engines" - ---help--- - DMA engines offload bulk memory operations from the CPU to dedicated - hardware, allowing the operations to happen asynchronously. + bool comment "DMA Clients" + depends on DMA_ENGINE config NET_DMA bool "Network: TCP receive copy offload" depends on DMA_ENGINE && NET default y - ---help--- + help This enables the use of DMA engines in the network stack to offload receive copy-to-user operations, freeing CPU cycles. Since this is the main user of the DMA engine, it should be enabled; say Y here. -comment "DMA Devices" - -config INTEL_IOATDMA - tristate "Intel I/OAT DMA support" - depends on DMA_ENGINE && PCI - default m - ---help--- - Enable support for the Intel(R) I/OAT DMA engine. - -config INTEL_IOP_ADMA - tristate "Intel IOP ADMA support" - depends on DMA_ENGINE && (ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX) - select ASYNC_CORE - default m - ---help--- - Enable support for the Intel(R) IOP Series RAID engines. - -endmenu +endif diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index b3839b687ae..b152cd84e12 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -1,4 +1,5 @@ obj-$(CONFIG_DMA_ENGINE) += dmaengine.o obj-$(CONFIG_NET_DMA) += iovlock.o obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o +ioatdma-objs := ioat.o ioat_dma.o ioat_dca.o obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o diff --git a/drivers/dma/ioat.c b/drivers/dma/ioat.c new file mode 100644 index 00000000000..f7276bf2fe7 --- /dev/null +++ b/drivers/dma/ioat.c @@ -0,0 +1,211 @@ +/* + * Intel I/OAT DMA Linux driver + * Copyright(c) 2007 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * + * The full GNU General Public License is included in this distribution in + * the file called "COPYING". + * + */ + +/* + * This driver supports an Intel I/OAT DMA engine, which does asynchronous + * copy operations. + */ + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/interrupt.h> +#include <linux/dca.h> +#include "ioatdma.h" +#include "ioatdma_registers.h" +#include "ioatdma_hw.h" + +MODULE_VERSION("1.24"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Intel Corporation"); + +static struct pci_device_id ioat_pci_tbl[] = { + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) }, + { PCI_DEVICE(PCI_VENDOR_ID_UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) }, + { 0, } +}; + +struct ioat_device { + struct pci_dev *pdev; + void __iomem *iobase; + struct ioatdma_device *dma; + struct dca_provider *dca; +}; + +static int __devinit ioat_probe(struct pci_dev *pdev, + const struct pci_device_id *id); +#ifdef IOAT_DMA_REMOVE +static void __devexit ioat_remove(struct pci_dev *pdev); +#endif + +static int ioat_dca_enabled = 1; +module_param(ioat_dca_enabled, int, 0644); +MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); + +static int ioat_setup_functionality(struct pci_dev *pdev, void __iomem *iobase) +{ + struct ioat_device *device = pci_get_drvdata(pdev); + u8 version; + int err = 0; + + version = readb(iobase + IOAT_VER_OFFSET); + switch (version) { + case IOAT_VER_1_2: + device->dma = ioat_dma_probe(pdev, iobase); + if (ioat_dca_enabled) + device->dca = ioat_dca_init(pdev, iobase); + break; + default: + err = -ENODEV; + break; + } + return err; +} + +static void ioat_shutdown_functionality(struct pci_dev *pdev) +{ + struct ioat_device *device = pci_get_drvdata(pdev); + + if (device->dma) { + ioat_dma_remove(device->dma); + device->dma = NULL; + } + + if (device->dca) { + unregister_dca_provider(device->dca); + free_dca_provider(device->dca); + device->dca = NULL; + } + +} + +static struct pci_driver ioat_pci_drv = { + .name = "ioatdma", + .id_table = ioat_pci_tbl, + .probe = ioat_probe, + .shutdown = ioat_shutdown_functionality, +#ifdef IOAT_DMA_REMOVE + .remove = __devexit_p(ioat_remove), +#endif +}; + +static int __devinit ioat_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + void __iomem *iobase; + struct ioat_device *device; + unsigned long mmio_start, mmio_len; + int err; + + err = pci_enable_device(pdev); + if (err) + goto err_enable_device; + + err = pci_request_regions(pdev, ioat_pci_drv.name); + if (err) + goto err_request_regions; + + err = pci_set_dma_mask(pdev, DMA_64BIT_MASK); + if (err) + err = pci_set_dma_mask(pdev, DMA_32BIT_MASK); + if (err) + goto err_set_dma_mask; + + err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK); + if (err) + err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK); + if (err) + goto err_set_dma_mask; + + mmio_start = pci_resource_start(pdev, 0); + mmio_len = pci_resource_len(pdev, 0); + iobase = ioremap(mmio_start, mmio_len); + if (!iobase) { + err = -ENOMEM; + goto err_ioremap; + } + + device = kzalloc(sizeof(*device), GFP_KERNEL); + if (!device) { + err = -ENOMEM; + goto err_kzalloc; + } + device->pdev = pdev; + pci_set_drvdata(pdev, device); + device->iobase = iobase; + + pci_set_master(pdev); + + err = ioat_setup_functionality(pdev, iobase); + if (err) + goto err_version; + + return 0; + +err_version: + kfree(device); +err_kzalloc: + iounmap(iobase); +err_ioremap: +err_set_dma_mask: + pci_release_regions(pdev); + pci_disable_device(pdev); +err_request_regions: +err_enable_device: + return err; +} + +#ifdef IOAT_DMA_REMOVE +/* + * It is unsafe to remove this module: if removed while a requested + * dma is outstanding, esp. from tcp, it is possible to hang while + * waiting for something that will never finish, thus hanging at + * least one cpu. However, if you're feeling lucky and need to do + * some testing, this usually works just fine. + */ +static void __devexit ioat_remove(struct pci_dev *pdev) +{ + struct ioat_device *device = pci_get_drvdata(pdev); + + ioat_shutdown_functionality(pdev); + + kfree(device); + + iounmap(device->iobase); + pci_release_regions(pdev); + pci_disable_device(pdev); +} +#endif + +static int __init ioat_init_module(void) +{ + return pci_register_driver(&ioat_pci_drv); +} +module_init(ioat_init_module); + +static void __exit ioat_exit_module(void) +{ + pci_unregister_driver(&ioat_pci_drv); +} +module_exit(ioat_exit_module); diff --git a/drivers/dma/ioat_dca.c b/drivers/dma/ioat_dca.c new file mode 100644 index 00000000000..2ae04c30ede --- /dev/null +++ b/drivers/dma/ioat_dca.c @@ -0,0 +1,263 @@ +/* + * Intel I/OAT DMA Linux driver + * Copyright(c) 2007 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * + * The full GNU General Public License is included in this distribution in + * the file called "COPYING". + * + */ + +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/smp.h> +#include <linux/interrupt.h> +#include <linux/dca.h> + +/* either a kernel change is needed, or we need something like this in kernel */ +#ifndef CONFIG_SMP +#include <asm/smp.h> +#undef cpu_physical_id +#define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24) +#endif + +#include "ioatdma.h" +#include "ioatdma_registers.h" + +/* + * Bit 16 of a tag map entry is the "valid" bit, if it is set then bits 0:15 + * contain the bit number of the APIC ID to map into the DCA tag. If the valid + * bit is not set, then the value must be 0 or 1 and defines the bit in the tag. + */ +#define DCA_TAG_MAP_VALID 0x80 + +/* + * "Legacy" DCA systems do not implement the DCA register set in the + * I/OAT device. Software needs direct support for their tag mappings. + */ + +#define APICID_BIT(x) (DCA_TAG_MAP_VALID | (x)) +#define IOAT_TAG_MAP_LEN 8 + +static u8 ioat_tag_map_BNB[IOAT_TAG_MAP_LEN] = { + 1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), }; +static u8 ioat_tag_map_SCNB[IOAT_TAG_MAP_LEN] = { + 1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), }; +static u8 ioat_tag_map_CNB[IOAT_TAG_MAP_LEN] = { + 1, APICID_BIT(1), APICID_BIT(3), APICID_BIT(4), APICID_BIT(2), }; +static u8 ioat_tag_map_UNISYS[IOAT_TAG_MAP_LEN] = { 0 }; + +/* pack PCI B/D/F into a u16 */ +static inline u16 dcaid_from_pcidev(struct pci_dev *pci) +{ + return (pci->bus->number << 8) | pci->devfn; +} + +static int dca_enabled_in_bios(void) +{ + /* CPUID level 9 returns DCA configuration */ + /* Bit 0 indicates DCA enabled by the BIOS */ + unsigned long cpuid_level_9; + int res; + + cpuid_level_9 = cpuid_eax(9); + res = test_bit(0, &cpuid_level_9); + if (!res) + printk(KERN_ERR "ioat dma: DCA is disabled in BIOS\n"); + + return res; +} + +static int system_has_dca_enabled(void) +{ + if (boot_cpu_has(X86_FEATURE_DCA)) + return dca_enabled_in_bios(); + + printk(KERN_ERR "ioat dma: boot cpu doesn't have X86_FEATURE_DCA\n"); + return 0; +} + +struct ioat_dca_slot { + struct pci_dev *pdev; /* requester device */ + u16 rid; /* requester id, as used by IOAT */ +}; + +#define IOAT_DCA_MAX_REQ 6 + +struct ioat_dca_priv { + void __iomem *iobase; + void *dca_base; + int max_requesters; + int requester_count; + u8 tag_map[IOAT_TAG_MAP_LEN]; + struct ioat_dca_slot req_slots[0]; +}; + +/* 5000 series chipset DCA Port Requester ID Table Entry Format + * [15:8] PCI-Express Bus Number + * [7:3] PCI-Express Device Number + * [2:0] PCI-Express Function Number + * + * 5000 series chipset DCA control register format + * [7:1] Reserved (0) + * [0] Ignore Function Number + */ + +static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev) +{ + struct ioat_dca_priv *ioatdca = dca_priv(dca); + struct pci_dev *pdev; + int i; + u16 id; + + /* This implementation only supports PCI-Express */ + if (dev->bus != &pci_bus_type) + return -ENODEV; + pdev = to_pci_dev(dev); + id = dcaid_from_pcidev(pdev); + + if (ioatdca->requester_count == ioatdca->max_requesters) + return -ENODEV; + + for (i = 0; i < ioatdca->max_requesters; i++) { + if (ioatdca->req_slots[i].pdev == NULL) { + /* found an empty slot */ + ioatdca->requester_count++; + ioatdca->req_slots[i].pdev = pdev; + ioatdca->req_slots[i].rid = id; + writew(id, ioatdca->dca_base + (i * 4)); + /* make sure the ignore function bit is off */ + writeb(0, ioatdca->dca_base + (i * 4) + 2); + return i; + } + } + /* Error, ioatdma->requester_count is out of whack */ + return -EFAULT; +} + +static int ioat_dca_remove_requester(struct dca_provider *dca, + struct device *dev) +{ + struct ioat_dca_priv *ioatdca = dca_priv(dca); + struct pci_dev *pdev; + int i; + + /* This implementation only supports PCI-Express */ + if (dev->bus != &pci_bus_type) + return -ENODEV; + pdev = to_pci_dev(dev); + + for (i = 0; i < ioatdca->max_requesters; i++) { + if (ioatdca->req_slots[i].pdev == pdev) { + writew(0, ioatdca->dca_base + (i * 4)); + ioatdca->req_slots[i].pdev = NULL; + ioatdca->req_slots[i].rid = 0; + ioatdca->requester_count--; + return i; + } + } + return -ENODEV; +} + +static u8 ioat_dca_get_tag(struct dca_provider *dca, int cpu) +{ + struct ioat_dca_priv *ioatdca = dca_priv(dca); + int i, apic_id, bit, value; + u8 entry, tag; + + tag = 0; + apic_id = cpu_physical_id(cpu); + + for (i = 0; i < IOAT_TAG_MAP_LEN; i++) { + entry = ioatdca->tag_map[i]; + if (entry & DCA_TAG_MAP_VALID) { + bit = entry & ~DCA_TAG_MAP_VALID; + value = (apic_id & (1 << bit)) ? 1 : 0; + } else { + value = entry ? 1 : 0; + } + tag |= (value << i); + } + return tag; +} + +static struct dca_ops ioat_dca_ops = { + .add_requester = ioat_dca_add_requester, + .remove_requester = ioat_dca_remove_requester, + .get_tag = ioat_dca_get_tag, +}; + + +struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase) +{ + struct dca_provider *dca; + struct ioat_dca_priv *ioatdca; + u8 *tag_map = NULL; + int i; + int err; + + if (!system_has_dca_enabled()) + return NULL; + + /* I/OAT v1 systems must have a known tag_map to support DCA */ + switch (pdev->vendor) { + case PCI_VENDOR_ID_INTEL: + switch (pdev->device) { + case PCI_DEVICE_ID_INTEL_IOAT: + tag_map = ioat_tag_map_BNB; + break; + case PCI_DEVICE_ID_INTEL_IOAT_CNB: + tag_map = ioat_tag_map_CNB; + break; + case PCI_DEVICE_ID_INTEL_IOAT_SCNB: + tag_map = ioat_tag_map_SCNB; + break; + } + break; + case PCI_VENDOR_ID_UNISYS: + switch (pdev->device) { + case PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR: + tag_map = ioat_tag_map_UNISYS; + break; + } + break; + } + if (tag_map == NULL) + return NULL; + + dca = alloc_dca_provider(&ioat_dca_ops, + sizeof(*ioatdca) + + (sizeof(struct ioat_dca_slot) * IOAT_DCA_MAX_REQ)); + if (!dca) + return NULL; + + ioatdca = dca_priv(dca); + ioatdca->max_requesters = IOAT_DCA_MAX_REQ; + + ioatdca->dca_base = iobase + 0x54; + + /* copy over the APIC ID to DCA tag mapping */ + for (i = 0; i < IOAT_TAG_MAP_LEN; i++) + ioatdca->tag_map[i] = tag_map[i]; + + err = register_dca_provider(dca, &pdev->dev); + if (err) { + free_dca_provider(dca); + return NULL; + } + + return dca; +} + diff --git a/drivers/dma/ioatdma.c b/drivers/dma/ioat_dma.c index 41b18c5a314..66c5bb53211 100644 --- a/drivers/dma/ioatdma.c +++ b/drivers/dma/ioat_dma.c @@ -1,10 +1,10 @@ /* - * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. + * Intel I/OAT DMA Linux driver + * Copyright(c) 2004 - 2007 Intel Corporation. * * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or @@ -12,11 +12,12 @@ * more details. * * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * + * The full GNU General Public License is included in this distribution in + * the file called "COPYING". * - * The full GNU General Public License is included in this distribution in the - * file called COPYING. */ /* @@ -35,17 +36,77 @@ #include "ioatdma_registers.h" #include "ioatdma_hw.h" +#define INITIAL_IOAT_DESC_COUNT 128 + #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) -#define to_ioat_device(dev) container_of(dev, struct ioat_device, common) +#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common) #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) #define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx) /* internal functions */ -static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); -static void ioat_shutdown(struct pci_dev *pdev); -static void __devexit ioat_remove(struct pci_dev *pdev); +static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan); +static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan); -static int enumerate_dma_channels(struct ioat_device *device) +static struct ioat_dma_chan *ioat_lookup_chan_by_index(struct ioatdma_device *device, + int index) +{ + return device->idx[index]; +} + +/** + * ioat_dma_do_interrupt - handler used for single vector interrupt mode + * @irq: interrupt id + * @data: interrupt data + */ +static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) +{ + struct ioatdma_device *instance = data; + struct ioat_dma_chan *ioat_chan; + unsigned long attnstatus; + int bit; + u8 intrctrl; + + intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET); + + if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) + return IRQ_NONE; + + if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { + writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); + return IRQ_NONE; + } + + attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); + for_each_bit(bit, &attnstatus, BITS_PER_LONG) { + ioat_chan = ioat_lookup_chan_by_index(instance, bit); + tasklet_schedule(&ioat_chan->cleanup_task); + } + + writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); + return IRQ_HANDLED; +} + +/** + * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode + * @irq: interrupt id + * @data: interrupt data + */ +static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) +{ + struct ioat_dma_chan *ioat_chan = data; + + tasklet_schedule(&ioat_chan->cleanup_task); + + return IRQ_HANDLED; +} + +static void ioat_dma_cleanup_tasklet(unsigned long data); + +/** + * ioat_dma_enumerate_channels - find and initialize the device's channels + * @device: the device to be enumerated + */ +static int ioat_dma_enumerate_channels(struct ioatdma_device *device) { u8 xfercap_scale; u32 xfercap; @@ -73,13 +134,19 @@ static int enumerate_dma_channels(struct ioat_device *device) /* This should be made common somewhere in dmaengine.c */ ioat_chan->common.device = &device->common; list_add_tail(&ioat_chan->common.device_node, - &device->common.channels); + &device->common.channels); + device->idx[i] = ioat_chan; + tasklet_init(&ioat_chan->cleanup_task, + ioat_dma_cleanup_tasklet, + (unsigned long) ioat_chan); + tasklet_disable(&ioat_chan->cleanup_task); } return device->common.chancnt; } -static void -ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) +static void ioat_set_src(dma_addr_t addr, + struct dma_async_tx_descriptor *tx, + int index) { struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx); struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); @@ -93,8 +160,9 @@ ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) } -static void -ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) +static void ioat_set_dest(dma_addr_t addr, + struct dma_async_tx_descriptor *tx, + int index) { struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx); struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); @@ -107,8 +175,7 @@ ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) } } -static dma_cookie_t -ioat_tx_submit(struct dma_async_tx_descriptor *tx) +static dma_cookie_t ioat_tx_submit(struct dma_async_tx_descriptor *tx) { struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); struct ioat_desc_sw *desc = tx_to_ioat_desc(tx); @@ -141,27 +208,27 @@ ioat_tx_submit(struct dma_async_tx_descriptor *tx) if (append) writeb(IOAT_CHANCMD_APPEND, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); - + return cookie; } static struct ioat_desc_sw *ioat_dma_alloc_descriptor( - struct ioat_dma_chan *ioat_chan, - gfp_t flags) + struct ioat_dma_chan *ioat_chan, + gfp_t flags) { struct ioat_dma_descriptor *desc; struct ioat_desc_sw *desc_sw; - struct ioat_device *ioat_device; + struct ioatdma_device *ioatdma_device; dma_addr_t phys; - ioat_device = to_ioat_device(ioat_chan->common.device); - desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys); + ioatdma_device = to_ioatdma_device(ioat_chan->common.device); + desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys); if (unlikely(!desc)) return NULL; desc_sw = kzalloc(sizeof(*desc_sw), flags); if (unlikely(!desc_sw)) { - pci_pool_free(ioat_device->dma_pool, desc, phys); + pci_pool_free(ioatdma_device->dma_pool, desc, phys); return NULL; } @@ -177,10 +244,6 @@ static struct ioat_desc_sw *ioat_dma_alloc_descriptor( return desc_sw; } -#define INITIAL_IOAT_DESC_COUNT 128 - -static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan); - /* returns the actual number of allocated descriptors */ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) { @@ -195,15 +258,16 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) if (!list_empty(&ioat_chan->free_desc)) return INITIAL_IOAT_DESC_COUNT; - /* Setup register to interrupt and write completion status on error */ + /* Setup register to interrupt and write completion status on error */ chanctrl = IOAT_CHANCTRL_ERR_INT_EN | IOAT_CHANCTRL_ANY_ERR_ABORT_EN | IOAT_CHANCTRL_ERR_COMPLETION_EN; - writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); + writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); if (chanerr) { - printk("IOAT: CHANERR = %x, clearing\n", chanerr); + dev_err(&ioat_chan->device->pdev->dev, + "ioatdma: CHANERR = %x, clearing\n", chanerr); writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); } @@ -211,7 +275,8 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) { desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); if (!desc) { - printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i); + dev_err(&ioat_chan->device->pdev->dev, + "ioatdma: Only %d initial descriptors\n", i); break; } list_add_tail(&desc->node, &tmp_list); @@ -224,8 +289,8 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ ioat_chan->completion_virt = pci_pool_alloc(ioat_chan->device->completion_pool, - GFP_KERNEL, - &ioat_chan->completion_addr); + GFP_KERNEL, + &ioat_chan->completion_addr); memset(ioat_chan->completion_virt, 0, sizeof(*ioat_chan->completion_virt)); writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF, @@ -233,54 +298,88 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) writel(((u64) ioat_chan->completion_addr) >> 32, ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); - ioat_start_null_desc(ioat_chan); + tasklet_enable(&ioat_chan->cleanup_task); + ioat_dma_start_null_desc(ioat_chan); return i; } -static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan); - static void ioat_dma_free_chan_resources(struct dma_chan *chan) { struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); - struct ioat_device *ioat_device = to_ioat_device(chan->device); + struct ioatdma_device *ioatdma_device = to_ioatdma_device(chan->device); struct ioat_desc_sw *desc, *_desc; - u16 chanctrl; int in_use_descs = 0; + tasklet_disable(&ioat_chan->cleanup_task); ioat_dma_memcpy_cleanup(ioat_chan); + /* Delay 100ms after reset to allow internal DMA logic to quiesce + * before removing DMA descriptor resources. + */ writeb(IOAT_CHANCMD_RESET, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); + mdelay(100); spin_lock_bh(&ioat_chan->desc_lock); list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { in_use_descs++; list_del(&desc->node); - pci_pool_free(ioat_device->dma_pool, desc->hw, + pci_pool_free(ioatdma_device->dma_pool, desc->hw, desc->async_tx.phys); kfree(desc); } list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { list_del(&desc->node); - pci_pool_free(ioat_device->dma_pool, desc->hw, + pci_pool_free(ioatdma_device->dma_pool, desc->hw, desc->async_tx.phys); kfree(desc); } spin_unlock_bh(&ioat_chan->desc_lock); - pci_pool_free(ioat_device->completion_pool, - ioat_chan->completion_virt, - ioat_chan->completion_addr); + pci_pool_free(ioatdma_device->completion_pool, + ioat_chan->completion_virt, + ioat_chan->completion_addr); /* one is ok since we left it on there on purpose */ if (in_use_descs > 1) - printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n", + dev_err(&ioat_chan->device->pdev->dev, + "ioatdma: Freeing %d in use descriptors!\n", in_use_descs - 1); ioat_chan->last_completion = ioat_chan->completion_addr = 0; + ioat_chan->pending = 0; +} +/** + * ioat_dma_get_next_descriptor - return the next available descriptor + * @ioat_chan: IOAT DMA channel handle + * + * Gets the next descriptor from the chain, and must be called with the + * channel's desc_lock held. Allocates more descriptors if the channel + * has run out. + */ +static struct ioat_desc_sw *ioat_dma_get_next_descriptor( + struct ioat_dma_chan *ioat_chan) +{ + struct ioat_desc_sw *new = NULL; + + if (!list_empty(&ioat_chan->free_desc)) { + new = to_ioat_desc(ioat_chan->free_desc.next); + list_del(&new->node); + } else { + /* try to get another desc */ + new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC); + /* will this ever happen? */ + /* TODO add upper limit on these */ + BUG_ON(!new); + } + + prefetch(new->hw); + return new; } -static struct dma_async_tx_descriptor * -ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en) +static struct dma_async_tx_descriptor *ioat_dma_prep_memcpy( + struct dma_chan *chan, + size_t len, + int int_en) { struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); struct ioat_desc_sw *first, *prev, *new; @@ -299,17 +398,7 @@ ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en) spin_lock_bh(&ioat_chan->desc_lock); while (len) { - if (!list_empty(&ioat_chan->free_desc)) { - new = to_ioat_desc(ioat_chan->free_desc.next); - list_del(&new->node); - } else { - /* try to get another desc */ - new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC); - /* will this ever happen? */ - /* TODO add upper limit on these */ - BUG_ON(!new); - } - + new = ioat_dma_get_next_descriptor(ioat_chan); copy = min((u32) len, ioat_chan->xfercap); new->hw->size = copy; @@ -343,12 +432,11 @@ ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en) return new ? &new->async_tx : NULL; } - /** - * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw + * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended + * descriptors to hw * @chan: DMA channel handle */ - static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan) { struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); @@ -360,15 +448,23 @@ static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan) } } -static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) +static void ioat_dma_cleanup_tasklet(unsigned long data) +{ + struct ioat_dma_chan *chan = (void *)data; + ioat_dma_memcpy_cleanup(chan); + writew(IOAT_CHANCTRL_INT_DISABLE, + chan->reg_base + IOAT_CHANCTRL_OFFSET); +} + +static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan) { unsigned long phys_complete; struct ioat_desc_sw *desc, *_desc; dma_cookie_t cookie = 0; - prefetch(chan->completion_virt); + prefetch(ioat_chan->completion_virt); - if (!spin_trylock(&chan->cleanup_lock)) + if (!spin_trylock(&ioat_chan->cleanup_lock)) return; /* The completion writeback can happen at any time, @@ -378,26 +474,28 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) #if (BITS_PER_LONG == 64) phys_complete = - chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; + ioat_chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; #else - phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK; + phys_complete = ioat_chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK; #endif - if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == - IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { - printk("IOAT: Channel halted, chanerr = %x\n", - readl(chan->reg_base + IOAT_CHANERR_OFFSET)); + if ((ioat_chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == + IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { + dev_err(&ioat_chan->device->pdev->dev, + "ioatdma: Channel halted, chanerr = %x\n", + readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET)); /* TODO do something to salvage the situation */ } - if (phys_complete == chan->last_completion) { - spin_unlock(&chan->cleanup_lock); + if (phys_complete == ioat_chan->last_completion) { + spin_unlock(&ioat_chan->cleanup_lock); return; } - spin_lock_bh(&chan->desc_lock); - list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) { + cookie = 0; + spin_lock_bh(&ioat_chan->desc_lock); + list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { /* * Incoming DMA requests may use multiple descriptors, due to @@ -407,31 +505,36 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) if (desc->async_tx.cookie) { cookie = desc->async_tx.cookie; - /* yes we are unmapping both _page and _single alloc'd - regions with unmap_page. Is this *really* that bad? - */ - pci_unmap_page(chan->device->pdev, + /* + * yes we are unmapping both _page and _single alloc'd + * regions with unmap_page. Is this *really* that bad? + */ + pci_unmap_page(ioat_chan->device->pdev, pci_unmap_addr(desc, dst), pci_unmap_len(desc, len), PCI_DMA_FROMDEVICE); - pci_unmap_page(chan->device->pdev, + pci_unmap_page(ioat_chan->device->pdev, pci_unmap_addr(desc, src), pci_unmap_len(desc, len), PCI_DMA_TODEVICE); } if (desc->async_tx.phys != phys_complete) { - /* a completed entry, but not the last, so cleanup + /* + * a completed entry, but not the last, so cleanup * if the client is done with the descriptor */ if (desc->async_tx.ack) { list_del(&desc->node); - list_add_tail(&desc->node, &chan->free_desc); + list_add_tail(&desc->node, + &ioat_chan->free_desc); } else desc->async_tx.cookie = 0; } else { - /* last used desc. Do not remove, so we can append from - it, but don't look at it next time, either */ + /* + * last used desc. Do not remove, so we can append from + * it, but don't look at it next time, either + */ desc->async_tx.cookie = 0; /* TODO check status bits? */ @@ -439,13 +542,13 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) } } - spin_unlock_bh(&chan->desc_lock); + spin_unlock_bh(&ioat_chan->desc_lock); - chan->last_completion = phys_complete; + ioat_chan->last_completion = phys_complete; if (cookie != 0) - chan->completed_cookie = cookie; + ioat_chan->completed_cookie = cookie; - spin_unlock(&chan->cleanup_lock); + spin_unlock(&ioat_chan->cleanup_lock); } static void ioat_dma_dependency_added(struct dma_chan *chan) @@ -466,11 +569,10 @@ static void ioat_dma_dependency_added(struct dma_chan *chan) * @done: if not %NULL, updated with last completed transaction * @used: if not %NULL, updated with last used transaction */ - static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, - dma_cookie_t cookie, - dma_cookie_t *done, - dma_cookie_t *used) + dma_cookie_t cookie, + dma_cookie_t *done, + dma_cookie_t *used) { struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); dma_cookie_t last_used; @@ -481,7 +583,7 @@ static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, last_complete = ioat_chan->completed_cookie; if (done) - *done= last_complete; + *done = last_complete; if (used) *used = last_used; @@ -495,7 +597,7 @@ static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, last_complete = ioat_chan->completed_cookie; if (done) - *done= last_complete; + *done = last_complete; if (used) *used = last_used; @@ -504,63 +606,13 @@ static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, /* PCI API */ -static struct pci_device_id ioat_pci_tbl[] = { - { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, - { PCI_DEVICE(PCI_VENDOR_ID_UNISYS, - PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) }, - { 0, } -}; - -static struct pci_driver ioat_pci_driver = { - .name = "ioatdma", - .id_table = ioat_pci_tbl, - .probe = ioat_probe, - .shutdown = ioat_shutdown, - .remove = __devexit_p(ioat_remove), -}; - -static irqreturn_t ioat_do_interrupt(int irq, void *data) -{ - struct ioat_device *instance = data; - unsigned long attnstatus; - u8 intrctrl; - - intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET); - - if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) - return IRQ_NONE; - - if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { - writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); - return IRQ_NONE; - } - - attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); - - printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus); - - writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); - return IRQ_HANDLED; -} - -static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan) +static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan) { struct ioat_desc_sw *desc; spin_lock_bh(&ioat_chan->desc_lock); - if (!list_empty(&ioat_chan->free_desc)) { - desc = to_ioat_desc(ioat_chan->free_desc.next); - list_del(&desc->node); - } else { - /* try to get another desc */ - spin_unlock_bh(&ioat_chan->desc_lock); - desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); - spin_lock_bh(&ioat_chan->desc_lock); - /* will this ever happen? */ - BUG_ON(!desc); - } - + desc = ioat_dma_get_next_descriptor(ioat_chan); desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; desc->hw->next = 0; desc->async_tx.ack = 1; @@ -581,7 +633,11 @@ static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan) */ #define IOAT_TEST_SIZE 2000 -static int ioat_self_test(struct ioat_device *device) +/** + * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works. + * @device: device to be tested + */ +static int ioat_dma_self_test(struct ioatdma_device *device) { int i; u8 *src; @@ -607,9 +663,11 @@ static int ioat_self_test(struct ioat_device *device) /* Start copy, using first DMA channel */ dma_chan = container_of(device->common.channels.next, - struct dma_chan, - device_node); + struct dma_chan, + device_node); if (ioat_dma_alloc_chan_resources(dma_chan) < 1) { + dev_err(&device->pdev->dev, + "selftest cannot allocate chan resource\n"); err = -ENODEV; goto out; } @@ -627,12 +685,14 @@ static int ioat_self_test(struct ioat_device *device) msleep(1); if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { - printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n"); + dev_err(&device->pdev->dev, + "ioatdma: Self-test copy timed out, disabling\n"); err = -ENODEV; goto free_resources; } if (memcmp(src, dest, IOAT_TEST_SIZE)) { - printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n"); + dev_err(&device->pdev->dev, + "ioatdma: Self-test copy failed compare, disabling\n"); err = -ENODEV; goto free_resources; } @@ -645,147 +705,252 @@ out: return err; } -static int __devinit ioat_probe(struct pci_dev *pdev, - const struct pci_device_id *ent) +static char ioat_interrupt_style[32] = "msix"; +module_param_string(ioat_interrupt_style, ioat_interrupt_style, + sizeof(ioat_interrupt_style), 0644); +MODULE_PARM_DESC(ioat_interrupt_style, + "set ioat interrupt style: msix (default), " + "msix-single-vector, msi, intx)"); + +/** + * ioat_dma_setup_interrupts - setup interrupt handler + * @device: ioat device + */ +static int ioat_dma_setup_interrupts(struct ioatdma_device *device) { - int err; - unsigned long mmio_start, mmio_len; - void __iomem *reg_base; - struct ioat_device *device; + struct ioat_dma_chan *ioat_chan; + int err, i, j, msixcnt; + u8 intrctrl = 0; + + if (!strcmp(ioat_interrupt_style, "msix")) + goto msix; + if (!strcmp(ioat_interrupt_style, "msix-single-vector")) + goto msix_single_vector; + if (!strcmp(ioat_interrupt_style, "msi")) + goto msi; + if (!strcmp(ioat_interrupt_style, "intx")) + goto intx; + +msix: + /* The number of MSI-X vectors should equal the number of channels */ + msixcnt = device->common.chancnt; + for (i = 0; i < msixcnt; i++) + device->msix_entries[i].entry = i; + + err = pci_enable_msix(device->pdev, device->msix_entries, msixcnt); + if (err < 0) + goto msi; + if (err > 0) + goto msix_single_vector; + + for (i = 0; i < msixcnt; i++) { + ioat_chan = ioat_lookup_chan_by_index(device, i); + err = request_irq(device->msix_entries[i].vector, + ioat_dma_do_interrupt_msix, + 0, "ioat-msix", ioat_chan); + if (err) { + for (j = 0; j < i; j++) { + ioat_chan = + ioat_lookup_chan_by_index(device, j); + free_irq(device->msix_entries[j].vector, + ioat_chan); + } + goto msix_single_vector; + } + } + intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL; + device->irq_mode = msix_multi_vector; + goto done; - err = pci_enable_device(pdev); +msix_single_vector: + device->msix_entries[0].entry = 0; + err = pci_enable_msix(device->pdev, device->msix_entries, 1); if (err) - goto err_enable_device; + goto msi; - err = pci_set_dma_mask(pdev, DMA_64BIT_MASK); - if (err) - err = pci_set_dma_mask(pdev, DMA_32BIT_MASK); + err = request_irq(device->msix_entries[0].vector, ioat_dma_do_interrupt, + 0, "ioat-msix", device); + if (err) { + pci_disable_msix(device->pdev); + goto msi; + } + device->irq_mode = msix_single_vector; + goto done; + +msi: + err = pci_enable_msi(device->pdev); if (err) - goto err_set_dma_mask; + goto intx; - err = pci_request_regions(pdev, ioat_pci_driver.name); + err = request_irq(device->pdev->irq, ioat_dma_do_interrupt, + 0, "ioat-msi", device); + if (err) { + pci_disable_msi(device->pdev); + goto intx; + } + /* + * CB 1.2 devices need a bit set in configuration space to enable MSI + */ + if (device->version == IOAT_VER_1_2) { + u32 dmactrl; + pci_read_config_dword(device->pdev, + IOAT_PCI_DMACTRL_OFFSET, &dmactrl); + dmactrl |= IOAT_PCI_DMACTRL_MSI_EN; + pci_write_config_dword(device->pdev, + IOAT_PCI_DMACTRL_OFFSET, dmactrl); + } + device->irq_mode = msi; + goto done; + +intx: + err = request_irq(device->pdev->irq, ioat_dma_do_interrupt, + IRQF_SHARED, "ioat-intx", device); if (err) - goto err_request_regions; + goto err_no_irq; + device->irq_mode = intx; - mmio_start = pci_resource_start(pdev, 0); - mmio_len = pci_resource_len(pdev, 0); +done: + intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN; + writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET); + return 0; - reg_base = ioremap(mmio_start, mmio_len); - if (!reg_base) { - err = -ENOMEM; - goto err_ioremap; +err_no_irq: + /* Disable all interrupt generation */ + writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); + dev_err(&device->pdev->dev, "no usable interrupts\n"); + device->irq_mode = none; + return -1; +} + +/** + * ioat_dma_remove_interrupts - remove whatever interrupts were set + * @device: ioat device + */ +static void ioat_dma_remove_interrupts(struct ioatdma_device *device) +{ + struct ioat_dma_chan *ioat_chan; + int i; + + /* Disable all interrupt generation */ + writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); + + switch (device->irq_mode) { + case msix_multi_vector: + for (i = 0; i < device->common.chancnt; i++) { + ioat_chan = ioat_lookup_chan_by_index(device, i); + free_irq(device->msix_entries[i].vector, ioat_chan); + } + pci_disable_msix(device->pdev); + break; + case msix_single_vector: + free_irq(device->msix_entries[0].vector, device); + pci_disable_msix(device->pdev); + break; + case msi: + free_irq(device->pdev->irq, device); + pci_disable_msi(device->pdev); + break; + case intx: + free_irq(device->pdev->irq, device); + break; + case none: + dev_warn(&device->pdev->dev, + "call to %s without interrupts setup\n", __func__); } + device->irq_mode = none; +} + +struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev, + void __iomem *iobase) +{ + int err; + struct ioatdma_device *device; device = kzalloc(sizeof(*device), GFP_KERNEL); if (!device) { err = -ENOMEM; goto err_kzalloc; } + device->pdev = pdev; + device->reg_base = iobase; + device->version = readb(device->reg_base + IOAT_VER_OFFSET); /* DMA coherent memory pool for DMA descriptor allocations */ device->dma_pool = pci_pool_create("dma_desc_pool", pdev, - sizeof(struct ioat_dma_descriptor), 64, 0); + sizeof(struct ioat_dma_descriptor), + 64, 0); if (!device->dma_pool) { err = -ENOMEM; goto err_dma_pool; } - device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES); + device->completion_pool = pci_pool_create("completion_pool", pdev, + sizeof(u64), SMP_CACHE_BYTES, + SMP_CACHE_BYTES); if (!device->completion_pool) { err = -ENOMEM; goto err_completion_pool; } - device->pdev = pdev; - pci_set_drvdata(pdev, device); -#ifdef CONFIG_PCI_MSI - if (pci_enable_msi(pdev) == 0) { - device->msi = 1; - } else { - device->msi = 0; - } -#endif - err = request_irq(pdev->irq, &ioat_do_interrupt, IRQF_SHARED, "ioat", - device); - if (err) - goto err_irq; - - device->reg_base = reg_base; - - writeb(IOAT_INTRCTRL_MASTER_INT_EN, device->reg_base + IOAT_INTRCTRL_OFFSET); - pci_set_master(pdev); - INIT_LIST_HEAD(&device->common.channels); - enumerate_dma_channels(device); + ioat_dma_enumerate_channels(device); dma_cap_set(DMA_MEMCPY, device->common.cap_mask); - device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; - device->common.device_free_chan_resources = ioat_dma_free_chan_resources; + device->common.device_alloc_chan_resources = + ioat_dma_alloc_chan_resources; + device->common.device_free_chan_resources = + ioat_dma_free_chan_resources; device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy; device->common.device_is_tx_complete = ioat_dma_is_complete; device->common.device_issue_pending = ioat_dma_memcpy_issue_pending; device->common.device_dependency_added = ioat_dma_dependency_added; device->common.dev = &pdev->dev; - printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", - device->common.chancnt); + dev_err(&device->pdev->dev, + "ioatdma: Intel(R) I/OAT DMA Engine found," + " %d channels, device version 0x%02x\n", + device->common.chancnt, device->version); - err = ioat_self_test(device); + err = ioat_dma_setup_interrupts(device); + if (err) + goto err_setup_interrupts; + + err = ioat_dma_self_test(device); if (err) goto err_self_test; dma_async_device_register(&device->common); - return 0; + return device; err_self_test: -err_irq: + ioat_dma_remove_interrupts(device); +err_setup_interrupts: pci_pool_destroy(device->completion_pool); err_completion_pool: pci_pool_destroy(device->dma_pool); err_dma_pool: kfree(device); err_kzalloc: - iounmap(reg_base); -err_ioremap: - pci_release_regions(pdev); -err_request_regions: -err_set_dma_mask: - pci_disable_device(pdev); -err_enable_device: - - printk(KERN_ERR "Intel(R) I/OAT DMA Engine initialization failed\n"); - - return err; + iounmap(iobase); + dev_err(&device->pdev->dev, + "ioatdma: Intel(R) I/OAT DMA Engine initialization failed\n"); + return NULL; } -static void ioat_shutdown(struct pci_dev *pdev) +void ioat_dma_remove(struct ioatdma_device *device) { - struct ioat_device *device; - device = pci_get_drvdata(pdev); - - dma_async_device_unregister(&device->common); -} - -static void __devexit ioat_remove(struct pci_dev *pdev) -{ - struct ioat_device *device; struct dma_chan *chan, *_chan; struct ioat_dma_chan *ioat_chan; - device = pci_get_drvdata(pdev); dma_async_device_unregister(&device->common); - free_irq(device->pdev->irq, device); -#ifdef CONFIG_PCI_MSI - if (device->msi) - pci_disable_msi(device->pdev); -#endif + ioat_dma_remove_interrupts(device); + pci_pool_destroy(device->dma_pool); pci_pool_destroy(device->completion_pool); - iounmap(device->reg_base); - pci_release_regions(pdev); - pci_disable_device(pdev); - list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) { + + list_for_each_entry_safe(chan, _chan, + &device->common.channels, device_node) { ioat_chan = to_ioat_chan(chan); list_del(&chan->device_node); kfree(ioat_chan); @@ -793,25 +958,3 @@ static void __devexit ioat_remove(struct pci_dev *pdev) kfree(device); } -/* MODULE API */ -MODULE_VERSION("1.9"); -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Intel Corporation"); - -static int __init ioat_init_module(void) -{ - /* it's currently unsafe to unload this module */ - /* if forced, worst case is that rmmod hangs */ - __unsafe(THIS_MODULE); - - return pci_register_driver(&ioat_pci_driver); -} - -module_init(ioat_init_module); - -static void __exit ioat_exit_module(void) -{ - pci_unregister_driver(&ioat_pci_driver); -} - -module_exit(ioat_exit_module); diff --git a/drivers/dma/ioatdma.h b/drivers/dma/ioatdma.h index bf4dad70e0f..2a319e124ec 100644 --- a/drivers/dma/ioatdma.h +++ b/drivers/dma/ioatdma.h @@ -28,25 +28,35 @@ #include <linux/cache.h> #include <linux/pci_ids.h> +enum ioat_interrupt { + none = 0, + msix_multi_vector = 1, + msix_single_vector = 2, + msi = 3, + intx = 4, +}; + #define IOAT_LOW_COMPLETION_MASK 0xffffffc0 /** - * struct ioat_device - internal representation of a IOAT device + * struct ioatdma_device - internal representation of a IOAT device * @pdev: PCI-Express device * @reg_base: MMIO register space base address * @dma_pool: for allocating DMA descriptors * @common: embedded struct dma_device - * @msi: Message Signaled Interrupt number + * @version: version of ioatdma device */ -struct ioat_device { +struct ioatdma_device { struct pci_dev *pdev; void __iomem *reg_base; struct pci_pool *dma_pool; struct pci_pool *completion_pool; - struct dma_device common; - u8 msi; + u8 version; + enum ioat_interrupt irq_mode; + struct msix_entry msix_entries[4]; + struct ioat_dma_chan *idx[4]; }; /** @@ -84,7 +94,7 @@ struct ioat_dma_chan { int pending; - struct ioat_device *device; + struct ioatdma_device *device; struct dma_chan common; dma_addr_t completion_addr; @@ -95,6 +105,7 @@ struct ioat_dma_chan { u32 high; }; } *completion_virt; + struct tasklet_struct cleanup_task; }; /* wrapper around hardware descriptor format + additional software fields */ @@ -117,4 +128,16 @@ struct ioat_desc_sw { struct dma_async_tx_descriptor async_tx; }; +#if defined(CONFIG_INTEL_IOATDMA) || defined(CONFIG_INTEL_IOATDMA_MODULE) +struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev, + void __iomem *iobase); +void ioat_dma_remove(struct ioatdma_device *device); +struct dca_provider *ioat_dca_init(struct pci_dev *pdev, + void __iomem *iobase); +#else +#define ioat_dma_probe(pdev, iobase) NULL +#define ioat_dma_remove(device) do { } while (0) +#define ioat_dca_init(pdev, iobase) NULL +#endif + #endif /* IOATDMA_H */ diff --git a/drivers/dma/ioatdma_hw.h b/drivers/dma/ioatdma_hw.h index 4d7a12880be..9e7434e1551 100644 --- a/drivers/dma/ioatdma_hw.h +++ b/drivers/dma/ioatdma_hw.h @@ -27,7 +27,7 @@ #define IOAT_PCI_RID 0x00 #define IOAT_PCI_SVID 0x8086 #define IOAT_PCI_SID 0x8086 -#define IOAT_VER 0x12 /* Version 1.2 */ +#define IOAT_VER_1_2 0x12 /* Version 1.2 */ struct ioat_dma_descriptor { uint32_t size; diff --git a/drivers/dma/ioatdma_registers.h b/drivers/dma/ioatdma_registers.h index a30c7349075..baaab5ea146 100644 --- a/drivers/dma/ioatdma_registers.h +++ b/drivers/dma/ioatdma_registers.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. + * Copyright(c) 2004 - 2007 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -21,6 +21,9 @@ #ifndef _IOAT_REGISTERS_H_ #define _IOAT_REGISTERS_H_ +#define IOAT_PCI_DMACTRL_OFFSET 0x48 +#define IOAT_PCI_DMACTRL_DMA_EN 0x00000001 +#define IOAT_PCI_DMACTRL_MSI_EN 0x00000002 /* MMIO Device Registers */ #define IOAT_CHANCNT_OFFSET 0x00 /* 8-bit */ @@ -39,6 +42,7 @@ #define IOAT_INTRCTRL_MASTER_INT_EN 0x01 /* Master Interrupt Enable */ #define IOAT_INTRCTRL_INT_STATUS 0x02 /* ATTNSTATUS -or- Channel Int */ #define IOAT_INTRCTRL_INT 0x04 /* INT_STATUS -and- MASTER_INT_EN */ +#define IOAT_INTRCTRL_MSIX_VECTOR_CONTROL 0x08 /* Enable all MSI-X vectors */ #define IOAT_ATTNSTATUS_OFFSET 0x04 /* Each bit is a channel */ |