From e22dde9904c2d26a522f1a2b89854a8238bf0933 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 17 Nov 2009 11:34:31 -0700 Subject: ioat: silence "dca disabled" messages Turning off dca is not an "error", and the dca-enabled state can be viewed from sysfs. Signed-off-by: Dan Williams --- drivers/dma/ioat/dca.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c index 69d02615c4d..14f0a782df4 100644 --- a/drivers/dma/ioat/dca.c +++ b/drivers/dma/ioat/dca.c @@ -98,7 +98,7 @@ static int dca_enabled_in_bios(struct pci_dev *pdev) cpuid_level_9 = cpuid_eax(9); res = test_bit(0, &cpuid_level_9); if (!res) - dev_err(&pdev->dev, "DCA is disabled in BIOS\n"); + dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n"); return res; } @@ -108,7 +108,7 @@ static int system_has_dca_enabled(struct pci_dev *pdev) if (boot_cpu_has(X86_FEATURE_DCA)) return dca_enabled_in_bios(pdev); - dev_err(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n"); + dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n"); return 0; } -- cgit v1.2.3 From 228c4f5cfbf1cda411d9aa7204a612a63c89b1e8 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:07:10 -0700 Subject: ioat3: dca and raid operations are incompatible RAID operations cause a system hang on platforms with DCA (Direct-Cache-Access) enabled. So turn off RAID capabilities in this case. Signed-off-by: Dan Williams --- drivers/dma/ioat/dca.c | 2 +- drivers/dma/ioat/dma_v3.c | 6 ++++++ drivers/dma/ioat/hw.h | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c index 14f0a782df4..abd9038e06b 100644 --- a/drivers/dma/ioat/dca.c +++ b/drivers/dma/ioat/dca.c @@ -103,7 +103,7 @@ static int dca_enabled_in_bios(struct pci_dev *pdev) return res; } -static int system_has_dca_enabled(struct pci_dev *pdev) +int system_has_dca_enabled(struct pci_dev *pdev) { if (boot_cpu_has(X86_FEATURE_DCA)) return dca_enabled_in_bios(pdev); diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 35d1e33afd5..4f305f61f87 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -1117,6 +1117,7 @@ static int __devinit ioat3_dma_self_test(struct ioatdma_device *device) int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) { struct pci_dev *pdev = device->pdev; + int dca_en = system_has_dca_enabled(pdev); struct dma_device *dma; struct dma_chan *c; struct ioat_chan_common *chan; @@ -1137,6 +1138,11 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock; cap = readl(device->reg_base + IOAT_DMA_CAP_OFFSET); + + /* dca is incompatible with raid operations */ + if (dca_en && (cap & (IOAT_CAP_XOR|IOAT_CAP_PQ))) + cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ); + if (cap & IOAT_CAP_XOR) { is_raid_device = true; dma->max_xor = 8; diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h index 99afb12bd40..60e675455b6 100644 --- a/drivers/dma/ioat/hw.h +++ b/drivers/dma/ioat/hw.h @@ -39,6 +39,8 @@ #define IOAT_VER_3_0 0x30 /* Version 3.0 */ #define IOAT_VER_3_2 0x32 /* Version 3.2 */ +int system_has_dca_enabled(struct pci_dev *pdev); + struct ioat_dma_descriptor { uint32_t size; union { -- cgit v1.2.3 From 6f82b83b7a56bc6e9dd6d7b93531dde6027c5309 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:07:57 -0700 Subject: ioat2,3: disable asynchronous error notifications Error interrupts and error completions may cause channel hangs, so poll the channel status register after a timeout. Signed-off-by: Dan Williams --- drivers/dma/ioat/registers.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h index 63038e18ab0..f015ec19670 100644 --- a/drivers/dma/ioat/registers.h +++ b/drivers/dma/ioat/registers.h @@ -92,9 +92,7 @@ #define IOAT_CHANCTRL_ERR_COMPLETION_EN 0x0004 #define IOAT_CHANCTRL_INT_REARM 0x0001 #define IOAT_CHANCTRL_RUN (IOAT_CHANCTRL_INT_REARM |\ - IOAT_CHANCTRL_ERR_COMPLETION_EN |\ - IOAT_CHANCTRL_ANY_ERR_ABORT_EN |\ - IOAT_CHANCTRL_ERR_INT_EN) + IOAT_CHANCTRL_ANY_ERR_ABORT_EN) #define IOAT_DMA_COMP_OFFSET 0x02 /* 16-bit DMA channel compatibility */ #define IOAT_DMA_COMP_V1 0x0001 /* Compatibility with DMA version 1 */ -- cgit v1.2.3 From de581b65f6fe78168affa552c3bd15b8c80ed614 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:08:45 -0700 Subject: ioat3: specify valid address for disabled-Q or disabled-P Although disabled, hardware still checks address validity, so duplicate the known address. Signed-off-by: Dan Williams --- drivers/dma/ioat/dma_v3.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 4f305f61f87..fad2e9fba49 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -736,10 +736,16 @@ ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, unsigned int src_cnt, const unsigned char *scf, size_t len, unsigned long flags) { + /* specify valid address for disabled result */ + if (flags & DMA_PREP_PQ_DISABLE_P) + dst[0] = dst[1]; + if (flags & DMA_PREP_PQ_DISABLE_Q) + dst[1] = dst[0]; + /* handle the single source multiply case from the raid6 * recovery path */ - if (unlikely((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1)) { + if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) { dma_addr_t single_source[2]; unsigned char single_source_coef[2]; @@ -761,6 +767,12 @@ ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, unsigned int src_cnt, const unsigned char *scf, size_t len, enum sum_check_flags *pqres, unsigned long flags) { + /* specify valid address for disabled result */ + if (flags & DMA_PREP_PQ_DISABLE_P) + pq[0] = pq[1]; + if (flags & DMA_PREP_PQ_DISABLE_Q) + pq[1] = pq[0]; + /* the cleanup routine only sets bits on validate failure, it * does not clear bits on validate success... so clear it here */ @@ -778,9 +790,9 @@ ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, dma_addr_t pq[2]; memset(scf, 0, src_cnt); - flags |= DMA_PREP_PQ_DISABLE_Q; pq[0] = dst; - pq[1] = ~0; + flags |= DMA_PREP_PQ_DISABLE_Q; + pq[1] = dst; /* specify valid address for disabled result */ return __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len, flags); @@ -800,9 +812,9 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, *result = 0; memset(scf, 0, src_cnt); - flags |= DMA_PREP_PQ_DISABLE_Q; pq[0] = src[0]; - pq[1] = ~0; + flags |= DMA_PREP_PQ_DISABLE_Q; + pq[1] = pq[0]; /* specify valid address for disabled result */ return __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, scf, len, flags); -- cgit v1.2.3 From b57014def9afc2bd8a62299d2f51b77dad5ae0c7 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:10:07 -0700 Subject: ioat2,3: report all uncorrectable errors Modify is_ioat_bug() to catch all errors that are uncorrectable, or not currently handled. Signed-off-by: Dan Williams --- drivers/dma/ioat/dma.h | 4 +--- drivers/dma/ioat/dma_v2.c | 2 ++ drivers/dma/ioat/dma_v3.c | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index c14fdfeb7f3..45edde99648 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -297,9 +297,7 @@ static inline bool is_ioat_suspended(unsigned long status) /* channel was fatally programmed */ static inline bool is_ioat_bug(unsigned long err) { - return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR|IOAT_CHANERR_DEST_ADDR_ERR| - IOAT_CHANERR_NEXT_ADDR_ERR|IOAT_CHANERR_CONTROL_ERR| - IOAT_CHANERR_LENGTH_ERR)); + return !!err; } static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len, diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c index 96ffab7d37a..8f1f7f05dea 100644 --- a/drivers/dma/ioat/dma_v2.c +++ b/drivers/dma/ioat/dma_v2.c @@ -279,6 +279,8 @@ void ioat2_timer_event(unsigned long data) u32 chanerr; chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); + dev_err(to_dev(chan), "%s: Channel halted (%x)\n", + __func__, chanerr); BUG_ON(is_ioat_bug(chanerr)); } diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index fad2e9fba49..252cf2d3f1d 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -378,6 +378,8 @@ static void ioat3_timer_event(unsigned long data) u32 chanerr; chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); + dev_err(to_dev(chan), "%s: Channel halted (%x)\n", + __func__, chanerr); BUG_ON(is_ioat_bug(chanerr)); } -- cgit v1.2.3 From 4499a24dec00e037da7d09caccad45e7594a9c19 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:10:25 -0700 Subject: dmaengine: include xor/pq validate in device_has_all_tx_types() A channel must include these capabilities to satisfy ASYNC_TX_DISABLE_CHANNEL_SWITCH. Signed-off-by: Dan Williams --- drivers/dma/dmaengine.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'drivers') diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index bd0b248de2c..b6442f09d0f 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -632,11 +632,15 @@ static bool device_has_all_tx_types(struct dma_device *device) #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE) if (!dma_has_cap(DMA_XOR, device->cap_mask)) return false; + if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask)) + return false; #endif #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE) if (!dma_has_cap(DMA_PQ, device->cap_mask)) return false; + if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask)) + return false; #endif return true; -- cgit v1.2.3 From 7b3cc2b1fc2066391e498f3387204908c4eced21 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:10:37 -0700 Subject: async_tx: build-time toggling of async_{syndrome,xor}_val dma support ioat3.2 does not support asynchronous error notifications which makes the driver experience latencies when non-zero pq validate results are expected. Provide a mechanism for turning off async_xor_val and async_syndrome_val via Kconfig. This approach is generally useful for any driver that specifies ASYNC_TX_DISABLE_CHANNEL_SWITCH and would like to force the async_tx api to fall back to the synchronous path for certain operations. Signed-off-by: Dan Williams --- drivers/dma/Kconfig | 2 ++ drivers/dma/dmaengine.c | 6 ++++++ drivers/dma/ioat/dma_v3.c | 10 ++++++++++ 3 files changed, 18 insertions(+) (limited to 'drivers') diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 5903a88351b..b401dadad4a 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -26,6 +26,8 @@ config INTEL_IOATDMA select DMA_ENGINE select DCA select ASYNC_TX_DISABLE_CHANNEL_SWITCH + select ASYNC_TX_DISABLE_PQ_VAL_DMA + select ASYNC_TX_DISABLE_XOR_VAL_DMA help Enable support for the Intel(R) I/OAT DMA engine present in recent Intel Xeon chipsets. diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index b6442f09d0f..8f99354082c 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -632,16 +632,22 @@ static bool device_has_all_tx_types(struct dma_device *device) #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE) if (!dma_has_cap(DMA_XOR, device->cap_mask)) return false; + + #ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask)) return false; #endif + #endif #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE) if (!dma_has_cap(DMA_PQ, device->cap_mask)) return false; + + #ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask)) return false; #endif + #endif return true; } diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 252cf2d3f1d..189788f6351 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -1206,6 +1206,16 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) device->timer_fn = ioat2_timer_event; } + #ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA + dma_cap_clear(DMA_PQ_VAL, dma->cap_mask); + dma->device_prep_dma_pq_val = NULL; + #endif + + #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA + dma_cap_clear(DMA_XOR_VAL, dma->cap_mask); + dma->device_prep_dma_xor_val = NULL; + #endif + /* -= IOAT ver.3 workarounds =- */ /* Write CHANERRMSK_INT with 3E07h to mask out the errors * that can cause stability issues for IOAT ver.3 -- cgit v1.2.3 From 49954c1567cb0d70d28bb5512d471dc5bd4e2c3f Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:11:03 -0700 Subject: ioat3: fix pq completion versus channel deallocation race The completion of a pq operation is notified with a null descriptor appended to the end of the chain. This descriptor needs to be visible to dma clients otherwise the client is precluded from ensuring all operations are quiesced before freeing channel resources, i.e. due to descriptor polling it may get the completion notification ahead of the interrupt delivered by the null descriptor. Signed-off-by: Dan Williams --- drivers/dma/ioat/dma_v3.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 189788f6351..42f6f10fb0c 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -571,7 +571,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, dump_desc_dbg(ioat, compl_desc); /* we leave the channel locked to ensure in order submission */ - return &desc->txd; + return &compl_desc->txd; } static struct dma_async_tx_descriptor * @@ -730,7 +730,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, dump_desc_dbg(ioat, compl_desc); /* we leave the channel locked to ensure in order submission */ - return &desc->txd; + return &compl_desc->txd; } static struct dma_async_tx_descriptor * -- cgit v1.2.3 From 56adf7e8127d601b172e180b44551ce83404348f Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sun, 22 Nov 2009 12:10:10 -0700 Subject: shdma: fix initialization error handling 1/ Error handling code following a kzalloc should free the allocated data. 2/ Report an error when no platform data is detected Both problems fixed by moving the platform data check before the allocation, and allows a goto to be killed. Reported-by: Julia Lawall Acked-by: Julia Lawall Signed-off-by: Dan Williams --- drivers/dma/shdma.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c index b3b065c4e5c..034ecf0ace0 100644 --- a/drivers/dma/shdma.c +++ b/drivers/dma/shdma.c @@ -640,17 +640,16 @@ static int __init sh_dmae_probe(struct platform_device *pdev) #endif struct sh_dmae_device *shdev; + /* get platform data */ + if (!pdev->dev.platform_data) + return -ENODEV; + shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL); if (!shdev) { dev_err(&pdev->dev, "No enough memory\n"); - err = -ENOMEM; - goto shdev_err; + return -ENOMEM; } - /* get platform data */ - if (!pdev->dev.platform_data) - goto shdev_err; - /* platform data */ memcpy(&shdev->pdata, pdev->dev.platform_data, sizeof(struct sh_dmae_pdata)); @@ -722,7 +721,6 @@ eirq_err: rst_err: kfree(shdev); -shdev_err: return err; } -- cgit v1.2.3