From 9b513090a3c5e4964f9ac09016c1586988abb3d5 Mon Sep 17 00:00:00 2001 From: Ralph Campbell Date: Tue, 12 Dec 2006 14:27:41 -0800 Subject: IB: Add DMA mapping functions to allow device drivers to interpose The QLogic InfiniPath HCAs use programmed I/O instead of HW DMA. This patch allows a verbs device driver to interpose on DMA mapping function calls in order to avoid relying on bus_to_virt() and phys_to_virt() to undo the mappings created by dma_map_single(), dma_map_sg(), etc. Signed-off-by: Ralph Campbell Signed-off-by: Roland Dreier --- include/rdma/ib_verbs.h | 253 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) (limited to 'include/rdma/ib_verbs.h') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 8eacc351099..fd2353fa7e1 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -43,6 +43,8 @@ #include #include +#include +#include #include #include @@ -848,6 +850,49 @@ struct ib_cache { u8 *lmc_cache; }; +struct ib_dma_mapping_ops { + int (*mapping_error)(struct ib_device *dev, + u64 dma_addr); + u64 (*map_single)(struct ib_device *dev, + void *ptr, size_t size, + enum dma_data_direction direction); + void (*unmap_single)(struct ib_device *dev, + u64 addr, size_t size, + enum dma_data_direction direction); + u64 (*map_page)(struct ib_device *dev, + struct page *page, unsigned long offset, + size_t size, + enum dma_data_direction direction); + void (*unmap_page)(struct ib_device *dev, + u64 addr, size_t size, + enum dma_data_direction direction); + int (*map_sg)(struct ib_device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction direction); + void (*unmap_sg)(struct ib_device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction direction); + u64 (*dma_address)(struct ib_device *dev, + struct scatterlist *sg); + unsigned int (*dma_len)(struct ib_device *dev, + struct scatterlist *sg); + void (*sync_single_for_cpu)(struct ib_device *dev, + u64 dma_handle, + size_t size, + enum dma_data_direction dir); + void (*sync_single_for_device)(struct ib_device *dev, + u64 dma_handle, + size_t size, + enum dma_data_direction dir); + void *(*alloc_coherent)(struct ib_device *dev, + size_t size, + u64 *dma_handle, + gfp_t flag); + void (*free_coherent)(struct ib_device *dev, + size_t size, void *cpu_addr, + u64 dma_handle); +}; + struct iw_cm_verbs; struct ib_device { @@ -992,6 +1037,8 @@ struct ib_device { struct ib_mad *in_mad, struct ib_mad *out_mad); + struct ib_dma_mapping_ops *dma_ops; + struct module *owner; struct class_device class_dev; struct kobject ports_parent; @@ -1395,9 +1442,215 @@ static inline int ib_req_ncomp_notif(struct ib_cq *cq, int wc_cnt) * usable for DMA. * @pd: The protection domain associated with the memory region. * @mr_access_flags: Specifies the memory access rights. + * + * Note that the ib_dma_*() functions defined below must be used + * to create/destroy addresses used with the Lkey or Rkey returned + * by ib_get_dma_mr(). */ struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags); +/** + * ib_dma_mapping_error - check a DMA addr for error + * @dev: The device for which the dma_addr was created + * @dma_addr: The DMA address to check + */ +static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr) +{ + return dev->dma_ops ? + dev->dma_ops->mapping_error(dev, dma_addr) : + dma_mapping_error(dma_addr); +} + +/** + * ib_dma_map_single - Map a kernel virtual address to DMA address + * @dev: The device for which the dma_addr is to be created + * @cpu_addr: The kernel virtual address + * @size: The size of the region in bytes + * @direction: The direction of the DMA + */ +static inline u64 ib_dma_map_single(struct ib_device *dev, + void *cpu_addr, size_t size, + enum dma_data_direction direction) +{ + return dev->dma_ops ? + dev->dma_ops->map_single(dev, cpu_addr, size, direction) : + dma_map_single(dev->dma_device, cpu_addr, size, direction); +} + +/** + * ib_dma_unmap_single - Destroy a mapping created by ib_dma_map_single() + * @dev: The device for which the DMA address was created + * @addr: The DMA address + * @size: The size of the region in bytes + * @direction: The direction of the DMA + */ +static inline void ib_dma_unmap_single(struct ib_device *dev, + u64 addr, size_t size, + enum dma_data_direction direction) +{ + dev->dma_ops ? + dev->dma_ops->unmap_single(dev, addr, size, direction) : + dma_unmap_single(dev->dma_device, addr, size, direction); +} + +/** + * ib_dma_map_page - Map a physical page to DMA address + * @dev: The device for which the dma_addr is to be created + * @page: The page to be mapped + * @offset: The offset within the page + * @size: The size of the region in bytes + * @direction: The direction of the DMA + */ +static inline u64 ib_dma_map_page(struct ib_device *dev, + struct page *page, + unsigned long offset, + size_t size, + enum dma_data_direction direction) +{ + return dev->dma_ops ? + dev->dma_ops->map_page(dev, page, offset, size, direction) : + dma_map_page(dev->dma_device, page, offset, size, direction); +} + +/** + * ib_dma_unmap_page - Destroy a mapping created by ib_dma_map_page() + * @dev: The device for which the DMA address was created + * @addr: The DMA address + * @size: The size of the region in bytes + * @direction: The direction of the DMA + */ +static inline void ib_dma_unmap_page(struct ib_device *dev, + u64 addr, size_t size, + enum dma_data_direction direction) +{ + dev->dma_ops ? + dev->dma_ops->unmap_page(dev, addr, size, direction) : + dma_unmap_page(dev->dma_device, addr, size, direction); +} + +/** + * ib_dma_map_sg - Map a scatter/gather list to DMA addresses + * @dev: The device for which the DMA addresses are to be created + * @sg: The array of scatter/gather entries + * @nents: The number of scatter/gather entries + * @direction: The direction of the DMA + */ +static inline int ib_dma_map_sg(struct ib_device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction direction) +{ + return dev->dma_ops ? + dev->dma_ops->map_sg(dev, sg, nents, direction) : + dma_map_sg(dev->dma_device, sg, nents, direction); +} + +/** + * ib_dma_unmap_sg - Unmap a scatter/gather list of DMA addresses + * @dev: The device for which the DMA addresses were created + * @sg: The array of scatter/gather entries + * @nents: The number of scatter/gather entries + * @direction: The direction of the DMA + */ +static inline void ib_dma_unmap_sg(struct ib_device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction direction) +{ + dev->dma_ops ? + dev->dma_ops->unmap_sg(dev, sg, nents, direction) : + dma_unmap_sg(dev->dma_device, sg, nents, direction); +} + +/** + * ib_sg_dma_address - Return the DMA address from a scatter/gather entry + * @dev: The device for which the DMA addresses were created + * @sg: The scatter/gather entry + */ +static inline u64 ib_sg_dma_address(struct ib_device *dev, + struct scatterlist *sg) +{ + return dev->dma_ops ? + dev->dma_ops->dma_address(dev, sg) : sg_dma_address(sg); +} + +/** + * ib_sg_dma_len - Return the DMA length from a scatter/gather entry + * @dev: The device for which the DMA addresses were created + * @sg: The scatter/gather entry + */ +static inline unsigned int ib_sg_dma_len(struct ib_device *dev, + struct scatterlist *sg) +{ + return dev->dma_ops ? + dev->dma_ops->dma_len(dev, sg) : sg_dma_len(sg); +} + +/** + * ib_dma_sync_single_for_cpu - Prepare DMA region to be accessed by CPU + * @dev: The device for which the DMA address was created + * @addr: The DMA address + * @size: The size of the region in bytes + * @dir: The direction of the DMA + */ +static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev, + u64 addr, + size_t size, + enum dma_data_direction dir) +{ + dev->dma_ops ? + dev->dma_ops->sync_single_for_cpu(dev, addr, size, dir) : + dma_sync_single_for_cpu(dev->dma_device, addr, size, dir); +} + +/** + * ib_dma_sync_single_for_device - Prepare DMA region to be accessed by device + * @dev: The device for which the DMA address was created + * @addr: The DMA address + * @size: The size of the region in bytes + * @dir: The direction of the DMA + */ +static inline void ib_dma_sync_single_for_device(struct ib_device *dev, + u64 addr, + size_t size, + enum dma_data_direction dir) +{ + dev->dma_ops ? + dev->dma_ops->sync_single_for_device(dev, addr, size, dir) : + dma_sync_single_for_device(dev->dma_device, addr, size, dir); +} + +/** + * ib_dma_alloc_coherent - Allocate memory and map it for DMA + * @dev: The device for which the DMA address is requested + * @size: The size of the region to allocate in bytes + * @dma_handle: A pointer for returning the DMA address of the region + * @flag: memory allocator flags + */ +static inline void *ib_dma_alloc_coherent(struct ib_device *dev, + size_t size, + u64 *dma_handle, + gfp_t flag) +{ + return dev->dma_ops ? + dev->dma_ops->alloc_coherent(dev, size, dma_handle, flag) : + dma_alloc_coherent(dev->dma_device, size, dma_handle, flag); +} + +/** + * ib_dma_free_coherent - Free memory allocated by ib_dma_alloc_coherent() + * @dev: The device for which the DMA addresses were allocated + * @size: The size of the region + * @cpu_addr: the address returned by ib_dma_alloc_coherent() + * @dma_handle: the DMA address returned by ib_dma_alloc_coherent() + */ +static inline void ib_dma_free_coherent(struct ib_device *dev, + size_t size, void *cpu_addr, + u64 dma_handle) +{ + dev->dma_ops ? + dev->dma_ops->free_coherent(dev, size, cpu_addr, dma_handle) : + dma_free_coherent(dev->dma_device, size, cpu_addr, dma_handle); +} + /** * ib_reg_phys_mr - Prepares a virtually addressed memory region for use * by an HCA. -- cgit v1.2.3 From d1998ef38a13c4e74c69df55ccd38b0440c429b2 Mon Sep 17 00:00:00 2001 From: Ben Collins Date: Wed, 13 Dec 2006 22:10:05 -0500 Subject: [PATCH] ib_verbs: Use explicit if-else statements to avoid errors with do-while macros At least on PPC, the "op ? op : dma" construct causes a compile failure because the dma_* is a do{}while(0) macro. This turns all of them into proper if/else to avoid this problem. Signed-off-by: Ben Collins Signed-off-by: Linus Torvalds --- include/rdma/ib_verbs.h | 70 +++++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 31 deletions(-) (limited to 'include/rdma/ib_verbs.h') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index fd2353fa7e1..3c2e10574b2 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1456,9 +1456,9 @@ struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags); */ static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr) { - return dev->dma_ops ? - dev->dma_ops->mapping_error(dev, dma_addr) : - dma_mapping_error(dma_addr); + if (dev->dma_ops) + return dev->dma_ops->mapping_error(dev, dma_addr); + return dma_mapping_error(dma_addr); } /** @@ -1472,9 +1472,9 @@ static inline u64 ib_dma_map_single(struct ib_device *dev, void *cpu_addr, size_t size, enum dma_data_direction direction) { - return dev->dma_ops ? - dev->dma_ops->map_single(dev, cpu_addr, size, direction) : - dma_map_single(dev->dma_device, cpu_addr, size, direction); + if (dev->dma_ops) + return dev->dma_ops->map_single(dev, cpu_addr, size, direction); + return dma_map_single(dev->dma_device, cpu_addr, size, direction); } /** @@ -1488,8 +1488,9 @@ static inline void ib_dma_unmap_single(struct ib_device *dev, u64 addr, size_t size, enum dma_data_direction direction) { - dev->dma_ops ? - dev->dma_ops->unmap_single(dev, addr, size, direction) : + if (dev->dma_ops) + dev->dma_ops->unmap_single(dev, addr, size, direction); + else dma_unmap_single(dev->dma_device, addr, size, direction); } @@ -1507,9 +1508,9 @@ static inline u64 ib_dma_map_page(struct ib_device *dev, size_t size, enum dma_data_direction direction) { - return dev->dma_ops ? - dev->dma_ops->map_page(dev, page, offset, size, direction) : - dma_map_page(dev->dma_device, page, offset, size, direction); + if (dev->dma_ops) + return dev->dma_ops->map_page(dev, page, offset, size, direction); + return dma_map_page(dev->dma_device, page, offset, size, direction); } /** @@ -1523,8 +1524,9 @@ static inline void ib_dma_unmap_page(struct ib_device *dev, u64 addr, size_t size, enum dma_data_direction direction) { - dev->dma_ops ? - dev->dma_ops->unmap_page(dev, addr, size, direction) : + if (dev->dma_ops) + dev->dma_ops->unmap_page(dev, addr, size, direction); + else dma_unmap_page(dev->dma_device, addr, size, direction); } @@ -1539,9 +1541,9 @@ static inline int ib_dma_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction) { - return dev->dma_ops ? - dev->dma_ops->map_sg(dev, sg, nents, direction) : - dma_map_sg(dev->dma_device, sg, nents, direction); + if (dev->dma_ops) + return dev->dma_ops->map_sg(dev, sg, nents, direction); + return dma_map_sg(dev->dma_device, sg, nents, direction); } /** @@ -1555,8 +1557,9 @@ static inline void ib_dma_unmap_sg(struct ib_device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction) { - dev->dma_ops ? - dev->dma_ops->unmap_sg(dev, sg, nents, direction) : + if (dev->dma_ops) + dev->dma_ops->unmap_sg(dev, sg, nents, direction); + else dma_unmap_sg(dev->dma_device, sg, nents, direction); } @@ -1568,8 +1571,9 @@ static inline void ib_dma_unmap_sg(struct ib_device *dev, static inline u64 ib_sg_dma_address(struct ib_device *dev, struct scatterlist *sg) { - return dev->dma_ops ? - dev->dma_ops->dma_address(dev, sg) : sg_dma_address(sg); + if (dev->dma_ops) + return dev->dma_ops->dma_address(dev, sg); + return sg_dma_address(sg); } /** @@ -1580,8 +1584,9 @@ static inline u64 ib_sg_dma_address(struct ib_device *dev, static inline unsigned int ib_sg_dma_len(struct ib_device *dev, struct scatterlist *sg) { - return dev->dma_ops ? - dev->dma_ops->dma_len(dev, sg) : sg_dma_len(sg); + if (dev->dma_ops) + return dev->dma_ops->dma_len(dev, sg); + return sg_dma_len(sg); } /** @@ -1596,8 +1601,9 @@ static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev, size_t size, enum dma_data_direction dir) { - dev->dma_ops ? - dev->dma_ops->sync_single_for_cpu(dev, addr, size, dir) : + if (dev->dma_ops) + dev->dma_ops->sync_single_for_cpu(dev, addr, size, dir); + else dma_sync_single_for_cpu(dev->dma_device, addr, size, dir); } @@ -1613,8 +1619,9 @@ static inline void ib_dma_sync_single_for_device(struct ib_device *dev, size_t size, enum dma_data_direction dir) { - dev->dma_ops ? - dev->dma_ops->sync_single_for_device(dev, addr, size, dir) : + if (dev->dma_ops) + dev->dma_ops->sync_single_for_device(dev, addr, size, dir); + else dma_sync_single_for_device(dev->dma_device, addr, size, dir); } @@ -1630,9 +1637,9 @@ static inline void *ib_dma_alloc_coherent(struct ib_device *dev, u64 *dma_handle, gfp_t flag) { - return dev->dma_ops ? - dev->dma_ops->alloc_coherent(dev, size, dma_handle, flag) : - dma_alloc_coherent(dev->dma_device, size, dma_handle, flag); + if (dev->dma_ops) + return dev->dma_ops->alloc_coherent(dev, size, dma_handle, flag); + return dma_alloc_coherent(dev->dma_device, size, dma_handle, flag); } /** @@ -1646,8 +1653,9 @@ static inline void ib_dma_free_coherent(struct ib_device *dev, size_t size, void *cpu_addr, u64 dma_handle) { - dev->dma_ops ? - dev->dma_ops->free_coherent(dev, size, cpu_addr, dma_handle) : + if (dev->dma_ops) + dev->dma_ops->free_coherent(dev, size, cpu_addr, dma_handle); + else dma_free_coherent(dev->dma_device, size, cpu_addr, dma_handle); } -- cgit v1.2.3 From c59a3da1342ff456e5123361739bc331446cda21 Mon Sep 17 00:00:00 2001 From: Roland Dreier Date: Fri, 15 Dec 2006 13:57:26 -0800 Subject: IB: Fix ib_dma_alloc_coherent() wrapper The ib_dma_alloc_coherent() wrapper uses a u64* for the dma_handle parameter, unlike dma_alloc_coherent, which uses dma_addr_t*. This means that we need a temporary variable to handle the case when ib_dma_alloc_coherent() just falls through directly to dma_alloc_coherent() on architectures where sizeof u64 != sizeof dma_addr_t. Signed-off-by: Roland Dreier --- include/rdma/ib_verbs.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'include/rdma/ib_verbs.h') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 3c2e10574b2..0bfa3328d68 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1639,7 +1639,14 @@ static inline void *ib_dma_alloc_coherent(struct ib_device *dev, { if (dev->dma_ops) return dev->dma_ops->alloc_coherent(dev, size, dma_handle, flag); - return dma_alloc_coherent(dev->dma_device, size, dma_handle, flag); + else { + dma_addr_t handle; + void *ret; + + ret = dma_alloc_coherent(dev->dma_device, size, &handle, flag); + *dma_handle = handle; + return ret; + } } /** -- cgit v1.2.3