From c3673464ebc004a3d82063cd41b9cf74d1b55db2 Mon Sep 17 00:00:00 2001 From: Karen Xie Date: Tue, 9 Dec 2008 14:15:32 -0800 Subject: [SCSI] cxgb3i: Add cxgb3i iSCSI driver. This patch implements the cxgb3i iscsi connection acceleration for the open-iscsi initiator. The cxgb3i driver offers the iscsi PDU based offload: - digest insertion and verification - payload direct-placement into host memory buffer. Signed-off-by: Karen Xie Signed-off-by: James Bottomley --- drivers/scsi/cxgb3i/cxgb3i_pdu.c | 402 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 402 insertions(+) create mode 100644 drivers/scsi/cxgb3i/cxgb3i_pdu.c (limited to 'drivers/scsi/cxgb3i/cxgb3i_pdu.c') diff --git a/drivers/scsi/cxgb3i/cxgb3i_pdu.c b/drivers/scsi/cxgb3i/cxgb3i_pdu.c new file mode 100644 index 00000000000..ce7ce8c6094 --- /dev/null +++ b/drivers/scsi/cxgb3i/cxgb3i_pdu.c @@ -0,0 +1,402 @@ +/* + * cxgb3i_pdu.c: Chelsio S3xx iSCSI driver. + * + * Copyright (c) 2008 Chelsio Communications, Inc. + * Copyright (c) 2008 Mike Christie + * Copyright (c) 2008 Red Hat, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + * + * Written by: Karen Xie (kxie@chelsio.com) + */ + +#include +#include +#include +#include + +#include "cxgb3i.h" +#include "cxgb3i_pdu.h" + +#ifdef __DEBUG_CXGB3I_RX__ +#define cxgb3i_rx_debug cxgb3i_log_debug +#else +#define cxgb3i_rx_debug(fmt...) +#endif + +#ifdef __DEBUG_CXGB3I_TX__ +#define cxgb3i_tx_debug cxgb3i_log_debug +#else +#define cxgb3i_tx_debug(fmt...) +#endif + +static struct page *pad_page; + +/* + * pdu receive, interact with libiscsi_tcp + */ +static inline int read_pdu_skb(struct iscsi_conn *conn, struct sk_buff *skb, + unsigned int offset, int offloaded) +{ + int status = 0; + int bytes_read; + + bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status); + switch (status) { + case ISCSI_TCP_CONN_ERR: + return -EIO; + case ISCSI_TCP_SUSPENDED: + /* no transfer - just have caller flush queue */ + return bytes_read; + case ISCSI_TCP_SKB_DONE: + /* + * pdus should always fit in the skb and we should get + * segment done notifcation. + */ + iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb."); + return -EFAULT; + case ISCSI_TCP_SEGMENT_DONE: + return bytes_read; + default: + iscsi_conn_printk(KERN_ERR, conn, "Invalid iscsi_tcp_recv_skb " + "status %d\n", status); + return -EINVAL; + } +} + +static int cxgb3i_conn_read_pdu_skb(struct iscsi_conn *conn, + struct sk_buff *skb) +{ + struct iscsi_tcp_conn *tcp_conn = conn->dd_data; + bool offloaded = 0; + unsigned int offset; + int rc; + + cxgb3i_rx_debug("conn 0x%p, skb 0x%p, len %u, flag 0x%x.\n", + conn, skb, skb->len, skb_ulp_mode(skb)); + + if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) { + iscsi_conn_failure(conn, ISCSI_ERR_PROTO); + return -EIO; + } + + if (conn->hdrdgst_en && (skb_ulp_mode(skb) & ULP2_FLAG_HCRC_ERROR)) { + iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST); + return -EIO; + } + + if (conn->datadgst_en && (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) { + iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST); + return -EIO; + } + + /* iscsi hdr */ + rc = read_pdu_skb(conn, skb, 0, 0); + if (rc <= 0) + return rc; + + if (iscsi_tcp_recv_segment_is_hdr(tcp_conn)) + return 0; + + offset = rc; + if (conn->hdrdgst_en) + offset += ISCSI_DIGEST_SIZE; + + /* iscsi data */ + if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) { + cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, ddp'ed, " + "itt 0x%x.\n", + skb, + tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK, + tcp_conn->in.datalen, + ntohl(tcp_conn->in.hdr->itt)); + offloaded = 1; + } else { + cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, NOT ddp'ed, " + "itt 0x%x.\n", + skb, + tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK, + tcp_conn->in.datalen, + ntohl(tcp_conn->in.hdr->itt)); + offset += sizeof(struct cpl_iscsi_hdr_norss); + } + + rc = read_pdu_skb(conn, skb, offset, offloaded); + if (rc < 0) + return rc; + else + return 0; +} + +/* + * pdu transmit, interact with libiscsi_tcp + */ +static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc) +{ + u8 submode = 0; + + if (hcrc) + submode |= 1; + if (dcrc) + submode |= 2; + skb_ulp_mode(skb) = (ULP_MODE_ISCSI << 4) | submode; +} + +void cxgb3i_conn_cleanup_task(struct iscsi_task *task) +{ + struct iscsi_tcp_task *tcp_task = task->dd_data; + + /* never reached the xmit task callout */ + if (tcp_task->dd_data) + kfree_skb(tcp_task->dd_data); + tcp_task->dd_data = NULL; + + /* MNC - Do we need a check in case this is called but + * cxgb3i_conn_alloc_pdu has never been called on the task */ + cxgb3i_release_itt(task, task->hdr_itt); + iscsi_tcp_cleanup_task(task); +} + +/* + * We do not support ahs yet + */ +int cxgb3i_conn_alloc_pdu(struct iscsi_task *task, u8 opcode) +{ + struct iscsi_tcp_task *tcp_task = task->dd_data; + struct sk_buff *skb; + + task->hdr = NULL; + /* always allocate rooms for AHS */ + skb = alloc_skb(sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE + + TX_HEADER_LEN, GFP_ATOMIC); + if (!skb) + return -ENOMEM; + + cxgb3i_tx_debug("task 0x%p, opcode 0x%x, skb 0x%p.\n", + task, opcode, skb); + + tcp_task->dd_data = skb; + skb_reserve(skb, TX_HEADER_LEN); + task->hdr = (struct iscsi_hdr *)skb->data; + task->hdr_max = sizeof(struct iscsi_hdr); + + /* data_out uses scsi_cmd's itt */ + if (opcode != ISCSI_OP_SCSI_DATA_OUT) + cxgb3i_reserve_itt(task, &task->hdr->itt); + + return 0; +} + +int cxgb3i_conn_init_pdu(struct iscsi_task *task, unsigned int offset, + unsigned int count) +{ + struct iscsi_tcp_task *tcp_task = task->dd_data; + struct sk_buff *skb = tcp_task->dd_data; + struct iscsi_conn *conn = task->conn; + struct page *pg; + unsigned int datalen = count; + int i, padlen = iscsi_padding(count); + skb_frag_t *frag; + + cxgb3i_tx_debug("task 0x%p,0x%p, offset %u, count %u, skb 0x%p.\n", + task, task->sc, offset, count, skb); + + skb_put(skb, task->hdr_len); + tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0); + if (!count) + return 0; + + if (task->sc) { + struct scatterlist *sg; + struct scsi_data_buffer *sdb; + unsigned int sgoffset = offset; + struct page *sgpg; + unsigned int sglen; + + sdb = scsi_out(task->sc); + sg = sdb->table.sgl; + + for_each_sg(sdb->table.sgl, sg, sdb->table.nents, i) { + cxgb3i_tx_debug("sg %d, page 0x%p, len %u offset %u\n", + i, sg_page(sg), sg->length, sg->offset); + + if (sgoffset < sg->length) + break; + sgoffset -= sg->length; + } + sgpg = sg_page(sg); + sglen = sg->length - sgoffset; + + do { + int j = skb_shinfo(skb)->nr_frags; + unsigned int copy; + + if (!sglen) { + sg = sg_next(sg); + sgpg = sg_page(sg); + sgoffset = 0; + sglen = sg->length; + ++i; + } + copy = min(sglen, datalen); + if (j && skb_can_coalesce(skb, j, sgpg, + sg->offset + sgoffset)) { + skb_shinfo(skb)->frags[j - 1].size += copy; + } else { + get_page(sgpg); + skb_fill_page_desc(skb, j, sgpg, + sg->offset + sgoffset, copy); + } + sgoffset += copy; + sglen -= copy; + datalen -= copy; + } while (datalen); + } else { + pg = virt_to_page(task->data); + + while (datalen) { + i = skb_shinfo(skb)->nr_frags; + frag = &skb_shinfo(skb)->frags[i]; + + get_page(pg); + frag->page = pg; + frag->page_offset = 0; + frag->size = min((unsigned int)PAGE_SIZE, datalen); + + skb_shinfo(skb)->nr_frags++; + datalen -= frag->size; + pg++; + } + } + + if (padlen) { + i = skb_shinfo(skb)->nr_frags; + frag = &skb_shinfo(skb)->frags[i]; + frag->page = pad_page; + frag->page_offset = 0; + frag->size = padlen; + skb_shinfo(skb)->nr_frags++; + } + + datalen = count + padlen; + skb->data_len += datalen; + skb->truesize += datalen; + skb->len += datalen; + return 0; +} + +int cxgb3i_conn_xmit_pdu(struct iscsi_task *task) +{ + struct iscsi_tcp_task *tcp_task = task->dd_data; + struct sk_buff *skb = tcp_task->dd_data; + struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; + struct cxgb3i_conn *cconn = tcp_conn->dd_data; + unsigned int datalen; + int err; + + if (!skb) + return 0; + + datalen = skb->data_len; + tcp_task->dd_data = NULL; + err = cxgb3i_c3cn_send_pdus(cconn->cep->c3cn, skb); + cxgb3i_tx_debug("task 0x%p, skb 0x%p, len %u/%u, rv %d.\n", + task, skb, skb->len, skb->data_len, err); + if (err > 0) { + int pdulen = err; + + if (task->conn->hdrdgst_en) + pdulen += ISCSI_DIGEST_SIZE; + if (datalen && task->conn->datadgst_en) + pdulen += ISCSI_DIGEST_SIZE; + + task->conn->txdata_octets += pdulen; + return 0; + } + + if (err < 0 && err != -EAGAIN) { + kfree_skb(skb); + cxgb3i_tx_debug("itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n", + task->itt, skb, skb->len, skb->data_len, err); + iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err); + iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED); + return err; + } + /* reset skb to send when we are called again */ + tcp_task->dd_data = skb; + return -EAGAIN; +} + +int cxgb3i_pdu_init(void) +{ + pad_page = alloc_page(GFP_KERNEL); + if (!pad_page) + return -ENOMEM; + memset(page_address(pad_page), 0, PAGE_SIZE); + return 0; +} + +void cxgb3i_pdu_cleanup(void) +{ + if (pad_page) { + __free_page(pad_page); + pad_page = NULL; + } +} + +void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn) +{ + struct sk_buff *skb; + unsigned int read = 0; + struct iscsi_conn *conn = c3cn->user_data; + int err = 0; + + cxgb3i_rx_debug("cn 0x%p.\n", c3cn); + + read_lock(&c3cn->callback_lock); + if (unlikely(!conn || conn->suspend_rx)) { + cxgb3i_rx_debug("conn 0x%p, id %d, suspend_rx %lu!\n", + conn, conn ? conn->id : 0xFF, + conn ? conn->suspend_rx : 0xFF); + read_unlock(&c3cn->callback_lock); + return; + } + skb = skb_peek(&c3cn->receive_queue); + while (!err && skb) { + __skb_unlink(skb, &c3cn->receive_queue); + read += skb_ulp_pdulen(skb); + err = cxgb3i_conn_read_pdu_skb(conn, skb); + __kfree_skb(skb); + skb = skb_peek(&c3cn->receive_queue); + } + read_unlock(&c3cn->callback_lock); + if (c3cn) { + c3cn->copied_seq += read; + cxgb3i_c3cn_rx_credits(c3cn, read); + } + conn->rxdata_octets += read; +} + +void cxgb3i_conn_tx_open(struct s3_conn *c3cn) +{ + struct iscsi_conn *conn = c3cn->user_data; + + cxgb3i_tx_debug("cn 0x%p.\n", c3cn); + if (conn) { + cxgb3i_tx_debug("cn 0x%p, cid %d.\n", c3cn, conn->id); + scsi_queue_work(conn->session->host, &conn->xmitwork); + } +} + +void cxgb3i_conn_closing(struct s3_conn *c3cn) +{ + struct iscsi_conn *conn; + + read_lock(&c3cn->callback_lock); + conn = c3cn->user_data; + if (conn && c3cn->state != C3CN_STATE_ESTABLISHED) + iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); + read_unlock(&c3cn->callback_lock); +} -- cgit v1.2.3