diff options
Diffstat (limited to 'drivers/net/virtio_net.c')
-rw-r--r-- | drivers/net/virtio_net.c | 246 |
1 files changed, 100 insertions, 146 deletions
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 5c498d2b043..b9e002fccbc 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -1,4 +1,4 @@ -/* A simple network driver using virtio. +/* A network driver using virtio. * * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation * @@ -22,7 +22,6 @@ #include <linux/ethtool.h> #include <linux/module.h> #include <linux/virtio.h> -#include <linux/virtio_ids.h> #include <linux/virtio_net.h> #include <linux/scatterlist.h> #include <linux/if_vlan.h> @@ -48,19 +47,9 @@ struct virtnet_info struct napi_struct napi; unsigned int status; - /* The skb we couldn't send because buffers were full. */ - struct sk_buff *last_xmit_skb; - - /* If we need to free in a timer, this is it. */ - struct timer_list xmit_free_timer; - /* Number of input buffers, and max we've ever had. */ unsigned int num, max; - /* For cleaning up after transmission. */ - struct tasklet_struct tasklet; - bool free_in_tasklet; - /* I like... big packets and I cannot lie! */ bool big_packets; @@ -78,9 +67,17 @@ struct virtnet_info struct page *pages; }; -static inline void *skb_vnet_hdr(struct sk_buff *skb) +struct skb_vnet_hdr { + union { + struct virtio_net_hdr hdr; + struct virtio_net_hdr_mrg_rxbuf mhdr; + }; + unsigned int num_sg; +}; + +static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb) { - return (struct virtio_net_hdr *)skb->cb; + return (struct skb_vnet_hdr *)skb->cb; } static void give_a_page(struct virtnet_info *vi, struct page *page) @@ -119,17 +116,13 @@ static void skb_xmit_done(struct virtqueue *svq) /* We were probably waiting for more output buffers. */ netif_wake_queue(vi->dev); - - /* Make sure we re-xmit last_xmit_skb: if there are no more packets - * queued, start_xmit won't be called. */ - tasklet_schedule(&vi->tasklet); } static void receive_skb(struct net_device *dev, struct sk_buff *skb, unsigned len) { struct virtnet_info *vi = netdev_priv(dev); - struct virtio_net_hdr *hdr = skb_vnet_hdr(skb); + struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb); int err; int i; @@ -140,7 +133,6 @@ static void receive_skb(struct net_device *dev, struct sk_buff *skb, } if (vi->mergeable_rx_bufs) { - struct virtio_net_hdr_mrg_rxbuf *mhdr = skb_vnet_hdr(skb); unsigned int copy; char *p = page_address(skb_shinfo(skb)->frags[0].page); @@ -148,8 +140,8 @@ static void receive_skb(struct net_device *dev, struct sk_buff *skb, len = PAGE_SIZE; len -= sizeof(struct virtio_net_hdr_mrg_rxbuf); - memcpy(hdr, p, sizeof(*mhdr)); - p += sizeof(*mhdr); + memcpy(&hdr->mhdr, p, sizeof(hdr->mhdr)); + p += sizeof(hdr->mhdr); copy = len; if (copy > skb_tailroom(skb)) @@ -164,13 +156,13 @@ static void receive_skb(struct net_device *dev, struct sk_buff *skb, skb_shinfo(skb)->nr_frags--; } else { skb_shinfo(skb)->frags[0].page_offset += - sizeof(*mhdr) + copy; + sizeof(hdr->mhdr) + copy; skb_shinfo(skb)->frags[0].size = len; skb->data_len += len; skb->len += len; } - while (--mhdr->num_buffers) { + while (--hdr->mhdr.num_buffers) { struct sk_buff *nskb; i = skb_shinfo(skb)->nr_frags; @@ -184,7 +176,7 @@ static void receive_skb(struct net_device *dev, struct sk_buff *skb, nskb = vi->rvq->vq_ops->get_buf(vi->rvq, &len); if (!nskb) { pr_debug("%s: rx error: %d buffers missing\n", - dev->name, mhdr->num_buffers); + dev->name, hdr->mhdr.num_buffers); dev->stats.rx_length_errors++; goto drop; } @@ -205,7 +197,7 @@ static void receive_skb(struct net_device *dev, struct sk_buff *skb, skb->len += len; } } else { - len -= sizeof(struct virtio_net_hdr); + len -= sizeof(hdr->hdr); if (len <= MAX_PACKET_LEN) trim_pages(vi, skb); @@ -223,9 +215,11 @@ static void receive_skb(struct net_device *dev, struct sk_buff *skb, dev->stats.rx_bytes += skb->len; dev->stats.rx_packets++; - if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { + if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { pr_debug("Needs csum!\n"); - if (!skb_partial_csum_set(skb,hdr->csum_start,hdr->csum_offset)) + if (!skb_partial_csum_set(skb, + hdr->hdr.csum_start, + hdr->hdr.csum_offset)) goto frame_err; } @@ -233,9 +227,9 @@ static void receive_skb(struct net_device *dev, struct sk_buff *skb, pr_debug("Receiving skb proto 0x%04x len %i type %i\n", ntohs(skb->protocol), skb->len, skb->pkt_type); - if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { + if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) { pr_debug("GSO!\n"); - switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { + switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { case VIRTIO_NET_HDR_GSO_TCPV4: skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; break; @@ -248,14 +242,14 @@ static void receive_skb(struct net_device *dev, struct sk_buff *skb, default: if (net_ratelimit()) printk(KERN_WARNING "%s: bad gso type %u.\n", - dev->name, hdr->gso_type); + dev->name, hdr->hdr.gso_type); goto frame_err; } - if (hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) + if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN) skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; - skb_shinfo(skb)->gso_size = hdr->gso_size; + skb_shinfo(skb)->gso_size = hdr->hdr.gso_size; if (skb_shinfo(skb)->gso_size == 0) { if (net_ratelimit()) printk(KERN_WARNING "%s: zero gso size.\n", @@ -285,8 +279,8 @@ static bool try_fill_recv_maxbufs(struct virtnet_info *vi, gfp_t gfp) bool oom = false; sg_init_table(sg, 2+MAX_SKB_FRAGS); - for (;;) { - struct virtio_net_hdr *hdr; + do { + struct skb_vnet_hdr *hdr; skb = netdev_alloc_skb(vi->dev, MAX_PACKET_LEN + NET_IP_ALIGN); if (unlikely(!skb)) { @@ -298,7 +292,7 @@ static bool try_fill_recv_maxbufs(struct virtnet_info *vi, gfp_t gfp) skb_put(skb, MAX_PACKET_LEN); hdr = skb_vnet_hdr(skb); - sg_set_buf(sg, hdr, sizeof(*hdr)); + sg_set_buf(sg, &hdr->hdr, sizeof(hdr->hdr)); if (vi->big_packets) { for (i = 0; i < MAX_SKB_FRAGS; i++) { @@ -328,7 +322,7 @@ static bool try_fill_recv_maxbufs(struct virtnet_info *vi, gfp_t gfp) break; } vi->num++; - } + } while (err >= num); if (unlikely(vi->num > vi->max)) vi->max = vi->num; vi->rvq->vq_ops->kick(vi->rvq); @@ -346,7 +340,7 @@ static bool try_fill_recv(struct virtnet_info *vi, gfp_t gfp) if (!vi->mergeable_rx_bufs) return try_fill_recv_maxbufs(vi, gfp); - for (;;) { + do { skb_frag_t *f; skb = netdev_alloc_skb(vi->dev, GOOD_COPY_LEN + NET_IP_ALIGN); @@ -380,7 +374,7 @@ static bool try_fill_recv(struct virtnet_info *vi, gfp_t gfp) break; } vi->num++; - } + } while (err > 0); if (unlikely(vi->num > vi->max)) vi->max = vi->num; vi->rvq->vq_ops->kick(vi->rvq); @@ -448,42 +442,26 @@ again: return received; } -static void free_old_xmit_skbs(struct virtnet_info *vi) +static unsigned int free_old_xmit_skbs(struct virtnet_info *vi) { struct sk_buff *skb; - unsigned int len; + unsigned int len, tot_sgs = 0; while ((skb = vi->svq->vq_ops->get_buf(vi->svq, &len)) != NULL) { pr_debug("Sent skb %p\n", skb); __skb_unlink(skb, &vi->send); vi->dev->stats.tx_bytes += skb->len; vi->dev->stats.tx_packets++; - kfree_skb(skb); + tot_sgs += skb_vnet_hdr(skb)->num_sg; + dev_kfree_skb_any(skb); } -} - -/* If the virtio transport doesn't always notify us when all in-flight packets - * are consumed, we fall back to using this function on a timer to free them. */ -static void xmit_free(unsigned long data) -{ - struct virtnet_info *vi = (void *)data; - - netif_tx_lock(vi->dev); - - free_old_xmit_skbs(vi); - - if (!skb_queue_empty(&vi->send)) - mod_timer(&vi->xmit_free_timer, jiffies + (HZ/10)); - - netif_tx_unlock(vi->dev); + return tot_sgs; } static int xmit_skb(struct virtnet_info *vi, struct sk_buff *skb) { - int num, err; struct scatterlist sg[2+MAX_SKB_FRAGS]; - struct virtio_net_hdr_mrg_rxbuf *mhdr = skb_vnet_hdr(skb); - struct virtio_net_hdr *hdr = skb_vnet_hdr(skb); + struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb); const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest; sg_init_table(sg, 2+MAX_SKB_FRAGS); @@ -491,108 +469,97 @@ static int xmit_skb(struct virtnet_info *vi, struct sk_buff *skb) pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); if (skb->ip_summed == CHECKSUM_PARTIAL) { - hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; - hdr->csum_start = skb->csum_start - skb_headroom(skb); - hdr->csum_offset = skb->csum_offset; + hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; + hdr->hdr.csum_start = skb->csum_start - skb_headroom(skb); + hdr->hdr.csum_offset = skb->csum_offset; } else { - hdr->flags = 0; - hdr->csum_offset = hdr->csum_start = 0; + hdr->hdr.flags = 0; + hdr->hdr.csum_offset = hdr->hdr.csum_start = 0; } if (skb_is_gso(skb)) { - hdr->hdr_len = skb_headlen(skb); - hdr->gso_size = skb_shinfo(skb)->gso_size; + hdr->hdr.hdr_len = skb_headlen(skb); + hdr->hdr.gso_size = skb_shinfo(skb)->gso_size; if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) - hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; + hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4; else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) - hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; + hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6; else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP) - hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP; + hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP; else BUG(); if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN) - hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; + hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN; } else { - hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE; - hdr->gso_size = hdr->hdr_len = 0; + hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE; + hdr->hdr.gso_size = hdr->hdr.hdr_len = 0; } - mhdr->num_buffers = 0; + hdr->mhdr.num_buffers = 0; /* Encode metadata header at front. */ if (vi->mergeable_rx_bufs) - sg_set_buf(sg, mhdr, sizeof(*mhdr)); + sg_set_buf(sg, &hdr->mhdr, sizeof(hdr->mhdr)); else - sg_set_buf(sg, hdr, sizeof(*hdr)); - - num = skb_to_sgvec(skb, sg+1, 0, skb->len) + 1; - - err = vi->svq->vq_ops->add_buf(vi->svq, sg, num, 0, skb); - if (err >= 0 && !vi->free_in_tasklet) - mod_timer(&vi->xmit_free_timer, jiffies + (HZ/10)); + sg_set_buf(sg, &hdr->hdr, sizeof(hdr->hdr)); - return err; -} - -static void xmit_tasklet(unsigned long data) -{ - struct virtnet_info *vi = (void *)data; - - netif_tx_lock_bh(vi->dev); - if (vi->last_xmit_skb && xmit_skb(vi, vi->last_xmit_skb) >= 0) { - vi->svq->vq_ops->kick(vi->svq); - vi->last_xmit_skb = NULL; - } - if (vi->free_in_tasklet) - free_old_xmit_skbs(vi); - netif_tx_unlock_bh(vi->dev); + hdr->num_sg = skb_to_sgvec(skb, sg+1, 0, skb->len) + 1; + return vi->svq->vq_ops->add_buf(vi->svq, sg, hdr->num_sg, 0, skb); } static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) { struct virtnet_info *vi = netdev_priv(dev); + int capacity; again: /* Free up any pending old buffers before queueing new ones. */ free_old_xmit_skbs(vi); - /* If we has a buffer left over from last time, send it now. */ - if (unlikely(vi->last_xmit_skb) && - xmit_skb(vi, vi->last_xmit_skb) < 0) - goto stop_queue; - - vi->last_xmit_skb = NULL; + /* Try to transmit */ + capacity = xmit_skb(vi, skb); - /* Put new one in send queue and do transmit */ - if (likely(skb)) { - __skb_queue_head(&vi->send, skb); - if (xmit_skb(vi, skb) < 0) { - vi->last_xmit_skb = skb; - skb = NULL; - goto stop_queue; + /* This can happen with OOM and indirect buffers. */ + if (unlikely(capacity < 0)) { + netif_stop_queue(dev); + dev_warn(&dev->dev, "Unexpected full queue\n"); + if (unlikely(!vi->svq->vq_ops->enable_cb(vi->svq))) { + vi->svq->vq_ops->disable_cb(vi->svq); + netif_start_queue(dev); + goto again; } + return NETDEV_TX_BUSY; } -done: vi->svq->vq_ops->kick(vi->svq); - return NETDEV_TX_OK; - -stop_queue: - pr_debug("%s: virtio not prepared to send\n", dev->name); - netif_stop_queue(dev); - /* Activate callback for using skbs: if this returns false it - * means some were used in the meantime. */ - if (unlikely(!vi->svq->vq_ops->enable_cb(vi->svq))) { - vi->svq->vq_ops->disable_cb(vi->svq); - netif_start_queue(dev); - goto again; - } - if (skb) { - /* Drop this skb: we only queue one. */ - vi->dev->stats.tx_dropped++; - kfree_skb(skb); + /* + * Put new one in send queue. You'd expect we'd need this before + * xmit_skb calls add_buf(), since the callback can be triggered + * immediately after that. But since the callback just triggers + * another call back here, normal network xmit locking prevents the + * race. + */ + __skb_queue_head(&vi->send, skb); + + /* Don't wait up for transmitted skbs to be freed. */ + skb_orphan(skb); + nf_reset(skb); + + /* Apparently nice girls don't return TX_BUSY; stop the queue + * before it gets out of hand. Naturally, this wastes entries. */ + if (capacity < 2+MAX_SKB_FRAGS) { + netif_stop_queue(dev); + if (unlikely(!vi->svq->vq_ops->enable_cb(vi->svq))) { + /* More just got used, free them then recheck. */ + capacity += free_old_xmit_skbs(vi); + if (capacity >= 2+MAX_SKB_FRAGS) { + netif_start_queue(dev); + vi->svq->vq_ops->disable_cb(vi->svq); + } + } } - goto done; + + return NETDEV_TX_OK; } static int virtnet_set_mac_address(struct net_device *dev, void *p) @@ -925,10 +892,6 @@ static int virtnet_probe(struct virtio_device *vdev) vi->pages = NULL; INIT_DELAYED_WORK(&vi->refill, refill_work); - /* If they give us a callback when all buffers are done, we don't need - * the timer. */ - vi->free_in_tasklet = virtio_has_feature(vdev,VIRTIO_F_NOTIFY_ON_EMPTY); - /* If we can receive ANY GSO packets, we must allocate large ones. */ if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) @@ -960,11 +923,6 @@ static int virtnet_probe(struct virtio_device *vdev) skb_queue_head_init(&vi->recv); skb_queue_head_init(&vi->send); - tasklet_init(&vi->tasklet, xmit_tasklet, (unsigned long)vi); - - if (!vi->free_in_tasklet) - setup_timer(&vi->xmit_free_timer, xmit_free, (unsigned long)vi); - err = register_netdev(dev); if (err) { pr_debug("virtio_net: registering device failed\n"); @@ -997,7 +955,7 @@ free: return err; } -static void virtnet_remove(struct virtio_device *vdev) +static void __devexit virtnet_remove(struct virtio_device *vdev) { struct virtnet_info *vi = vdev->priv; struct sk_buff *skb; @@ -1005,9 +963,6 @@ static void virtnet_remove(struct virtio_device *vdev) /* Stop all the virtqueues. */ vdev->config->reset(vdev); - if (!vi->free_in_tasklet) - del_timer_sync(&vi->xmit_free_timer); - /* Free our skbs in send and recv queues, if any. */ while ((skb = __skb_dequeue(&vi->recv)) != NULL) { kfree_skb(skb); @@ -1041,10 +996,9 @@ static unsigned int features[] = { VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, - VIRTIO_F_NOTIFY_ON_EMPTY, }; -static struct virtio_driver virtio_net = { +static struct virtio_driver virtio_net_driver = { .feature_table = features, .feature_table_size = ARRAY_SIZE(features), .driver.name = KBUILD_MODNAME, @@ -1057,12 +1011,12 @@ static struct virtio_driver virtio_net = { static int __init init(void) { - return register_virtio_driver(&virtio_net); + return register_virtio_driver(&virtio_net_driver); } static void __exit fini(void) { - unregister_virtio_driver(&virtio_net); + unregister_virtio_driver(&virtio_net_driver); } module_init(init); module_exit(fini); |