blob: 5b450c6100add5686796fd4e00c805f41f5ffd2f [file] [log] [blame]
// SPDX-License-Identifier: (GPL-2.0 OR MIT)
/* Google virtual Ethernet (gve) driver
*
* Copyright (C) 2015-2019 Google, Inc.
*/
#include <linux/cpumask.h>
#include <linux/etherdevice.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/sched.h>
#include <linux/timer.h>
#include <linux/workqueue.h>
#include <net/sch_generic.h>
#include "gve.h"
#include "gve_adminq.h"
#include "gve_register.h"
#define GVE_DEFAULT_RX_COPYBREAK (256)
#define DEFAULT_MSG_LEVEL (NETIF_MSG_DRV | NETIF_MSG_LINK)
#define GVE_VERSION "1.0.0"
#define GVE_VERSION_PREFIX "GVE-"
const char gve_version_str[] = GVE_VERSION;
static const char gve_version_prefix[] = GVE_VERSION_PREFIX;
static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s)
{
struct gve_priv *priv = netdev_priv(dev);
unsigned int start;
u64 packets, bytes;
int ring;
if (priv->rx) {
for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) {
do {
start =
u64_stats_fetch_begin(&priv->rx[ring].statss);
packets = priv->rx[ring].rpackets;
bytes = priv->rx[ring].rbytes;
} while (u64_stats_fetch_retry(&priv->rx[ring].statss,
start));
s->rx_packets += packets;
s->rx_bytes += bytes;
}
}
if (priv->tx) {
for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) {
do {
start =
u64_stats_fetch_begin(&priv->tx[ring].statss);
packets = priv->tx[ring].pkt_done;
bytes = priv->tx[ring].bytes_done;
} while (u64_stats_fetch_retry(&priv->tx[ring].statss,
start));
s->tx_packets += packets;
s->tx_bytes += bytes;
}
}
}
static int gve_alloc_counter_array(struct gve_priv *priv)
{
priv->counter_array =
dma_alloc_coherent(&priv->pdev->dev,
priv->num_event_counters *
sizeof(*priv->counter_array),
&priv->counter_array_bus, GFP_KERNEL);
if (!priv->counter_array)
return -ENOMEM;
return 0;
}
static void gve_free_counter_array(struct gve_priv *priv)
{
dma_free_coherent(&priv->pdev->dev,
priv->num_event_counters *
sizeof(*priv->counter_array),
priv->counter_array, priv->counter_array_bus);
priv->counter_array = NULL;
}
static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
{
struct gve_priv *priv = arg;
queue_work(priv->gve_wq, &priv->service_task);
return IRQ_HANDLED;
}
static irqreturn_t gve_intr(int irq, void *arg)
{
struct gve_notify_block *block = arg;
struct gve_priv *priv = block->priv;
iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block));
napi_schedule_irqoff(&block->napi);
return IRQ_HANDLED;
}
static int gve_napi_poll(struct napi_struct *napi, int budget)
{
struct gve_notify_block *block;
__be32 __iomem *irq_doorbell;
bool reschedule = false;
struct gve_priv *priv;
block = container_of(napi, struct gve_notify_block, napi);
priv = block->priv;
if (block->tx)
reschedule |= gve_tx_poll(block, budget);
if (block->rx)
reschedule |= gve_rx_poll(block, budget);
if (reschedule)
return budget;
napi_complete(napi);
irq_doorbell = gve_irq_doorbell(priv, block);
iowrite32be(GVE_IRQ_ACK | GVE_IRQ_EVENT, irq_doorbell);
/* Double check we have no extra work.
* Ensure unmask synchronizes with checking for work.
*/
mb();
if (block->tx)
reschedule |= gve_tx_poll(block, -1);
if (block->rx)
reschedule |= gve_rx_poll(block, -1);
if (reschedule && napi_reschedule(napi))
iowrite32be(GVE_IRQ_MASK, irq_doorbell);
return 0;
}
static int gve_alloc_notify_blocks(struct gve_priv *priv)
{
int num_vecs_requested = priv->num_ntfy_blks + 1;
char *name = priv->dev->name;
unsigned int active_cpus;
int vecs_enabled;
int i, j;
int err;
priv->msix_vectors = kvzalloc(num_vecs_requested *
sizeof(*priv->msix_vectors), GFP_KERNEL);
if (!priv->msix_vectors)
return -ENOMEM;
for (i = 0; i < num_vecs_requested; i++)
priv->msix_vectors[i].entry = i;
vecs_enabled = pci_enable_msix_range(priv->pdev, priv->msix_vectors,
GVE_MIN_MSIX, num_vecs_requested);
if (vecs_enabled < 0) {
dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n",
GVE_MIN_MSIX, vecs_enabled);
err = vecs_enabled;
goto abort_with_msix_vectors;
}
if (vecs_enabled != num_vecs_requested) {
int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1;
int vecs_per_type = new_num_ntfy_blks / 2;
int vecs_left = new_num_ntfy_blks % 2;
priv->num_ntfy_blks = new_num_ntfy_blks;
priv->mgmt_msix_idx = priv->num_ntfy_blks;
priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues,
vecs_per_type);
priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues,
vecs_per_type + vecs_left);
dev_err(&priv->pdev->dev,
"Could not enable desired msix, only enabled %d, adjusting tx max queues to %d, and rx max queues to %d\n",
vecs_enabled, priv->tx_cfg.max_queues,
priv->rx_cfg.max_queues);
if (priv->tx_cfg.num_queues > priv->tx_cfg.max_queues)
priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
if (priv->rx_cfg.num_queues > priv->rx_cfg.max_queues)
priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
}
/* Half the notification blocks go to TX and half to RX */
active_cpus = min_t(int, priv->num_ntfy_blks / 2, num_online_cpus());
/* Setup Management Vector - the last vector */
snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "%s-mgmnt",
name);
err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
if (err) {
dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
goto abort_with_msix_enabled;
}
priv->ntfy_blocks =
dma_alloc_coherent(&priv->pdev->dev,
priv->num_ntfy_blks *
sizeof(*priv->ntfy_blocks),
&priv->ntfy_block_bus, GFP_KERNEL);
if (!priv->ntfy_blocks) {
err = -ENOMEM;
goto abort_with_mgmt_vector;
}
/* Setup the other blocks - the first n-1 vectors */
for (i = 0; i < priv->num_ntfy_blks; i++) {
struct gve_notify_block *block = &priv->ntfy_blocks[i];
int msix_idx = i;
snprintf(block->name, sizeof(block->name), "%s-ntfy-block.%d",
name, i);
block->priv = priv;
err = request_irq(priv->msix_vectors[msix_idx].vector,
gve_intr, 0, block->name, block);
if (err) {
dev_err(&priv->pdev->dev,
"Failed to receive msix vector %d\n", i);
goto abort_with_some_ntfy_blocks;
}
irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
get_cpu_mask(i % active_cpus));
}
return 0;
abort_with_some_ntfy_blocks:
for (j = 0; j < i; j++) {
struct gve_notify_block *block = &priv->ntfy_blocks[j];
int msix_idx = j;
irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
NULL);
free_irq(priv->msix_vectors[msix_idx].vector, block);
}
dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
sizeof(*priv->ntfy_blocks),
priv->ntfy_blocks, priv->ntfy_block_bus);
priv->ntfy_blocks = NULL;
abort_with_mgmt_vector:
free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
abort_with_msix_enabled:
pci_disable_msix(priv->pdev);
abort_with_msix_vectors:
kvfree(priv->msix_vectors);
priv->msix_vectors = NULL;
return err;
}
static void gve_free_notify_blocks(struct gve_priv *priv)
{
int i;
if (priv->msix_vectors) {
/* Free the irqs */
for (i = 0; i < priv->num_ntfy_blks; i++) {
struct gve_notify_block *block = &priv->ntfy_blocks[i];
int msix_idx = i;
irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
NULL);
free_irq(priv->msix_vectors[msix_idx].vector, block);
}
free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
}
dma_free_coherent(&priv->pdev->dev,
priv->num_ntfy_blks * sizeof(*priv->ntfy_blocks),
priv->ntfy_blocks, priv->ntfy_block_bus);
priv->ntfy_blocks = NULL;
pci_disable_msix(priv->pdev);
kvfree(priv->msix_vectors);
priv->msix_vectors = NULL;
}
static int gve_setup_device_resources(struct gve_priv *priv)
{
int err;
err = gve_alloc_counter_array(priv);
if (err)
return err;
err = gve_alloc_notify_blocks(priv);
if (err)
goto abort_with_counter;
err = gve_adminq_configure_device_resources(priv,
priv->counter_array_bus,
priv->num_event_counters,
priv->ntfy_block_bus,
priv->num_ntfy_blks);
if (unlikely(err)) {
dev_err(&priv->pdev->dev,
"could not setup device_resources: err=%d\n", err);
err = -ENXIO;
goto abort_with_ntfy_blocks;
}
gve_set_device_resources_ok(priv);
return 0;
abort_with_ntfy_blocks:
gve_free_notify_blocks(priv);
abort_with_counter:
gve_free_counter_array(priv);
return err;
}
static void gve_trigger_reset(struct gve_priv *priv);
static void gve_teardown_device_resources(struct gve_priv *priv)
{
int err;
/* Tell device its resources are being freed */
if (gve_get_device_resources_ok(priv)) {
err = gve_adminq_deconfigure_device_resources(priv);
if (err) {
dev_err(&priv->pdev->dev,
"Could not deconfigure device resources: err=%d\n",
err);
gve_trigger_reset(priv);
}
}
gve_free_counter_array(priv);
gve_free_notify_blocks(priv);
gve_clear_device_resources_ok(priv);
}
static void gve_add_napi(struct gve_priv *priv, int ntfy_idx)
{
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
netif_napi_add(priv->dev, &block->napi, gve_napi_poll,
NAPI_POLL_WEIGHT);
}
static void gve_remove_napi(struct gve_priv *priv, int ntfy_idx)
{
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
netif_napi_del(&block->napi);
}
static int gve_register_qpls(struct gve_priv *priv)
{
int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
int err;
int i;
for (i = 0; i < num_qpls; i++) {
err = gve_adminq_register_page_list(priv, &priv->qpls[i]);
if (err) {
netif_err(priv, drv, priv->dev,
"failed to register queue page list %d\n",
priv->qpls[i].id);
/* This failure will trigger a reset - no need to clean
* up
*/
return err;
}
}
return 0;
}
static int gve_unregister_qpls(struct gve_priv *priv)
{
int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
int err;
int i;
for (i = 0; i < num_qpls; i++) {
err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id);
/* This failure will trigger a reset - no need to clean up */
if (err) {
netif_err(priv, drv, priv->dev,
"Failed to unregister queue page list %d\n",
priv->qpls[i].id);
return err;
}
}
return 0;
}
static int gve_create_rings(struct gve_priv *priv)
{
int err;
int i;
for (i = 0; i < priv->tx_cfg.num_queues; i++) {
err = gve_adminq_create_tx_queue(priv, i);
if (err) {
netif_err(priv, drv, priv->dev, "failed to create tx queue %d\n",
i);
/* This failure will trigger a reset - no need to clean
* up
*/
return err;
}
netif_dbg(priv, drv, priv->dev, "created tx queue %d\n", i);
}
for (i = 0; i < priv->rx_cfg.num_queues; i++) {
err = gve_adminq_create_rx_queue(priv, i);
if (err) {
netif_err(priv, drv, priv->dev, "failed to create rx queue %d\n",
i);
/* This failure will trigger a reset - no need to clean
* up
*/
return err;
}
/* Rx data ring has been prefilled with packet buffers at
* queue allocation time.
* Write the doorbell to provide descriptor slots and packet
* buffers to the NIC.
*/
gve_rx_write_doorbell(priv, &priv->rx[i]);
netif_dbg(priv, drv, priv->dev, "created rx queue %d\n", i);
}
return 0;
}
static int gve_alloc_rings(struct gve_priv *priv)
{
int ntfy_idx;
int err;
int i;
/* Setup tx rings */
priv->tx = kvzalloc(priv->tx_cfg.num_queues * sizeof(*priv->tx),
GFP_KERNEL);
if (!priv->tx)
return -ENOMEM;
err = gve_tx_alloc_rings(priv);
if (err)
goto free_tx;
/* Setup rx rings */
priv->rx = kvzalloc(priv->rx_cfg.num_queues * sizeof(*priv->rx),
GFP_KERNEL);
if (!priv->rx) {
err = -ENOMEM;
goto free_tx_queue;
}
err = gve_rx_alloc_rings(priv);
if (err)
goto free_rx;
/* Add tx napi & init sync stats*/
for (i = 0; i < priv->tx_cfg.num_queues; i++) {
u64_stats_init(&priv->tx[i].statss);
ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
gve_add_napi(priv, ntfy_idx);
}
/* Add rx napi & init sync stats*/
for (i = 0; i < priv->rx_cfg.num_queues; i++) {
u64_stats_init(&priv->rx[i].statss);
ntfy_idx = gve_rx_idx_to_ntfy(priv, i);
gve_add_napi(priv, ntfy_idx);
}
return 0;
free_rx:
kvfree(priv->rx);
priv->rx = NULL;
free_tx_queue:
gve_tx_free_rings(priv);
free_tx:
kvfree(priv->tx);
priv->tx = NULL;
return err;
}
static int gve_destroy_rings(struct gve_priv *priv)
{
int err;
int i;
for (i = 0; i < priv->tx_cfg.num_queues; i++) {
err = gve_adminq_destroy_tx_queue(priv, i);
if (err) {
netif_err(priv, drv, priv->dev,
"failed to destroy tx queue %d\n",
i);
/* This failure will trigger a reset - no need to clean
* up
*/
return err;
}
netif_dbg(priv, drv, priv->dev, "destroyed tx queue %d\n", i);
}
for (i = 0; i < priv->rx_cfg.num_queues; i++) {
err = gve_adminq_destroy_rx_queue(priv, i);
if (err) {
netif_err(priv, drv, priv->dev,
"failed to destroy rx queue %d\n",
i);
/* This failure will trigger a reset - no need to clean
* up
*/
return err;
}
netif_dbg(priv, drv, priv->dev, "destroyed rx queue %d\n", i);
}
return 0;
}
static void gve_free_rings(struct gve_priv *priv)
{
int ntfy_idx;
int i;
if (priv->tx) {
for (i = 0; i < priv->tx_cfg.num_queues; i++) {
ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
gve_remove_napi(priv, ntfy_idx);
}
gve_tx_free_rings(priv);
kvfree(priv->tx);
priv->tx = NULL;
}
if (priv->rx) {
for (i = 0; i < priv->rx_cfg.num_queues; i++) {
ntfy_idx = gve_rx_idx_to_ntfy(priv, i);
gve_remove_napi(priv, ntfy_idx);
}
gve_rx_free_rings(priv);
kvfree(priv->rx);
priv->rx = NULL;
}
}
int gve_alloc_page(struct device *dev, struct page **page, dma_addr_t *dma,
enum dma_data_direction dir)
{
*page = alloc_page(GFP_KERNEL);
if (!*page)
return -ENOMEM;
*dma = dma_map_page(dev, *page, 0, PAGE_SIZE, dir);
if (dma_mapping_error(dev, *dma)) {
put_page(*page);
return -ENOMEM;
}
return 0;
}
static int gve_alloc_queue_page_list(struct gve_priv *priv, u32 id,
int pages)
{
struct gve_queue_page_list *qpl = &priv->qpls[id];
int err;
int i;
if (pages + priv->num_registered_pages > priv->max_registered_pages) {
netif_err(priv, drv, priv->dev,
"Reached max number of registered pages %llu > %llu\n",
pages + priv->num_registered_pages,
priv->max_registered_pages);
return -EINVAL;
}
qpl->id = id;
qpl->num_entries = 0;
qpl->pages = kvzalloc(pages * sizeof(*qpl->pages), GFP_KERNEL);
/* caller handles clean up */
if (!qpl->pages)
return -ENOMEM;
qpl->page_buses = kvzalloc(pages * sizeof(*qpl->page_buses),
GFP_KERNEL);
/* caller handles clean up */
if (!qpl->page_buses)
return -ENOMEM;
for (i = 0; i < pages; i++) {
err = gve_alloc_page(&priv->pdev->dev, &qpl->pages[i],
&qpl->page_buses[i],
gve_qpl_dma_dir(priv, id));
/* caller handles clean up */
if (err)
return -ENOMEM;
qpl->num_entries++;
}
priv->num_registered_pages += pages;
return 0;
}
void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma,
enum dma_data_direction dir)
{
if (!dma_mapping_error(dev, dma))
dma_unmap_page(dev, dma, PAGE_SIZE, dir);
if (page)
put_page(page);
}
static void gve_free_queue_page_list(struct gve_priv *priv,
int id)
{
struct gve_queue_page_list *qpl = &priv->qpls[id];
int i;
if (!qpl->pages)
return;
if (!qpl->page_buses)
goto free_pages;
for (i = 0; i < qpl->num_entries; i++)
gve_free_page(&priv->pdev->dev, qpl->pages[i],
qpl->page_buses[i], gve_qpl_dma_dir(priv, id));
kvfree(qpl->page_buses);
free_pages:
kvfree(qpl->pages);
priv->num_registered_pages -= qpl->num_entries;
}
static int gve_alloc_qpls(struct gve_priv *priv)
{
int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
int i, j;
int err;
priv->qpls = kvzalloc(num_qpls * sizeof(*priv->qpls), GFP_KERNEL);
if (!priv->qpls)
return -ENOMEM;
for (i = 0; i < gve_num_tx_qpls(priv); i++) {
err = gve_alloc_queue_page_list(priv, i,
priv->tx_pages_per_qpl);
if (err)
goto free_qpls;
}
for (; i < num_qpls; i++) {
err = gve_alloc_queue_page_list(priv, i,
priv->rx_pages_per_qpl);
if (err)
goto free_qpls;
}
priv->qpl_cfg.qpl_map_size = BITS_TO_LONGS(num_qpls) *
sizeof(unsigned long) * BITS_PER_BYTE;
priv->qpl_cfg.qpl_id_map = kvzalloc(BITS_TO_LONGS(num_qpls) *
sizeof(unsigned long), GFP_KERNEL);
if (!priv->qpl_cfg.qpl_id_map) {
err = -ENOMEM;
goto free_qpls;
}
return 0;
free_qpls:
for (j = 0; j <= i; j++)
gve_free_queue_page_list(priv, j);
kvfree(priv->qpls);
return err;
}
static void gve_free_qpls(struct gve_priv *priv)
{
int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
int i;
kvfree(priv->qpl_cfg.qpl_id_map);
for (i = 0; i < num_qpls; i++)
gve_free_queue_page_list(priv, i);
kvfree(priv->qpls);
}
/* Use this to schedule a reset when the device is capable of continuing
* to handle other requests in its current state. If it is not, do a reset
* in thread instead.
*/
void gve_schedule_reset(struct gve_priv *priv)
{
gve_set_do_reset(priv);
queue_work(priv->gve_wq, &priv->service_task);
}
static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up);
static int gve_reset_recovery(struct gve_priv *priv, bool was_up);
static void gve_turndown(struct gve_priv *priv);
static void gve_turnup(struct gve_priv *priv);
static int gve_open(struct net_device *dev)
{
struct gve_priv *priv = netdev_priv(dev);
int err;
err = gve_alloc_qpls(priv);
if (err)
return err;
err = gve_alloc_rings(priv);
if (err)
goto free_qpls;
err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues);
if (err)
goto free_rings;
err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues);
if (err)
goto free_rings;
err = gve_register_qpls(priv);
if (err)
goto reset;
err = gve_create_rings(priv);
if (err)
goto reset;
gve_set_device_rings_ok(priv);
gve_turnup(priv);
netif_carrier_on(dev);
return 0;
free_rings:
gve_free_rings(priv);
free_qpls:
gve_free_qpls(priv);
return err;
reset:
/* This must have been called from a reset due to the rtnl lock
* so just return at this point.
*/
if (gve_get_reset_in_progress(priv))
return err;
/* Otherwise reset before returning */
gve_reset_and_teardown(priv, true);
/* if this fails there is nothing we can do so just ignore the return */
gve_reset_recovery(priv, false);
/* return the original error */
return err;
}
static int gve_close(struct net_device *dev)
{
struct gve_priv *priv = netdev_priv(dev);
int err;
netif_carrier_off(dev);
if (gve_get_device_rings_ok(priv)) {
gve_turndown(priv);
err = gve_destroy_rings(priv);
if (err)
goto err;
err = gve_unregister_qpls(priv);
if (err)
goto err;
gve_clear_device_rings_ok(priv);
}
gve_free_rings(priv);
gve_free_qpls(priv);
return 0;
err:
/* This must have been called from a reset due to the rtnl lock
* so just return at this point.
*/
if (gve_get_reset_in_progress(priv))
return err;
/* Otherwise reset before returning */
gve_reset_and_teardown(priv, true);
return gve_reset_recovery(priv, false);
}
int gve_adjust_queues(struct gve_priv *priv,
struct gve_queue_config new_rx_config,
struct gve_queue_config new_tx_config)
{
int err;
if (netif_carrier_ok(priv->dev)) {
/* To make this process as simple as possible we teardown the
* device, set the new configuration, and then bring the device
* up again.
*/
err = gve_close(priv->dev);
/* we have already tried to reset in close,
* just fail at this point
*/
if (err)
return err;
priv->tx_cfg = new_tx_config;
priv->rx_cfg = new_rx_config;
err = gve_open(priv->dev);
if (err)
goto err;
return 0;
}
/* Set the config for the next up. */
priv->tx_cfg = new_tx_config;
priv->rx_cfg = new_rx_config;
return 0;
err:
netif_err(priv, drv, priv->dev,
"Adjust queues failed! !!! DISABLING ALL QUEUES !!!\n");
gve_turndown(priv);
return err;
}
static void gve_turndown(struct gve_priv *priv)
{
int idx;
if (netif_carrier_ok(priv->dev))
netif_carrier_off(priv->dev);
if (!gve_get_napi_enabled(priv))
return;
/* Disable napi to prevent more work from coming in */
for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
napi_disable(&block->napi);
}
for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
napi_disable(&block->napi);
}
/* Stop tx queues */
netif_tx_disable(priv->dev);
gve_clear_napi_enabled(priv);
}
static void gve_turnup(struct gve_priv *priv)
{
int idx;
/* Start the tx queues */
netif_tx_start_all_queues(priv->dev);
/* Enable napi and unmask interrupts for all queues */
for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
napi_enable(&block->napi);
iowrite32be(0, gve_irq_doorbell(priv, block));
}
for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
napi_enable(&block->napi);
iowrite32be(0, gve_irq_doorbell(priv, block));
}
gve_set_napi_enabled(priv);
}
static void gve_tx_timeout(struct net_device *dev)
{
struct gve_priv *priv = netdev_priv(dev);
gve_schedule_reset(priv);
priv->tx_timeo_cnt++;
}
static const struct net_device_ops gve_netdev_ops = {
.ndo_start_xmit = gve_tx,
.ndo_open = gve_open,
.ndo_stop = gve_close,
.ndo_get_stats64 = gve_get_stats,
.ndo_tx_timeout = gve_tx_timeout,
};
static void gve_handle_status(struct gve_priv *priv, u32 status)
{
if (GVE_DEVICE_STATUS_RESET_MASK & status) {
dev_info(&priv->pdev->dev, "Device requested reset.\n");
gve_set_do_reset(priv);
}
}
static void gve_handle_reset(struct gve_priv *priv)
{
/* A service task will be scheduled at the end of probe to catch any
* resets that need to happen, and we don't want to reset until
* probe is done.
*/
if (gve_get_probe_in_progress(priv))
return;
if (gve_get_do_reset(priv)) {
rtnl_lock();
gve_reset(priv, false);
rtnl_unlock();
}
}
/* Handle NIC status register changes and reset requests */
static void gve_service_task(struct work_struct *work)
{
struct gve_priv *priv = container_of(work, struct gve_priv,
service_task);
gve_handle_status(priv,
ioread32be(&priv->reg_bar0->device_status));
gve_handle_reset(priv);
}
static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
{
int num_ntfy;
int err;
/* Set up the adminq */
err = gve_adminq_alloc(&priv->pdev->dev, priv);
if (err) {
dev_err(&priv->pdev->dev,
"Failed to alloc admin queue: err=%d\n", err);
return err;
}
if (skip_describe_device)
goto setup_device;
/* Get the initial information we need from the device */
err = gve_adminq_describe_device(priv);
if (err) {
dev_err(&priv->pdev->dev,
"Could not get device information: err=%d\n", err);
goto err;
}
if (priv->dev->max_mtu > PAGE_SIZE) {
priv->dev->max_mtu = PAGE_SIZE;
err = gve_adminq_set_mtu(priv, priv->dev->mtu);
if (err) {
netif_err(priv, drv, priv->dev, "Could not set mtu");
goto err;
}
}
priv->dev->mtu = priv->dev->max_mtu;
num_ntfy = pci_msix_vec_count(priv->pdev);
if (num_ntfy <= 0) {
dev_err(&priv->pdev->dev,
"could not count MSI-x vectors: err=%d\n", num_ntfy);
err = num_ntfy;
goto err;
} else if (num_ntfy < GVE_MIN_MSIX) {
dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n",
GVE_MIN_MSIX, num_ntfy);
err = -EINVAL;
goto err;
}
priv->num_registered_pages = 0;
priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
/* gvnic has one Notification Block per MSI-x vector, except for the
* management vector
*/
priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1;
priv->mgmt_msix_idx = priv->num_ntfy_blks;
priv->tx_cfg.max_queues =
min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
priv->rx_cfg.max_queues =
min_t(int, priv->rx_cfg.max_queues, priv->num_ntfy_blks / 2);
priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
if (priv->default_num_queues > 0) {
priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues,
priv->tx_cfg.num_queues);
priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues,
priv->rx_cfg.num_queues);
}
netif_info(priv, drv, priv->dev, "TX queues %d, RX queues %d\n",
priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
netif_info(priv, drv, priv->dev, "Max TX queues %d, Max RX queues %d\n",
priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
setup_device:
err = gve_setup_device_resources(priv);
if (!err)
return 0;
err:
gve_adminq_free(&priv->pdev->dev, priv);
return err;
}
static void gve_teardown_priv_resources(struct gve_priv *priv)
{
gve_teardown_device_resources(priv);
gve_adminq_free(&priv->pdev->dev, priv);
}
static void gve_trigger_reset(struct gve_priv *priv)
{
/* Reset the device by releasing the AQ */
gve_adminq_release(priv);
}
static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up)
{
gve_trigger_reset(priv);
/* With the reset having already happened, close cannot fail */
if (was_up)
gve_close(priv->dev);
gve_teardown_priv_resources(priv);
}
static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
{
int err;
err = gve_init_priv(priv, true);
if (err)
goto err;
if (was_up) {
err = gve_open(priv->dev);
if (err)
goto err;
}
return 0;
err:
dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n");
gve_turndown(priv);
return err;
}
int gve_reset(struct gve_priv *priv, bool attempt_teardown)
{
bool was_up = netif_carrier_ok(priv->dev);
int err;
dev_info(&priv->pdev->dev, "Performing reset\n");
gve_clear_do_reset(priv);
gve_set_reset_in_progress(priv);
/* If we aren't attempting to teardown normally, just go turndown and
* reset right away.
*/
if (!attempt_teardown) {
gve_turndown(priv);
gve_reset_and_teardown(priv, was_up);
} else {
/* Otherwise attempt to close normally */
if (was_up) {
err = gve_close(priv->dev);
/* If that fails reset as we did above */
if (err)
gve_reset_and_teardown(priv, was_up);
}
/* Clean up any remaining resources */
gve_teardown_priv_resources(priv);
}
/* Set it all back up */
err = gve_reset_recovery(priv, was_up);
gve_clear_reset_in_progress(priv);
return err;
}
static void gve_write_version(u8 __iomem *driver_version_register)
{
const char *c = gve_version_prefix;
while (*c) {
writeb(*c, driver_version_register);
c++;
}
c = gve_version_str;
while (*c) {
writeb(*c, driver_version_register);
c++;
}
writeb('\n', driver_version_register);
}
static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
int max_tx_queues, max_rx_queues;
struct net_device *dev;
__be32 __iomem *db_bar;
struct gve_registers __iomem *reg_bar;
struct gve_priv *priv;
int err;
err = pci_enable_device(pdev);
if (err)
return -ENXIO;
err = pci_request_regions(pdev, "gvnic-cfg");
if (err)
goto abort_with_enabled;
pci_set_master(pdev);
err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
if (err) {
dev_err(&pdev->dev, "Failed to set dma mask: err=%d\n", err);
goto abort_with_pci_region;
}
err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
if (err) {
dev_err(&pdev->dev,
"Failed to set consistent dma mask: err=%d\n", err);
goto abort_with_pci_region;
}
reg_bar = pci_iomap(pdev, GVE_REGISTER_BAR, 0);
if (!reg_bar) {
dev_err(&pdev->dev, "Failed to map pci bar!\n");
err = -ENOMEM;
goto abort_with_pci_region;
}
db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0);
if (!db_bar) {
dev_err(&pdev->dev, "Failed to map doorbell bar!\n");
err = -ENOMEM;
goto abort_with_reg_bar;
}
gve_write_version(&reg_bar->driver_version);
/* Get max queues to alloc etherdev */
max_tx_queues = ioread32be(&reg_bar->max_tx_queues);
max_rx_queues = ioread32be(&reg_bar->max_rx_queues);
/* Alloc and setup the netdev and priv */
dev = alloc_etherdev_mqs(sizeof(*priv), max_tx_queues, max_rx_queues);
if (!dev) {
dev_err(&pdev->dev, "could not allocate netdev\n");
goto abort_with_db_bar;
}
SET_NETDEV_DEV(dev, &pdev->dev);
pci_set_drvdata(pdev, dev);
dev->ethtool_ops = &gve_ethtool_ops;
dev->netdev_ops = &gve_netdev_ops;
/* advertise features */
dev->hw_features = NETIF_F_HIGHDMA;
dev->hw_features |= NETIF_F_SG;
dev->hw_features |= NETIF_F_HW_CSUM;
dev->hw_features |= NETIF_F_TSO;
dev->hw_features |= NETIF_F_TSO6;
dev->hw_features |= NETIF_F_TSO_ECN;
dev->hw_features |= NETIF_F_RXCSUM;
dev->hw_features |= NETIF_F_RXHASH;
dev->features = dev->hw_features;
dev->watchdog_timeo = 5 * HZ;
dev->min_mtu = ETH_MIN_MTU;
netif_carrier_off(dev);
priv = netdev_priv(dev);
priv->dev = dev;
priv->pdev = pdev;
priv->msg_enable = DEFAULT_MSG_LEVEL;
priv->reg_bar0 = reg_bar;
priv->db_bar2 = db_bar;
priv->service_task_flags = 0x0;
priv->state_flags = 0x0;
gve_set_probe_in_progress(priv);
priv->gve_wq = alloc_ordered_workqueue("gve", 0);
if (!priv->gve_wq) {
dev_err(&pdev->dev, "Could not allocate workqueue");
err = -ENOMEM;
goto abort_with_netdev;
}
INIT_WORK(&priv->service_task, gve_service_task);
priv->tx_cfg.max_queues = max_tx_queues;
priv->rx_cfg.max_queues = max_rx_queues;
err = gve_init_priv(priv, false);
if (err)
goto abort_with_wq;
err = register_netdev(dev);
if (err)
goto abort_with_gve_init;
dev_info(&pdev->dev, "GVE version %s\n", gve_version_str);
gve_clear_probe_in_progress(priv);
queue_work(priv->gve_wq, &priv->service_task);
return 0;
abort_with_gve_init:
gve_teardown_priv_resources(priv);
abort_with_wq:
destroy_workqueue(priv->gve_wq);
abort_with_netdev:
free_netdev(dev);
abort_with_db_bar:
pci_iounmap(pdev, db_bar);
abort_with_reg_bar:
pci_iounmap(pdev, reg_bar);
abort_with_pci_region:
pci_release_regions(pdev);
abort_with_enabled:
pci_disable_device(pdev);
return -ENXIO;
}
static void gve_remove(struct pci_dev *pdev)
{
struct net_device *netdev = pci_get_drvdata(pdev);
struct gve_priv *priv = netdev_priv(netdev);
__be32 __iomem *db_bar = priv->db_bar2;
void __iomem *reg_bar = priv->reg_bar0;
unregister_netdev(netdev);
gve_teardown_priv_resources(priv);
destroy_workqueue(priv->gve_wq);
free_netdev(netdev);
pci_iounmap(pdev, db_bar);
pci_iounmap(pdev, reg_bar);
pci_release_regions(pdev);
pci_disable_device(pdev);
}
static const struct pci_device_id gve_id_table[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC) },
{ }
};
static struct pci_driver gvnic_driver = {
.name = "gvnic",
.id_table = gve_id_table,
.probe = gve_probe,
.remove = gve_remove,
};
module_pci_driver(gvnic_driver);
MODULE_DEVICE_TABLE(pci, gve_id_table);
MODULE_AUTHOR("Google, Inc.");
MODULE_DESCRIPTION("gVNIC Driver");
MODULE_LICENSE("Dual MIT/GPL");
MODULE_VERSION(GVE_VERSION);