gve: Add rx buffer pagecnt bias
Add a pagecnt bias field to rx buffer info struct to eliminate
needing to increment the atomic page ref count on every pass in the
rx hotpath.
Patches squashed into this patch:
- gve: Add rx buffer pagecnt bias
BUG=b/153867289
Change-Id: Iae4344d349f7ee2eeb080ebafc40046b75ef9e22
Reviewed-on: https://cos-review.googlesource.com/c/third_party/kernel/+/1578
Reviewed-by: Vaibhav Rustagi <vaibhavrustagi@google.com>
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 9708138c..56482df 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -54,6 +54,7 @@
struct page *page;
void *page_address;
u32 page_offset; /* offset to write to in page */
+ int pagecnt_bias; /* expected pagecnt if only the driver has a ref */
bool can_flip; /* page can be flipped and reused */
};
diff --git a/drivers/net/ethernet/google/gve/gve_rx.c b/drivers/net/ethernet/google/gve/gve_rx.c
index d730d75..302f443 100644
--- a/drivers/net/ethernet/google/gve/gve_rx.c
+++ b/drivers/net/ethernet/google/gve/gve_rx.c
@@ -23,6 +23,7 @@
dma_addr_t dma = (dma_addr_t)(be64_to_cpu(data_slot->addr) -
page_info->page_offset);
+ page_ref_sub(page_info->page, page_info->pagecnt_bias - 1);
gve_free_page(dev, page_info->page, dma, DMA_FROM_DEVICE);
}
@@ -70,6 +71,9 @@
page_info->page_offset = 0;
page_info->page_address = page_address(page);
slot->addr = cpu_to_be64(addr);
+ /* The page already has 1 ref */
+ page_ref_add(page, INT_MAX - 1);
+ page_info->pagecnt_bias = INT_MAX;
}
static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
@@ -358,21 +362,38 @@
#endif
}
-static int gve_rx_can_recycle_buffer(struct page *page) {
- int pagecount = page_count(page);
+static int gve_rx_can_recycle_buffer(struct gve_rx_slot_page_info *page_info)
+{
+ int pagecount = page_count(page_info->page);
/* This page is not being used by any SKBs - reuse */
- if (pagecount == 1) {
+ if (pagecount == page_info->pagecnt_bias) {
return 1;
/* This page is still being used by an SKB - we can't reuse */
- } else if (pagecount >= 2) {
+ } else if (pagecount > page_info->pagecnt_bias) {
return 0;
} else {
- WARN(pagecount < 1, "Pagecount should never be < 1");
+ WARN(pagecount < page_info->pagecnt_bias,
+ "Pagecount should never be less than the bias.");
return -1;
}
}
+static void gve_rx_update_pagecnt_bias(struct gve_rx_slot_page_info *page_info)
+{
+ page_info->pagecnt_bias--;
+ if (page_info->pagecnt_bias == 0) {
+ int pagecount = page_count(page_info->page);
+
+ /* If we have run out of bias - set it back up to INT_MAX
+ * minus the existing refs.
+ */
+ page_info->pagecnt_bias = INT_MAX - (pagecount);
+ /* Set pagecount back up to max */
+ page_ref_add(page_info->page, INT_MAX - pagecount);
+ }
+}
+
static struct sk_buff *
gve_rx_raw_addressing(struct device *dev, struct net_device *netdev,
struct gve_rx_slot_page_info *page_info, u16 len,
@@ -384,11 +405,11 @@
if (!skb)
return NULL;
- /* Optimistically stop the kernel from freeing the page by increasing
- * the page bias. We will check the refcount in refill to determine if
- * we need to alloc a new page.
+ /* Optimistically stop the kernel from freeing the page.
+ * We will check again in refill to determine if we need to alloc a
+ * new page.
*/
- get_page(page_info->page);
+ gve_rx_update_pagecnt_bias(page_info);
page_info->can_flip = can_flip;
return skb;
@@ -411,7 +432,7 @@
/* No point in recycling if we didn't get the skb */
if (skb) {
/* Make sure the networking stack can't free the page */
- get_page(page_info->page);
+ gve_rx_update_pagecnt_bias(page_info);
gve_rx_flip_buffer(page_info, data_slot);
}
} else {
@@ -468,7 +489,7 @@
int recycle = 0;
if (can_flip) {
- recycle = gve_rx_can_recycle_buffer(page_info->page);
+ recycle = gve_rx_can_recycle_buffer(page_info);
if (recycle < 0) {
gve_schedule_reset(priv);
return false;
@@ -561,7 +582,7 @@
* owns half the page it is impossible to tell which half. Either
* the whole page is free or it needs to be replaced.
*/
- int recycle = gve_rx_can_recycle_buffer(page_info->page);
+ int recycle = gve_rx_can_recycle_buffer(page_info);
if (recycle < 0) {
gve_schedule_reset(priv);
