diff --git a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h
index 7f7978b135a949daeea599879b5661155998904a..b818a169c193ccf6be8b78ff76db085f1c1185ff 100644
--- a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h
+++ b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h
@@ -491,6 +491,36 @@ enum ena_admin_llq_stride_ctrl {
ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY = 2,
};
+enum ena_admin_accel_mode_feat {
+ ENA_ADMIN_DISABLE_META_CACHING = 0,
+ ENA_ADMIN_LIMIT_TX_BURST = 1,
+};
+
+struct ena_admin_accel_mode_get {
+ /* bit field of enum ena_admin_accel_mode_feat */
+ u16 supported_flags;
+
+ /* maximum burst size between two doorbells. The size is in bytes */
+ u16 max_tx_burst_size;
+};
+
+struct ena_admin_accel_mode_set {
+ /* bit field of enum ena_admin_accel_mode_feat */
+ u16 enabled_flags;
+
+ u16 reserved;
+};
+
+struct ena_admin_accel_mode_req {
+ union {
+ u32 raw[2];
+
+ struct ena_admin_accel_mode_get get;
+
+ struct ena_admin_accel_mode_set set;
+ } u;
+};
+
struct ena_admin_feature_llq_desc {
u32 max_llq_num;
@@ -536,10 +566,13 @@ struct ena_admin_feature_llq_desc {
/* the stride control the driver selected to use */
u16 descriptors_stride_ctrl_enabled;
- /* Maximum size in bytes taken by llq entries in a single tx burst.
- * Set to 0 when there is no such limit.
+ /* reserved */
+ u32 reserved1;
+
+ /* accelerated low latency queues requirement. driver needs to
+ * support those requirements in order to use accelerated llq
*/
- u32 max_tx_burst_size;
+ struct ena_admin_accel_mode_req accel_mode;
};
struct ena_admin_queue_ext_feature_fields {
diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c
index 432f143559a1240d4ad942de831d7e4520629b4f..435bf05a853cecf6120bf290373b30f0c590f6d7 100644
--- a/drivers/net/ethernet/amazon/ena/ena_com.c
+++ b/drivers/net/ethernet/amazon/ena/ena_com.c
@@ -403,6 +403,8 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
0x0, io_sq->llq_info.desc_list_entry_size);
io_sq->llq_buf_ctrl.descs_left_in_line =
io_sq->llq_info.descs_num_before_header;
+ io_sq->disable_meta_caching =
+ io_sq->llq_info.disable_meta_caching;
if (io_sq->llq_info.max_entries_in_tx_burst > 0)
io_sq->entries_in_tx_burst_left =
@@ -626,6 +628,10 @@ static int ena_com_set_llq(struct ena_com_dev *ena_dev)
cmd.u.llq.desc_num_before_header_enabled = llq_info->descs_num_before_header;
cmd.u.llq.descriptors_stride_ctrl_enabled = llq_info->desc_stride_ctrl;
+ cmd.u.llq.accel_mode.u.set.enabled_flags =
+ BIT(ENA_ADMIN_DISABLE_META_CACHING) |
+ BIT(ENA_ADMIN_LIMIT_TX_BURST);
+
ret = ena_com_execute_admin_command(admin_queue,
(struct ena_admin_aq_entry *)&cmd,
sizeof(cmd),
@@ -643,6 +649,7 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
struct ena_llq_configurations *llq_default_cfg)
{
struct ena_com_llq_info *llq_info = &ena_dev->llq_info;
+ struct ena_admin_accel_mode_get llq_accel_mode_get;
u16 supported_feat;
int rc;
@@ -742,9 +749,17 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
llq_default_cfg->llq_num_decs_before_header,
supported_feat, llq_info->descs_num_before_header);
}
+ /* Check for accelerated queue supported */
+ llq_accel_mode_get = llq_features->accel_mode.u.get;
+
+ llq_info->disable_meta_caching =
+ !!(llq_accel_mode_get.supported_flags &
+ BIT(ENA_ADMIN_DISABLE_META_CACHING));
- llq_info->max_entries_in_tx_burst =
- (u16)(llq_features->max_tx_burst_size / llq_default_cfg->llq_ring_entry_size_value);
+ if (llq_accel_mode_get.supported_flags & BIT(ENA_ADMIN_LIMIT_TX_BURST))
+ llq_info->max_entries_in_tx_burst =
+ llq_accel_mode_get.max_tx_burst_size /
+ llq_default_cfg->llq_ring_entry_size_value;
rc = ena_com_set_llq(ena_dev);
if (rc)
diff --git a/drivers/net/ethernet/amazon/ena/ena_com.h b/drivers/net/ethernet/amazon/ena/ena_com.h
index 4c98f6f07882fe07d97407d0313347fef8f60eaf..4287d47b2b0bc1d1800035a02b0cc1d9fefcc99e 100644
--- a/drivers/net/ethernet/amazon/ena/ena_com.h
+++ b/drivers/net/ethernet/amazon/ena/ena_com.h
@@ -127,6 +127,7 @@ struct ena_com_llq_info {
u16 descs_num_before_header;
u16 descs_per_entry;
u16 max_entries_in_tx_burst;
+ bool disable_meta_caching;
};
struct ena_com_io_cq {
@@ -189,6 +190,8 @@ struct ena_com_io_sq {
enum queue_direction direction;
enum ena_admin_placement_policy_type mem_queue_type;
+ bool disable_meta_caching;
+
u32 msix_vector;
struct ena_com_tx_meta cached_tx_meta;
struct ena_com_llq_info llq_info;
diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.c b/drivers/net/ethernet/amazon/ena/ena_eth_com.c
index ec8ea25e988de48f3615df9e07cac2ed50eaf279..ccd440589565160108c0243749ea1f3d7a2a5e4d 100644
--- a/drivers/net/ethernet/amazon/ena/ena_eth_com.c
+++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.c
@@ -285,11 +285,10 @@ static u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
return count;
}
-static int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq,
- struct ena_com_tx_ctx *ena_tx_ctx)
+static int ena_com_create_meta(struct ena_com_io_sq *io_sq,
+ struct ena_com_tx_meta *ena_meta)
{
struct ena_eth_io_tx_meta_desc *meta_desc = NULL;
- struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
meta_desc = get_sq_desc(io_sq);
memset(meta_desc, 0x0, sizeof(struct ena_eth_io_tx_meta_desc));
@@ -309,12 +308,13 @@ static int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq,
/* Extended meta desc */
meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK;
- meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
meta_desc->len_ctrl |= (io_sq->phase <<
ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT) &
ENA_ETH_IO_TX_META_DESC_PHASE_MASK;
meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_FIRST_MASK;
+ meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
+
meta_desc->word2 |= ena_meta->l3_hdr_len &
ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK;
meta_desc->word2 |= (ena_meta->l3_hdr_offset <<
@@ -325,13 +325,36 @@ static int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq,
ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT) &
ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK;
- meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
+ return ena_com_sq_update_tail(io_sq);
+}
+
+static int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq,
+ struct ena_com_tx_ctx *ena_tx_ctx,
+ bool *have_meta)
+{
+ struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
- /* Cached the meta desc */
- memcpy(&io_sq->cached_tx_meta, ena_meta,
- sizeof(struct ena_com_tx_meta));
+ /* When disable meta caching is set, don't bother to save the meta and
+ * compare it to the stored version, just create the meta
+ */
+ if (io_sq->disable_meta_caching) {
+ if (unlikely(!ena_tx_ctx->meta_valid))
+ return -EINVAL;
- return ena_com_sq_update_tail(io_sq);
+ *have_meta = true;
+ return ena_com_create_meta(io_sq, ena_meta);
+ }
+
+ if (ena_com_meta_desc_changed(io_sq, ena_tx_ctx)) {
+ *have_meta = true;
+ /* Cache the meta desc */
+ memcpy(&io_sq->cached_tx_meta, ena_meta,
+ sizeof(struct ena_com_tx_meta));
+ return ena_com_create_meta(io_sq, ena_meta);
+ }
+
+ *have_meta = false;
+ return 0;
}
static void ena_com_rx_set_flags(struct ena_com_rx_ctx *ena_rx_ctx,
@@ -402,12 +425,10 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
if (unlikely(rc))
return rc;
- have_meta = ena_tx_ctx->meta_valid && ena_com_meta_desc_changed(io_sq,
- ena_tx_ctx);
- if (have_meta) {
- rc = ena_com_create_and_store_tx_meta_desc(io_sq, ena_tx_ctx);
- if (unlikely(rc))
- return rc;
+ rc = ena_com_create_and_store_tx_meta_desc(io_sq, ena_tx_ctx, &have_meta);
+ if (unlikely(rc)) {
+ pr_err("failed to create and store tx meta desc\n");
+ return rc;
}
/* If the caller doesn't want to send packets */
diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.h b/drivers/net/ethernet/amazon/ena/ena_eth_com.h
index 8b1afd3b32f26c27987792a92219080a3981262b..b6592cb93b045ce03f4bd9cd9089947b3bf1312c 100644
--- a/drivers/net/ethernet/amazon/ena/ena_eth_com.h
+++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.h
@@ -157,7 +157,8 @@ static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq,
llq_info = &io_sq->llq_info;
num_descs = ena_tx_ctx->num_bufs;
- if (unlikely(ena_com_meta_desc_changed(io_sq, ena_tx_ctx)))
+ if (llq_info->disable_meta_caching ||
+ unlikely(ena_com_meta_desc_changed(io_sq, ena_tx_ctx)))
++num_descs;
if (num_descs > llq_info->descs_num_before_header) {
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c
index 913d698b9834a2037b5f40f957fb996b35bca01a..6478c1e0d137e916dc0ab0d3a3dcf09d4cee7735 100644
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c
@@ -655,6 +655,7 @@ static void ena_init_io_rings(struct ena_adapter *adapter,
txr->sgl_size = adapter->max_tx_sgl_size;
txr->smoothed_interval =
ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
+ txr->disable_meta_caching = adapter->disable_meta_caching;
/* Don't init RX queues for xdp queues */
if (!ENA_IS_XDP_INDEX(adapter, i)) {
@@ -2783,7 +2784,9 @@ int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count)
return dev_was_up ? ena_open(adapter->netdev) : 0;
}
-static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct sk_buff *skb)
+static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx,
+ struct sk_buff *skb,
+ bool disable_meta_caching)
{
u32 mss = skb_shinfo(skb)->gso_size;
struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
@@ -2827,7 +2830,9 @@ static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct sk_buff *skb)
ena_meta->l3_hdr_len = skb_network_header_len(skb);
ena_meta->l3_hdr_offset = skb_network_offset(skb);
ena_tx_ctx->meta_valid = 1;
-
+ } else if (disable_meta_caching) {
+ memset(ena_meta, 0, sizeof(*ena_meta));
+ ena_tx_ctx->meta_valid = 1;
} else {
ena_tx_ctx->meta_valid = 0;
}
@@ -3011,7 +3016,7 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
ena_tx_ctx.header_len = header_len;
/* set flags and meta data */
- ena_tx_csum(&ena_tx_ctx, skb);
+ ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching);
rc = ena_xmit_common(dev,
tx_ring,
@@ -4260,6 +4265,11 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
adapter->xdp_num_queues = 0;
adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK;
+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
+ adapter->disable_meta_caching =
+ !!(get_feat_ctx.llq.accel_mode.u.get.supported_flags &
+ BIT(ENA_ADMIN_DISABLE_META_CACHING));
+
adapter->wd_state = wd_state;
snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found);
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.h b/drivers/net/ethernet/amazon/ena/ena_netdev.h
index 89304b403995d59be79091fdada6c97ec747206c..0c850400624715682a602eb855ebf5deea1acebf 100644
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.h
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.h
@@ -298,6 +298,7 @@ struct ena_ring {
u8 tx_max_header_size;
bool first_interrupt;
+ bool disable_meta_caching;
u16 no_interrupt_event_cnt;
/* cpu for TPH */
@@ -399,6 +400,7 @@ struct ena_adapter {
bool wd_state;
bool dev_up_before_reset;
+ bool disable_meta_caching;
unsigned long last_keep_alive_jiffies;
struct u64_stats_sync syncp;