Commit bec5545e authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'ntb-5.3' of git://github.com/jonmason/ntb

Pull NTB updates from Jon Mason:
 "New feature to add support for NTB virtual MSI interrupts, the ability
  to test and use this feature in the NTB transport layer.

  Also, bug fixes for the AMD and Switchtec drivers, as well as some
  general patches"

* tag 'ntb-5.3' of git://github.com/jonmason/ntb: (22 commits)
  NTB: Describe the ntb_msi_test client in the documentation.
  NTB: Add MSI interrupt support to ntb_transport
  NTB: Add ntb_msi_test support to ntb_test
  NTB: Introduce NTB MSI Test Client
  NTB: Introduce MSI library
  NTB: Rename ntb.c to support multiple source files in the module
  NTB: Introduce functions to calculate multi-port resource index
  NTB: Introduce helper functions to calculate logical port number
  PCI/switchtec: Add module parameter to request more interrupts
  PCI/MSI: Support allocating virtual MSI interrupts
  ntb_hw_switchtec: Fix setup MW with failure bug
  ntb_hw_switchtec: Skip unnecessary re-setup of shared memory window for crosslink case
  ntb_hw_switchtec: Remove redundant steps of switchtec_ntb_reinit_peer() function
  NTB: correct ntb_dev_ops and ntb_dev comment typos
  NTB: amd: Silence shift wrapping warning in amd_ntb_db_vector_mask()
  ntb_hw_switchtec: potential shift wrapping bug in switchtec_ntb_init_sndev()
  NTB: ntb_transport: Ensure qp->tx_mw_dma_addr is initaliazed
  NTB: ntb_hw_amd: set peer limit register
  NTB: ntb_perf: Clear stale values in doorbell and command SPAD register
  NTB: ntb_perf: Disable NTB link after clearing peer XLAT registers
  ...
parents f1a3b43c d9c53aa4
......@@ -200,6 +200,33 @@ Debugfs Files:
This file is used to read and write peer scratchpads. See
*spad* for details.
NTB MSI Test Client (ntb\_msi\_test)
------------------------------------
The MSI test client serves to test and debug the MSI library which
allows for passing MSI interrupts across NTB memory windows. The
test client is interacted with through the debugfs filesystem:
* *debugfs*/ntb\_tool/*hw*/
A directory in debugfs will be created for each
NTB device probed by the tool. This directory is shortened to *hw*
below.
* *hw*/port
This file describes the local port number
* *hw*/irq*_occurrences
One occurrences file exists for each interrupt and, when read,
returns the number of times the interrupt has been triggered.
* *hw*/peer*/port
This file describes the port number for each peer
* *hw*/peer*/count
This file describes the number of interrupts that can be
triggered on each peer
* *hw*/peer*/trigger
Writing an interrupt number (any number less than the value
specified in count) will trigger the interrupt on the
specified peer. That peer's interrupt's occurrence file
should be incremented.
NTB Hardware Drivers
====================
......
......@@ -13,6 +13,17 @@ menuconfig NTB
if NTB
config NTB_MSI
bool "MSI Interrupt Support"
depends on PCI_MSI
help
Support using MSI interrupt forwarding instead of (or in addition to)
hardware doorbells. MSI interrupts typically offer lower latency
than doorbells and more MSI interrupts can be made available to
clients. However this requires an extra memory window and support
in the hardware driver for creating the MSI interrupts.
If unsure, say N.
source "drivers/ntb/hw/Kconfig"
source "drivers/ntb/test/Kconfig"
......
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_NTB) += ntb.o hw/ test/
obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o
ntb-y := core.o
ntb-$(CONFIG_NTB_MSI) += msi.o
......@@ -160,8 +160,8 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
}
/* set and verify setting the limit */
write64(limit, mmio + limit_reg);
reg_val = read64(mmio + limit_reg);
write64(limit, peer_mmio + limit_reg);
reg_val = read64(peer_mmio + limit_reg);
if (reg_val != limit) {
write64(base_addr, mmio + limit_reg);
write64(0, peer_mmio + xlat_reg);
......@@ -183,8 +183,8 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
}
/* set and verify setting the limit */
writel(limit, mmio + limit_reg);
reg_val = readl(mmio + limit_reg);
writel(limit, peer_mmio + limit_reg);
reg_val = readl(peer_mmio + limit_reg);
if (reg_val != limit) {
writel(base_addr, mmio + limit_reg);
writel(0, peer_mmio + xlat_reg);
......@@ -333,7 +333,7 @@ static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
if (db_vector < 0 || db_vector > ndev->db_count)
return 0;
return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector);
return ntb_ndev(ntb)->db_valid_mask & (1ULL << db_vector);
}
static u64 amd_ntb_db_read(struct ntb_dev *ntb)
......
......@@ -532,9 +532,9 @@ static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
return 0;
}
int intel_ntb3_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
resource_size_t *db_size,
u64 *db_data, int db_bit)
static int intel_ntb3_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
resource_size_t *db_size,
u64 *db_data, int db_bit)
{
phys_addr_t db_addr_base;
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
......
......@@ -86,7 +86,8 @@ struct switchtec_ntb {
bool link_is_up;
enum ntb_speed link_speed;
enum ntb_width link_width;
struct work_struct link_reinit_work;
struct work_struct check_link_status_work;
bool link_force_down;
};
static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb)
......@@ -485,33 +486,11 @@ enum switchtec_msg {
static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
static void link_reinit_work(struct work_struct *work)
{
struct switchtec_ntb *sndev;
sndev = container_of(work, struct switchtec_ntb, link_reinit_work);
switchtec_ntb_reinit_peer(sndev);
}
static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
enum switchtec_msg msg)
static void switchtec_ntb_link_status_update(struct switchtec_ntb *sndev)
{
int link_sta;
int old = sndev->link_is_up;
if (msg == MSG_LINK_FORCE_DOWN) {
schedule_work(&sndev->link_reinit_work);
if (sndev->link_is_up) {
sndev->link_is_up = 0;
ntb_link_event(&sndev->ntb);
dev_info(&sndev->stdev->dev, "ntb link forced down\n");
}
return;
}
link_sta = sndev->self_shared->link_sta;
if (link_sta) {
u64 peer = ioread64(&sndev->peer_shared->magic);
......@@ -536,6 +515,38 @@ static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
}
}
static void check_link_status_work(struct work_struct *work)
{
struct switchtec_ntb *sndev;
sndev = container_of(work, struct switchtec_ntb,
check_link_status_work);
if (sndev->link_force_down) {
sndev->link_force_down = false;
switchtec_ntb_reinit_peer(sndev);
if (sndev->link_is_up) {
sndev->link_is_up = 0;
ntb_link_event(&sndev->ntb);
dev_info(&sndev->stdev->dev, "ntb link forced down\n");
}
return;
}
switchtec_ntb_link_status_update(sndev);
}
static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
enum switchtec_msg msg)
{
if (msg == MSG_LINK_FORCE_DOWN)
sndev->link_force_down = true;
schedule_work(&sndev->check_link_status_work);
}
static void switchtec_ntb_link_notification(struct switchtec_dev *stdev)
{
struct switchtec_ntb *sndev = stdev->sndev;
......@@ -568,7 +579,7 @@ static int switchtec_ntb_link_enable(struct ntb_dev *ntb,
sndev->self_shared->link_sta = 1;
switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP);
switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
switchtec_ntb_link_status_update(sndev);
return 0;
}
......@@ -582,7 +593,7 @@ static int switchtec_ntb_link_disable(struct ntb_dev *ntb)
sndev->self_shared->link_sta = 0;
switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN);
switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
switchtec_ntb_link_status_update(sndev);
return 0;
}
......@@ -835,7 +846,8 @@ static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
sndev->ntb.topo = NTB_TOPO_SWITCH;
sndev->ntb.ops = &switchtec_ntb_ops;
INIT_WORK(&sndev->link_reinit_work, link_reinit_work);
INIT_WORK(&sndev->check_link_status_work, check_link_status_work);
sndev->link_force_down = false;
sndev->self_partition = sndev->stdev->partition;
......@@ -872,7 +884,7 @@ static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
}
sndev->peer_partition = ffs(tpart_vec) - 1;
if (!(part_map & (1 << sndev->peer_partition))) {
if (!(part_map & (1ULL << sndev->peer_partition))) {
dev_err(&sndev->stdev->dev,
"ntb target partition is not NT partition\n");
return -ENODEV;
......@@ -1448,10 +1460,16 @@ static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)
static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
{
dev_info(&sndev->stdev->dev, "peer reinitialized\n");
switchtec_ntb_deinit_shared_mw(sndev);
switchtec_ntb_init_mw(sndev);
return switchtec_ntb_init_shared_mw(sndev);
int rc;
if (crosslink_is_enabled(sndev))
return 0;
dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n");
rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
sndev->self_partition,
sndev->self_shared_dma);
return rc;
}
static int switchtec_ntb_add(struct device *dev,
......
This diff is collapsed.
......@@ -93,6 +93,12 @@ static bool use_dma;
module_param(use_dma, bool, 0644);
MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy");
static bool use_msi;
#ifdef CONFIG_NTB_MSI
module_param(use_msi, bool, 0644);
MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells");
#endif
static struct dentry *nt_debugfs_dir;
/* Only two-ports NTB devices are supported */
......@@ -188,6 +194,11 @@ struct ntb_transport_qp {
u64 tx_err_no_buf;
u64 tx_memcpy;
u64 tx_async;
bool use_msi;
int msi_irq;
struct ntb_msi_desc msi_desc;
struct ntb_msi_desc peer_msi_desc;
};
struct ntb_transport_mw {
......@@ -221,6 +232,10 @@ struct ntb_transport_ctx {
u64 qp_bitmap;
u64 qp_bitmap_free;
bool use_msi;
unsigned int msi_spad_offset;
u64 msi_db_mask;
bool link_is_up;
struct delayed_work link_work;
struct work_struct link_cleanup;
......@@ -667,6 +682,114 @@ static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
return 0;
}
static irqreturn_t ntb_transport_isr(int irq, void *dev)
{
struct ntb_transport_qp *qp = dev;
tasklet_schedule(&qp->rxc_db_work);
return IRQ_HANDLED;
}
static void ntb_transport_setup_qp_peer_msi(struct ntb_transport_ctx *nt,
unsigned int qp_num)
{
struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
int spad = qp_num * 2 + nt->msi_spad_offset;
if (!nt->use_msi)
return;
if (spad >= ntb_spad_count(nt->ndev))
return;
qp->peer_msi_desc.addr_offset =
ntb_peer_spad_read(qp->ndev, PIDX, spad);
qp->peer_msi_desc.data =
ntb_peer_spad_read(qp->ndev, PIDX, spad + 1);
dev_dbg(&qp->ndev->pdev->dev, "QP%d Peer MSI addr=%x data=%x\n",
qp_num, qp->peer_msi_desc.addr_offset, qp->peer_msi_desc.data);
if (qp->peer_msi_desc.addr_offset) {
qp->use_msi = true;
dev_info(&qp->ndev->pdev->dev,
"Using MSI interrupts for QP%d\n", qp_num);
}
}
static void ntb_transport_setup_qp_msi(struct ntb_transport_ctx *nt,
unsigned int qp_num)
{
struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
int spad = qp_num * 2 + nt->msi_spad_offset;
int rc;
if (!nt->use_msi)
return;
if (spad >= ntb_spad_count(nt->ndev)) {
dev_warn_once(&qp->ndev->pdev->dev,
"Not enough SPADS to use MSI interrupts\n");
return;
}
ntb_spad_write(qp->ndev, spad, 0);
ntb_spad_write(qp->ndev, spad + 1, 0);
if (!qp->msi_irq) {
qp->msi_irq = ntbm_msi_request_irq(qp->ndev, ntb_transport_isr,
KBUILD_MODNAME, qp,
&qp->msi_desc);
if (qp->msi_irq < 0) {
dev_warn(&qp->ndev->pdev->dev,
"Unable to allocate MSI interrupt for qp%d\n",
qp_num);
return;
}
}
rc = ntb_spad_write(qp->ndev, spad, qp->msi_desc.addr_offset);
if (rc)
goto err_free_interrupt;
rc = ntb_spad_write(qp->ndev, spad + 1, qp->msi_desc.data);
if (rc)
goto err_free_interrupt;
dev_dbg(&qp->ndev->pdev->dev, "QP%d MSI %d addr=%x data=%x\n",
qp_num, qp->msi_irq, qp->msi_desc.addr_offset,
qp->msi_desc.data);
return;
err_free_interrupt:
devm_free_irq(&nt->ndev->dev, qp->msi_irq, qp);
}
static void ntb_transport_msi_peer_desc_changed(struct ntb_transport_ctx *nt)
{
int i;
dev_dbg(&nt->ndev->pdev->dev, "Peer MSI descriptors changed");
for (i = 0; i < nt->qp_count; i++)
ntb_transport_setup_qp_peer_msi(nt, i);
}
static void ntb_transport_msi_desc_changed(void *data)
{
struct ntb_transport_ctx *nt = data;
int i;
dev_dbg(&nt->ndev->pdev->dev, "MSI descriptors changed");
for (i = 0; i < nt->qp_count; i++)
ntb_transport_setup_qp_msi(nt, i);
ntb_peer_db_set(nt->ndev, nt->msi_db_mask);
}
static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
{
struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
......@@ -905,6 +1028,20 @@ static void ntb_transport_link_work(struct work_struct *work)
int rc = 0, i, spad;
/* send the local info, in the opposite order of the way we read it */
if (nt->use_msi) {
rc = ntb_msi_setup_mws(ndev);
if (rc) {
dev_warn(&pdev->dev,
"Failed to register MSI memory window: %d\n",
rc);
nt->use_msi = false;
}
}
for (i = 0; i < nt->qp_count; i++)
ntb_transport_setup_qp_msi(nt, i);
for (i = 0; i < nt->mw_count; i++) {
size = nt->mw_vec[i].phys_size;
......@@ -962,6 +1099,7 @@ static void ntb_transport_link_work(struct work_struct *work)
struct ntb_transport_qp *qp = &nt->qp_vec[i];
ntb_transport_setup_qp_mw(nt, i);
ntb_transport_setup_qp_peer_msi(nt, i);
if (qp->client_ready)
schedule_delayed_work(&qp->link_work, 0);
......@@ -1135,6 +1273,19 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
return -ENOMEM;
nt->ndev = ndev;
/*
* If we are using MSI, and have at least one extra memory window,
* we will reserve the last MW for the MSI window.
*/
if (use_msi && mw_count > 1) {
rc = ntb_msi_init(ndev, ntb_transport_msi_desc_changed);
if (!rc) {
mw_count -= 1;
nt->use_msi = true;
}
}
spad_count = ntb_spad_count(ndev);
/* Limit the MW's based on the availability of scratchpads */
......@@ -1148,6 +1299,8 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
max_mw_count_for_spads = (spad_count - MW0_SZ_HIGH) / 2;
nt->mw_count = min(mw_count, max_mw_count_for_spads);
nt->msi_spad_offset = nt->mw_count * 2 + MW0_SZ_HIGH;
nt->mw_vec = kcalloc_node(mw_count, sizeof(*nt->mw_vec),
GFP_KERNEL, node);
if (!nt->mw_vec) {
......@@ -1178,6 +1331,12 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
qp_bitmap = ntb_db_valid_mask(ndev);
qp_count = ilog2(qp_bitmap);
if (nt->use_msi) {
qp_count -= 1;
nt->msi_db_mask = 1 << qp_count;
ntb_db_clear_mask(ndev, nt->msi_db_mask);
}
if (max_num_clients && max_num_clients < qp_count)
qp_count = max_num_clients;
else if (nt->mw_count < qp_count)
......@@ -1601,7 +1760,10 @@ static void ntb_tx_copy_callback(void *data,
iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags);
ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num));
if (qp->use_msi)
ntb_msi_peer_trigger(qp->ndev, PIDX, &qp->peer_msi_desc);
else
ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num));
/* The entry length can only be zero if the packet is intended to be a
* "link down" or similar. Since no payload is being sent in these
......@@ -1869,6 +2031,7 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
qp->rx_dma_chan = NULL;
}
qp->tx_mw_dma_addr = 0;
if (qp->tx_dma_chan) {
qp->tx_mw_dma_addr =
dma_map_resource(qp->tx_dma_chan->device->dev,
......@@ -2268,6 +2431,11 @@ static void ntb_transport_doorbell_callback(void *data, int vector)
u64 db_bits;
unsigned int qp_num;
if (ntb_db_read(nt->ndev) & nt->msi_db_mask) {
ntb_transport_msi_peer_desc_changed(nt);
ntb_db_clear(nt->ndev, nt->msi_db_mask);
}
db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free &
ntb_db_vector_mask(nt->ndev, vector));
......
......@@ -26,3 +26,12 @@ config NTB_PERF
to and from the window without additional software interaction.
If unsure, say N.
config NTB_MSI_TEST
tristate "NTB MSI Test Client"
depends on NTB_MSI
help
This tool demonstrates the use of the NTB MSI library to
send MSI interrupts between peers.
If unsure, say N.
......@@ -2,3 +2,4 @@
obj-$(CONFIG_NTB_PINGPONG) += ntb_pingpong.o
obj-$(CONFIG_NTB_TOOL) += ntb_tool.o
obj-$(CONFIG_NTB_PERF) += ntb_perf.o
obj-$(CONFIG_NTB_MSI_TEST) += ntb_msi_test.o
// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
#include <linux/module.h>
#include <linux/debugfs.h>
#include <linux/ntb.h>
#include <linux/pci.h>
#include <linux/radix-tree.h>
#include <linux/workqueue.h>
MODULE_LICENSE("Dual BSD/GPL");
MODULE_VERSION("0.1");
MODULE_AUTHOR("Logan Gunthorpe <logang@deltatee.com>");
MODULE_DESCRIPTION("Test for sending MSI interrupts over an NTB memory window");
static int num_irqs = 4;
module_param(num_irqs, int, 0644);
MODULE_PARM_DESC(num_irqs, "number of irqs to use");
struct ntb_msit_ctx {
struct ntb_dev *ntb;
struct dentry *dbgfs_dir;
struct work_struct setup_work;
struct ntb_msit_isr_ctx {
int irq_idx;
int irq_num;
int occurrences;
struct ntb_msit_ctx *nm;
struct ntb_msi_desc desc;
} *isr_ctx;
struct ntb_msit_peer {
struct ntb_msit_ctx *nm;
int pidx;
int num_irqs;
struct completion init_comp;
struct ntb_msi_desc *msi_desc;
} peers[];
};
static struct dentry *ntb_msit_dbgfs_topdir;
static irqreturn_t ntb_msit_isr(int irq, void *dev)
{
struct ntb_msit_isr_ctx *isr_ctx = dev;
struct ntb_msit_ctx *nm = isr_ctx->nm;
dev_dbg(&nm->ntb->dev, "Interrupt Occurred: %d",
isr_ctx->irq_idx);
isr_ctx->occurrences++;
return IRQ_HANDLED;
}
static void ntb_msit_setup_work(struct work_struct *work)
{
struct ntb_msit_ctx *nm = container_of(work, struct ntb_msit_ctx,
setup_work);
int irq_count = 0;
int irq;
int ret;
uintptr_t i;
ret = ntb_msi_setup_mws(nm->ntb);
if (ret) {
dev_err(&nm->ntb->dev, "Unable to setup MSI windows: %d\n",
ret);
return;
}
for (i = 0; i < num_irqs; i++) {
nm->isr_ctx[i].irq_idx = i;
nm->isr_ctx[i].nm = nm;
if (!nm->isr_ctx[i].irq_num) {
irq = ntbm_msi_request_irq(nm->ntb, ntb_msit_isr,
KBUILD_MODNAME,
&nm->isr_ctx[i],
&nm->isr_ctx[i].desc);
if (irq < 0)
break;
nm->isr_ctx[i].irq_num = irq;
}
ret = ntb_spad_write(nm->ntb, 2 * i + 1,
nm->isr_ctx[i].desc.addr_offset);
if (ret)
break;
ret = ntb_spad_write(nm->ntb, 2 * i + 2,
nm->isr_ctx[i].desc.data);
if (ret)
break;
irq_count++;
}
ntb_spad_write(nm->ntb, 0, irq_count);
ntb_peer_db_set(nm->ntb, BIT(ntb_port_number(nm->ntb)));
}
static void ntb_msit_desc_changed(void *ctx)
{
struct ntb_msit_ctx *nm = ctx;
int i;
dev_dbg(&nm->ntb->dev, "MSI Descriptors Changed\n");
for (i = 0; i < num_irqs; i++) {
ntb_spad_write(nm->ntb, 2 * i + 1,
nm->isr_ctx[i].desc.addr_offset);
ntb_spad_write(nm->ntb, 2 * i + 2,
nm->isr_ctx[i].desc.data);
}
ntb_peer_db_set(nm->ntb, BIT(ntb_port_number(nm->ntb)));
}
static void ntb_msit_link_event(void *ctx)
{
struct ntb_msit_ctx *nm = ctx;
if (!ntb_link_is_up(nm->ntb, NULL, NULL))
return;
schedule_work