summaryrefslogtreecommitdiff
path: root/drivers/pci/endpoint
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/pci/endpoint')
-rw-r--r--drivers/pci/endpoint/functions/pci-epf-test.c10
-rw-r--r--drivers/pci/endpoint/functions/pci-epf-vntb.c153
-rw-r--r--drivers/pci/endpoint/pci-epf-core.c159
3 files changed, 268 insertions, 54 deletions
diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
index 31617772ad51..debd235253c5 100644
--- a/drivers/pci/endpoint/functions/pci-epf-test.c
+++ b/drivers/pci/endpoint/functions/pci-epf-test.c
@@ -331,9 +331,8 @@ static void pci_epf_test_print_rate(struct pci_epf_test *epf_test,
rate = div64_u64(size * NSEC_PER_SEC, ns * 1000);
dev_info(&epf_test->epf->dev,
- "%s => Size: %llu B, DMA: %s, Time: %llu.%09u s, Rate: %llu KB/s\n",
- op, size, dma ? "YES" : "NO",
- (u64)ts.tv_sec, (u32)ts.tv_nsec, rate);
+ "%s => Size: %llu B, DMA: %s, Time: %ptSp s, Rate: %llu KB/s\n",
+ op, size, dma ? "YES" : "NO", &ts, rate);
}
static void pci_epf_test_copy(struct pci_epf_test *epf_test,
@@ -730,8 +729,9 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test,
if (bar < BAR_0)
goto err_doorbell_cleanup;
- ret = request_irq(epf->db_msg[0].virq, pci_epf_test_doorbell_handler, 0,
- "pci-ep-test-doorbell", epf_test);
+ ret = request_threaded_irq(epf->db_msg[0].virq, NULL,
+ pci_epf_test_doorbell_handler, IRQF_ONESHOT,
+ "pci-ep-test-doorbell", epf_test);
if (ret) {
dev_err(&epf->dev,
"Failed to request doorbell IRQ: %d\n",
diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c
index 83e9ab10f9c4..3ecc5059f92b 100644
--- a/drivers/pci/endpoint/functions/pci-epf-vntb.c
+++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c
@@ -36,11 +36,13 @@
* PCIe Root Port PCI EP
*/
+#include <linux/atomic.h>
#include <linux/delay.h>
#include <linux/io.h>
#include <linux/module.h>
#include <linux/slab.h>
+#include <linux/pci-ep-msi.h>
#include <linux/pci-epc.h>
#include <linux/pci-epf.h>
#include <linux/ntb.h>
@@ -126,12 +128,13 @@ struct epf_ntb {
u32 db_count;
u32 spad_count;
u64 mws_size[MAX_MW];
- u64 db;
+ atomic64_t db;
u32 vbus_number;
u16 vntb_pid;
u16 vntb_vid;
bool linkup;
+ bool msi_doorbell;
u32 spad_size;
enum pci_barno epf_ntb_bar[VNTB_BAR_NUM];
@@ -258,9 +261,9 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
ntb = container_of(work, struct epf_ntb, cmd_handler.work);
- for (i = 1; i < ntb->db_count; i++) {
+ for (i = 1; i < ntb->db_count && !ntb->msi_doorbell; i++) {
if (ntb->epf_db[i]) {
- ntb->db |= 1 << (i - 1);
+ atomic64_or(1 << (i - 1), &ntb->db);
ntb_db_event(&ntb->ntb, i);
ntb->epf_db[i] = 0;
}
@@ -319,7 +322,21 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
reset_handler:
queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler,
- msecs_to_jiffies(5));
+ ntb->msi_doorbell ? msecs_to_jiffies(500) : msecs_to_jiffies(5));
+}
+
+static irqreturn_t epf_ntb_doorbell_handler(int irq, void *data)
+{
+ struct epf_ntb *ntb = data;
+ int i;
+
+ for (i = 1; i < ntb->db_count; i++)
+ if (irq == ntb->epf->db_msg[i].virq) {
+ atomic64_or(1 << (i - 1), &ntb->db);
+ ntb_db_event(&ntb->ntb, i);
+ }
+
+ return IRQ_HANDLED;
}
/**
@@ -500,6 +517,94 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
return 0;
}
+static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
+ struct pci_epf_bar *db_bar,
+ const struct pci_epc_features *epc_features,
+ enum pci_barno barno)
+{
+ struct pci_epf *epf = ntb->epf;
+ dma_addr_t low, high;
+ struct msi_msg *msg;
+ size_t sz;
+ int ret;
+ int i;
+
+ ret = pci_epf_alloc_doorbell(epf, ntb->db_count);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < ntb->db_count; i++) {
+ ret = request_irq(epf->db_msg[i].virq, epf_ntb_doorbell_handler,
+ 0, "pci_epf_vntb_db", ntb);
+
+ if (ret) {
+ dev_err(&epf->dev,
+ "Failed to request doorbell IRQ: %d\n",
+ epf->db_msg[i].virq);
+ goto err_free_irq;
+ }
+ }
+
+ msg = &epf->db_msg[0].msg;
+
+ high = 0;
+ low = (u64)msg->address_hi << 32 | msg->address_lo;
+
+ for (i = 0; i < ntb->db_count; i++) {
+ struct msi_msg *msg = &epf->db_msg[i].msg;
+ dma_addr_t addr = (u64)msg->address_hi << 32 | msg->address_lo;
+
+ low = min(low, addr);
+ high = max(high, addr);
+ }
+
+ sz = high - low + sizeof(u32);
+
+ ret = pci_epf_assign_bar_space(epf, sz, barno, epc_features, 0, low);
+ if (ret) {
+ dev_err(&epf->dev, "Failed to assign Doorbell BAR space\n");
+ goto err_free_irq;
+ }
+
+ ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no,
+ ntb->epf->vfunc_no, db_bar);
+ if (ret) {
+ dev_err(&epf->dev, "Failed to set Doorbell BAR\n");
+ goto err_free_irq;
+ }
+
+ for (i = 0; i < ntb->db_count; i++) {
+ struct msi_msg *msg = &epf->db_msg[i].msg;
+ dma_addr_t addr;
+ size_t offset;
+
+ ret = pci_epf_align_inbound_addr(epf, db_bar->barno,
+ ((u64)msg->address_hi << 32) | msg->address_lo,
+ &addr, &offset);
+
+ if (ret) {
+ ntb->msi_doorbell = false;
+ goto err_free_irq;
+ }
+
+ ntb->reg->db_data[i] = msg->data;
+ ntb->reg->db_offset[i] = offset;
+ }
+
+ ntb->reg->db_entry_size = 0;
+
+ ntb->msi_doorbell = true;
+
+ return 0;
+
+err_free_irq:
+ for (i--; i >= 0; i--)
+ free_irq(epf->db_msg[i].virq, ntb);
+
+ pci_epf_free_doorbell(ntb->epf);
+ return ret;
+}
+
/**
* epf_ntb_db_bar_init() - Configure Doorbell window BARs
* @ntb: NTB device that facilitates communication between HOST and VHOST
@@ -520,21 +625,25 @@ static int epf_ntb_db_bar_init(struct epf_ntb *ntb)
ntb->epf->func_no,
ntb->epf->vfunc_no);
barno = ntb->epf_ntb_bar[BAR_DB];
-
- mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, epc_features, 0);
- if (!mw_addr) {
- dev_err(dev, "Failed to allocate OB address\n");
- return -ENOMEM;
- }
-
- ntb->epf_db = mw_addr;
-
epf_bar = &ntb->epf->bar[barno];
- ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);
+ ret = epf_ntb_db_bar_init_msi_doorbell(ntb, epf_bar, epc_features, barno);
if (ret) {
- dev_err(dev, "Doorbell BAR set failed\n");
+ /* fall back to polling mode */
+ mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, epc_features, 0);
+ if (!mw_addr) {
+ dev_err(dev, "Failed to allocate OB address\n");
+ return -ENOMEM;
+ }
+
+ ntb->epf_db = mw_addr;
+
+ ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no,
+ ntb->epf->vfunc_no, epf_bar);
+ if (ret) {
+ dev_err(dev, "Doorbell BAR set failed\n");
goto err_alloc_peer_mem;
+ }
}
return ret;
@@ -554,6 +663,16 @@ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
{
enum pci_barno barno;
+ if (ntb->msi_doorbell) {
+ int i;
+
+ for (i = 0; i < ntb->db_count; i++)
+ free_irq(ntb->epf->db_msg[i].virq, ntb);
+ }
+
+ if (ntb->epf->db_msg)
+ pci_epf_free_doorbell(ntb->epf);
+
barno = ntb->epf_ntb_bar[BAR_DB];
pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0);
pci_epc_clear_bar(ntb->epf->epc,
@@ -1268,7 +1387,7 @@ static u64 vntb_epf_db_read(struct ntb_dev *ndev)
{
struct epf_ntb *ntb = ntb_ndev(ndev);
- return ntb->db;
+ return atomic64_read(&ntb->db);
}
static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx,
@@ -1308,7 +1427,7 @@ static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits)
{
struct epf_ntb *ntb = ntb_ndev(ndev);
- ntb->db &= ~db_bits;
+ atomic64_and(~db_bits, &ntb->db);
return 0;
}
diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c
index d54e18872aef..9a505c796370 100644
--- a/drivers/pci/endpoint/pci-epf-core.c
+++ b/drivers/pci/endpoint/pci-epf-core.c
@@ -208,6 +208,48 @@ void pci_epf_remove_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf)
}
EXPORT_SYMBOL_GPL(pci_epf_remove_vepf);
+static int pci_epf_get_required_bar_size(struct pci_epf *epf, size_t *bar_size,
+ size_t *aligned_mem_size,
+ enum pci_barno bar,
+ const struct pci_epc_features *epc_features,
+ enum pci_epc_interface_type type)
+{
+ u64 bar_fixed_size = epc_features->bar[bar].fixed_size;
+ size_t align = epc_features->align;
+ size_t size = *bar_size;
+
+ if (size < 128)
+ size = 128;
+
+ /* According to PCIe base spec, min size for a resizable BAR is 1 MB. */
+ if (epc_features->bar[bar].type == BAR_RESIZABLE && size < SZ_1M)
+ size = SZ_1M;
+
+ if (epc_features->bar[bar].type == BAR_FIXED && bar_fixed_size) {
+ if (size > bar_fixed_size) {
+ dev_err(&epf->dev,
+ "requested BAR size is larger than fixed size\n");
+ return -ENOMEM;
+ }
+ size = bar_fixed_size;
+ } else {
+ /* BAR size must be power of two */
+ size = roundup_pow_of_two(size);
+ }
+
+ *bar_size = size;
+
+ /*
+ * The EPC's BAR start address must meet alignment requirements. In most
+ * cases, the alignment will match the BAR size. However, differences
+ * can occur—for example, when the fixed BAR size (e.g., 128 bytes) is
+ * smaller than the required alignment (e.g., 4 KB).
+ */
+ *aligned_mem_size = align ? ALIGN(size, align) : size;
+
+ return 0;
+}
+
/**
* pci_epf_free_space() - free the allocated PCI EPF register space
* @epf: the EPF device from whom to free the memory
@@ -236,13 +278,13 @@ void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar,
}
dev = epc->dev.parent;
- dma_free_coherent(dev, epf_bar[bar].aligned_size, addr,
+ dma_free_coherent(dev, epf_bar[bar].mem_size, addr,
epf_bar[bar].phys_addr);
epf_bar[bar].phys_addr = 0;
epf_bar[bar].addr = NULL;
epf_bar[bar].size = 0;
- epf_bar[bar].aligned_size = 0;
+ epf_bar[bar].mem_size = 0;
epf_bar[bar].barno = 0;
epf_bar[bar].flags = 0;
}
@@ -264,40 +306,16 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
const struct pci_epc_features *epc_features,
enum pci_epc_interface_type type)
{
- u64 bar_fixed_size = epc_features->bar[bar].fixed_size;
- size_t aligned_size, align = epc_features->align;
struct pci_epf_bar *epf_bar;
dma_addr_t phys_addr;
struct pci_epc *epc;
struct device *dev;
+ size_t mem_size;
void *space;
- if (size < 128)
- size = 128;
-
- /* According to PCIe base spec, min size for a resizable BAR is 1 MB. */
- if (epc_features->bar[bar].type == BAR_RESIZABLE && size < SZ_1M)
- size = SZ_1M;
-
- if (epc_features->bar[bar].type == BAR_FIXED && bar_fixed_size) {
- if (size > bar_fixed_size) {
- dev_err(&epf->dev,
- "requested BAR size is larger than fixed size\n");
- return NULL;
- }
- size = bar_fixed_size;
- } else {
- /* BAR size must be power of two */
- size = roundup_pow_of_two(size);
- }
-
- /*
- * Allocate enough memory to accommodate the iATU alignment
- * requirement. In most cases, this will be the same as .size but
- * it might be different if, for example, the fixed size of a BAR
- * is smaller than align.
- */
- aligned_size = align ? ALIGN(size, align) : size;
+ if (pci_epf_get_required_bar_size(epf, &size, &mem_size, bar,
+ epc_features, type))
+ return NULL;
if (type == PRIMARY_INTERFACE) {
epc = epf->epc;
@@ -308,7 +326,7 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
}
dev = epc->dev.parent;
- space = dma_alloc_coherent(dev, aligned_size, &phys_addr, GFP_KERNEL);
+ space = dma_alloc_coherent(dev, mem_size, &phys_addr, GFP_KERNEL);
if (!space) {
dev_err(dev, "failed to allocate mem space\n");
return NULL;
@@ -317,7 +335,7 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
epf_bar[bar].phys_addr = phys_addr;
epf_bar[bar].addr = space;
epf_bar[bar].size = size;
- epf_bar[bar].aligned_size = aligned_size;
+ epf_bar[bar].mem_size = mem_size;
epf_bar[bar].barno = bar;
if (upper_32_bits(size) || epc_features->bar[bar].only_64bit)
epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;
@@ -328,6 +346,83 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
}
EXPORT_SYMBOL_GPL(pci_epf_alloc_space);
+/**
+ * pci_epf_assign_bar_space() - Assign PCI EPF BAR space
+ * @epf: EPF device to assign the BAR memory
+ * @size: Size of the memory that has to be assigned
+ * @bar: BAR number for which the memory is assigned
+ * @epc_features: Features provided by the EPC specific to this EPF
+ * @type: Identifies if the assignment is for primary EPC or secondary EPC
+ * @bar_addr: Address to be assigned for the @bar
+ *
+ * Invoke to assign memory for the PCI EPF BAR.
+ * Flag PCI_BASE_ADDRESS_MEM_TYPE_64 will automatically get set if the BAR
+ * can only be a 64-bit BAR, or if the requested size is larger than 2 GB.
+ */
+int pci_epf_assign_bar_space(struct pci_epf *epf, size_t size,
+ enum pci_barno bar,
+ const struct pci_epc_features *epc_features,
+ enum pci_epc_interface_type type,
+ dma_addr_t bar_addr)
+{
+ size_t bar_size, aligned_mem_size;
+ struct pci_epf_bar *epf_bar;
+ dma_addr_t limit;
+ int pos;
+
+ if (!size)
+ return -EINVAL;
+
+ limit = bar_addr + size - 1;
+
+ /*
+ * Bits: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
+ * bar_addr: U U U U U U 0 X X X X X X X X X
+ * limit: U U U U U U 1 X X X X X X X X X
+ *
+ * bar_addr^limit 0 0 0 0 0 0 1 X X X X X X X X X
+ *
+ * U: unchanged address bits in range [bar_addr, limit]
+ * X: bit 0 or 1
+ *
+ * (bar_addr^limit) & BIT_ULL(pos) will find the first set bit from MSB
+ * (pos). And value of (2 ^ pos) should be able to cover the BAR range.
+ */
+ for (pos = 8 * sizeof(dma_addr_t) - 1; pos > 0; pos--)
+ if ((limit ^ bar_addr) & BIT_ULL(pos))
+ break;
+
+ if (pos == 8 * sizeof(dma_addr_t) - 1)
+ return -EINVAL;
+
+ bar_size = BIT_ULL(pos + 1);
+ if (pci_epf_get_required_bar_size(epf, &bar_size, &aligned_mem_size,
+ bar, epc_features, type))
+ return -ENOMEM;
+
+ if (type == PRIMARY_INTERFACE)
+ epf_bar = epf->bar;
+ else
+ epf_bar = epf->sec_epc_bar;
+
+ epf_bar[bar].phys_addr = ALIGN_DOWN(bar_addr, aligned_mem_size);
+
+ if (epf_bar[bar].phys_addr + bar_size < limit)
+ return -ENOMEM;
+
+ epf_bar[bar].addr = NULL;
+ epf_bar[bar].size = bar_size;
+ epf_bar[bar].mem_size = aligned_mem_size;
+ epf_bar[bar].barno = bar;
+ if (upper_32_bits(size) || epc_features->bar[bar].only_64bit)
+ epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+ else
+ epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_32;
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(pci_epf_assign_bar_space);
+
static void pci_epf_remove_cfs(struct pci_epf_driver *driver)
{
struct config_group *group, *tmp;
generated by cgit (git 2.34.1) at 2025-12-05 15:54:30 +0000