diff options
Diffstat (limited to 'drivers/cxl')
| -rw-r--r-- | drivers/cxl/acpi.c | 73 | ||||
| -rw-r--r-- | drivers/cxl/core/cdat.c | 4 | ||||
| -rw-r--r-- | drivers/cxl/core/hdm.c | 3 | ||||
| -rw-r--r-- | drivers/cxl/core/pci.c | 87 | ||||
| -rw-r--r-- | drivers/cxl/core/port.c | 1 | ||||
| -rw-r--r-- | drivers/cxl/core/region.c | 311 | ||||
| -rw-r--r-- | drivers/cxl/cxl.h | 29 | ||||
| -rw-r--r-- | drivers/cxl/cxlpci.h | 1 | ||||
| -rw-r--r-- | drivers/cxl/pci.c | 2 |
9 files changed, 298 insertions, 213 deletions
diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index bd2e282ca93a..77ac940e3013 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -11,25 +11,36 @@ #include "cxlpci.h" #include "cxl.h" -struct cxl_cxims_data { - int nr_maps; - u64 xormaps[] __counted_by(nr_maps); -}; - static const guid_t acpi_cxl_qtg_id_guid = GUID_INIT(0xF365F9A6, 0xA7DE, 0x4071, 0xA6, 0x6A, 0xB4, 0x0C, 0x0B, 0x4F, 0x8E, 0x52); -static u64 cxl_apply_xor_maps(struct cxl_root_decoder *cxlrd, u64 addr) +#define HBIW_TO_NR_MAPS_SIZE (CXL_DECODER_MAX_INTERLEAVE + 1) +static const int hbiw_to_nr_maps[HBIW_TO_NR_MAPS_SIZE] = { + [1] = 0, [2] = 1, [3] = 0, [4] = 2, [6] = 1, [8] = 3, [12] = 2, [16] = 4 +}; + +static const int valid_hbiw[] = { 1, 2, 3, 4, 6, 8, 12, 16 }; + +u64 cxl_do_xormap_calc(struct cxl_cxims_data *cximsd, u64 addr, int hbiw) { - struct cxl_cxims_data *cximsd = cxlrd->platform_data; - int hbiw = cxlrd->cxlsd.nr_targets; + int nr_maps_to_apply = -1; u64 val; int pos; - /* No xormaps for host bridge interleave ways of 1 or 3 */ - if (hbiw == 1 || hbiw == 3) - return addr; + /* + * Strictly validate hbiw since this function is used for testing and + * that nullifies any expectation of trusted parameters from the CXL + * Region Driver. + */ + for (int i = 0; i < ARRAY_SIZE(valid_hbiw); i++) { + if (valid_hbiw[i] == hbiw) { + nr_maps_to_apply = hbiw_to_nr_maps[hbiw]; + break; + } + } + if (nr_maps_to_apply == -1 || nr_maps_to_apply > cximsd->nr_maps) + return ULLONG_MAX; /* * In regions using XOR interleave arithmetic the CXL HPA may not @@ -60,6 +71,14 @@ static u64 cxl_apply_xor_maps(struct cxl_root_decoder *cxlrd, u64 addr) return addr; } +EXPORT_SYMBOL_FOR_MODULES(cxl_do_xormap_calc, "cxl_translate"); + +static u64 cxl_apply_xor_maps(struct cxl_root_decoder *cxlrd, u64 addr) +{ + struct cxl_cxims_data *cximsd = cxlrd->platform_data; + + return cxl_do_xormap_calc(cximsd, addr, cxlrd->cxlsd.nr_targets); +} struct cxl_cxims_context { struct device *dev; @@ -353,7 +372,7 @@ static int cxl_acpi_set_cache_size(struct cxl_root_decoder *cxlrd) rc = hmat_get_extended_linear_cache_size(&res, nid, &cache_size); if (rc) - return rc; + return 0; /* * The cache range is expected to be within the CFMWS. @@ -378,21 +397,18 @@ static void cxl_setup_extended_linear_cache(struct cxl_root_decoder *cxlrd) int rc; rc = cxl_acpi_set_cache_size(cxlrd); - if (!rc) - return; - - if (rc != -EOPNOTSUPP) { + if (rc) { /* - * Failing to support extended linear cache region resize does not + * Failing to retrieve extended linear cache region resize does not * prevent the region from functioning. Only causes cxl list showing * incorrect region size. */ dev_warn(cxlrd->cxlsd.cxld.dev.parent, - "Extended linear cache calculation failed rc:%d\n", rc); - } + "Extended linear cache retrieval failed rc:%d\n", rc); - /* Ignoring return code */ - cxlrd->cache_size = 0; + /* Ignoring return code */ + cxlrd->cache_size = 0; + } } DEFINE_FREE(put_cxlrd, struct cxl_root_decoder *, @@ -453,8 +469,6 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, ig = CXL_DECODER_MIN_GRANULARITY; cxld->interleave_granularity = ig; - cxl_setup_extended_linear_cache(cxlrd); - if (cfmws->interleave_arithmetic == ACPI_CEDT_CFMWS_ARITHMETIC_XOR) { if (ways != 1 && ways != 3) { cxims_ctx = (struct cxl_cxims_context) { @@ -470,18 +484,13 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, return -EINVAL; } } + cxlrd->ops.hpa_to_spa = cxl_apply_xor_maps; + cxlrd->ops.spa_to_hpa = cxl_apply_xor_maps; } - cxlrd->qos_class = cfmws->qtg_id; - - if (cfmws->interleave_arithmetic == ACPI_CEDT_CFMWS_ARITHMETIC_XOR) { - cxlrd->ops = kzalloc(sizeof(*cxlrd->ops), GFP_KERNEL); - if (!cxlrd->ops) - return -ENOMEM; + cxl_setup_extended_linear_cache(cxlrd); - cxlrd->ops->hpa_to_spa = cxl_apply_xor_maps; - cxlrd->ops->spa_to_hpa = cxl_apply_xor_maps; - } + cxlrd->qos_class = cfmws->qtg_id; rc = cxl_decoder_add(cxld); if (rc) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index c4bd6e8a0cf0..7120b5f2e31f 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -826,7 +826,7 @@ static struct xarray *cxl_switch_gather_bandwidth(struct cxl_region *cxlr, cxl_coordinates_combine(coords, coords, ctx->coord); /* - * Take the min of the calculated bandwdith and the upstream + * Take the min of the calculated bandwidth and the upstream * switch SSLBIS bandwidth if there's a parent switch */ if (!is_root) @@ -949,7 +949,7 @@ static struct xarray *cxl_hb_gather_bandwidth(struct xarray *xa) /** * cxl_region_update_bandwidth - Update the bandwidth access coordinates of a region * @cxlr: The region being operated on - * @input_xa: xarray holds cxl_perf_ctx wht calculated bandwidth per ACPI0017 instance + * @input_xa: xarray holds cxl_perf_ctx with calculated bandwidth per ACPI0017 instance */ static void cxl_region_update_bandwidth(struct cxl_region *cxlr, struct xarray *input_xa) diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index d3a094ca01ad..1c5d2022c87a 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -905,6 +905,9 @@ static void cxl_decoder_reset(struct cxl_decoder *cxld) if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0) return; + if (test_bit(CXL_DECODER_F_LOCK, &cxld->flags)) + return; + if (port->commit_end == id) cxl_port_commit_reap(cxld); else diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 18825e1505d6..5b023a0178a4 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -71,85 +71,6 @@ struct cxl_dport *__devm_cxl_add_dport_by_dev(struct cxl_port *port, } EXPORT_SYMBOL_NS_GPL(__devm_cxl_add_dport_by_dev, "CXL"); -struct cxl_walk_context { - struct pci_bus *bus; - struct cxl_port *port; - int type; - int error; - int count; -}; - -static int match_add_dports(struct pci_dev *pdev, void *data) -{ - struct cxl_walk_context *ctx = data; - struct cxl_port *port = ctx->port; - int type = pci_pcie_type(pdev); - struct cxl_register_map map; - struct cxl_dport *dport; - u32 lnkcap, port_num; - int rc; - - if (pdev->bus != ctx->bus) - return 0; - if (!pci_is_pcie(pdev)) - return 0; - if (type != ctx->type) - return 0; - if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP, - &lnkcap)) - return 0; - - rc = cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map); - if (rc) - dev_dbg(&port->dev, "failed to find component registers\n"); - - port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap); - dport = devm_cxl_add_dport(port, &pdev->dev, port_num, map.resource); - if (IS_ERR(dport)) { - ctx->error = PTR_ERR(dport); - return PTR_ERR(dport); - } - ctx->count++; - - return 0; -} - -/** - * devm_cxl_port_enumerate_dports - enumerate downstream ports of the upstream port - * @port: cxl_port whose ->uport_dev is the upstream of dports to be enumerated - * - * Returns a positive number of dports enumerated or a negative error - * code. - */ -int devm_cxl_port_enumerate_dports(struct cxl_port *port) -{ - struct pci_bus *bus = cxl_port_to_pci_bus(port); - struct cxl_walk_context ctx; - int type; - - if (!bus) - return -ENXIO; - - if (pci_is_root_bus(bus)) - type = PCI_EXP_TYPE_ROOT_PORT; - else - type = PCI_EXP_TYPE_DOWNSTREAM; - - ctx = (struct cxl_walk_context) { - .port = port, - .bus = bus, - .type = type, - }; - pci_walk_bus(bus, match_add_dports, &ctx); - - if (ctx.count == 0) - return -ENODEV; - if (ctx.error) - return ctx.error; - return ctx.count; -} -EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, "CXL"); - static int cxl_dvsec_mem_range_valid(struct cxl_dev_state *cxlds, int id) { struct pci_dev *pdev = to_pci_dev(cxlds->dev); @@ -1217,6 +1138,14 @@ int cxl_gpf_port_setup(struct cxl_dport *dport) return 0; } +struct cxl_walk_context { + struct pci_bus *bus; + struct cxl_port *port; + int type; + int error; + int count; +}; + static int count_dports(struct pci_dev *pdev, void *data) { struct cxl_walk_context *ctx = data; diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 8128fd2b5b31..fef3aa0c6680 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -459,7 +459,6 @@ static void cxl_root_decoder_release(struct device *dev) if (atomic_read(&cxlrd->region_id) >= 0) memregion_free(atomic_read(&cxlrd->region_id)); __cxl_decoder_release(&cxlrd->cxlsd.cxld); - kfree(cxlrd->ops); kfree(cxlrd); } diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 41b64d871c5a..82d229c8f9bf 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -245,6 +245,9 @@ static void cxl_region_decode_reset(struct cxl_region *cxlr, int count) struct cxl_region_params *p = &cxlr->params; int i; + if (test_bit(CXL_REGION_F_LOCK, &cxlr->flags)) + return; + /* * Before region teardown attempt to flush, evict any data cached for * this region, or scream loudly about missing arch / platform support @@ -419,6 +422,9 @@ static ssize_t commit_store(struct device *dev, struct device_attribute *attr, return len; } + if (test_bit(CXL_REGION_F_LOCK, &cxlr->flags)) + return -EPERM; + rc = queue_reset(cxlr); if (rc) return rc; @@ -461,21 +467,6 @@ static ssize_t commit_show(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RW(commit); -static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a, - int n) -{ - struct device *dev = kobj_to_dev(kobj); - struct cxl_region *cxlr = to_cxl_region(dev); - - /* - * Support tooling that expects to find a 'uuid' attribute for all - * regions regardless of mode. - */ - if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_PARTMODE_PMEM) - return 0444; - return a->mode; -} - static ssize_t interleave_ways_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -754,6 +745,21 @@ static ssize_t size_show(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RW(size); +static ssize_t extended_linear_cache_size_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct cxl_region *cxlr = to_cxl_region(dev); + struct cxl_region_params *p = &cxlr->params; + ssize_t rc; + + ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region); + if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem))) + return rc; + return sysfs_emit(buf, "%#llx\n", p->cache_size); +} +static DEVICE_ATTR_RO(extended_linear_cache_size); + static struct attribute *cxl_region_attrs[] = { &dev_attr_uuid.attr, &dev_attr_commit.attr, @@ -762,9 +768,34 @@ static struct attribute *cxl_region_attrs[] = { &dev_attr_resource.attr, &dev_attr_size.attr, &dev_attr_mode.attr, + &dev_attr_extended_linear_cache_size.attr, NULL, }; +static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a, + int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct cxl_region *cxlr = to_cxl_region(dev); + + /* + * Support tooling that expects to find a 'uuid' attribute for all + * regions regardless of mode. + */ + if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_PARTMODE_PMEM) + return 0444; + + /* + * Don't display extended linear cache attribute if there is no + * extended linear cache. + */ + if (a == &dev_attr_extended_linear_cache_size.attr && + cxlr->params.cache_size == 0) + return 0; + + return a->mode; +} + static const struct attribute_group cxl_region_group = { .attrs = cxl_region_attrs, .is_visible = cxl_region_visible, @@ -838,16 +869,16 @@ static int match_free_decoder(struct device *dev, const void *data) return 1; } -static bool region_res_match_cxl_range(const struct cxl_region_params *p, - const struct range *range) +static bool spa_maps_hpa(const struct cxl_region_params *p, + const struct range *range) { if (!p->res) return false; /* - * If an extended linear cache region then the CXL range is assumed - * to be fronted by the DRAM range in current known implementation. - * This assumption will be made until a variant implementation exists. + * The extended linear cache region is constructed by a 1:1 ratio + * where the SPA maps equal amounts of DRAM and CXL HPA capacity with + * CXL decoders at the high end of the SPA range. */ return p->res->start + p->cache_size == range->start && p->res->end == range->end; @@ -865,7 +896,7 @@ static int match_auto_decoder(struct device *dev, const void *data) cxld = to_cxl_decoder(dev); r = &cxld->hpa_range; - if (region_res_match_cxl_range(p, r)) + if (spa_maps_hpa(p, r)) return 1; return 0; @@ -1059,6 +1090,16 @@ static int cxl_rr_assign_decoder(struct cxl_port *port, struct cxl_region *cxlr, return 0; } +static void cxl_region_set_lock(struct cxl_region *cxlr, + struct cxl_decoder *cxld) +{ + if (!test_bit(CXL_DECODER_F_LOCK, &cxld->flags)) + return; + + set_bit(CXL_REGION_F_LOCK, &cxlr->flags); + clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); +} + /** * cxl_port_attach_region() - track a region's interest in a port by endpoint * @port: port to add a new region reference 'struct cxl_region_ref' @@ -1170,6 +1211,8 @@ static int cxl_port_attach_region(struct cxl_port *port, } } + cxl_region_set_lock(cxlr, cxld); + rc = cxl_rr_ep_add(cxl_rr, cxled); if (rc) { dev_dbg(&cxlr->dev, @@ -1328,7 +1371,7 @@ static int cxl_port_setup_targets(struct cxl_port *port, struct cxl_endpoint_decoder *cxled) { struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); - int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos; + int parent_iw, parent_ig, ig, iw, rc, pos = cxled->pos; struct cxl_port *parent_port = to_cxl_port(port->dev.parent); struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); @@ -1465,7 +1508,7 @@ static int cxl_port_setup_targets(struct cxl_port *port, if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { if (cxld->interleave_ways != iw || (iw > 1 && cxld->interleave_granularity != ig) || - !region_res_match_cxl_range(p, &cxld->hpa_range) || + !spa_maps_hpa(p, &cxld->hpa_range) || ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) { dev_err(&cxlr->dev, "%s:%s %s expected iw: %d ig: %d %pr\n", @@ -1520,9 +1563,8 @@ add_target: cxlsd->target[cxl_rr->nr_targets_set] = ep->dport; cxlsd->cxld.target_map[cxl_rr->nr_targets_set] = ep->dport->port_id; } - inc = 1; + cxl_rr->nr_targets_set++; out_target_set: - cxl_rr->nr_targets_set += inc; dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n", dev_name(port->uport_dev), dev_name(&port->dev), cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev), @@ -2439,6 +2481,7 @@ static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int i dev->bus = &cxl_bus_type; dev->type = &cxl_region_type; cxlr->id = id; + cxl_region_set_lock(cxlr, &cxlrd->cxlsd.cxld); return cxlr; } @@ -2924,38 +2967,119 @@ static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos) return false; } -static bool has_hpa_to_spa(struct cxl_root_decoder *cxlrd) +#define CXL_POS_ZERO 0 +/** + * cxl_validate_translation_params + * @eiw: encoded interleave ways + * @eig: encoded interleave granularity + * @pos: position in interleave + * + * Callers pass CXL_POS_ZERO when no position parameter needs validating. + * + * Returns: 0 on success, -EINVAL on first invalid parameter + */ +int cxl_validate_translation_params(u8 eiw, u16 eig, int pos) { - return cxlrd->ops && cxlrd->ops->hpa_to_spa; + int ways, gran; + + if (eiw_to_ways(eiw, &ways)) { + pr_debug("%s: invalid eiw=%u\n", __func__, eiw); + return -EINVAL; + } + if (eig_to_granularity(eig, &gran)) { + pr_debug("%s: invalid eig=%u\n", __func__, eig); + return -EINVAL; + } + if (pos < 0 || pos >= ways) { + pr_debug("%s: invalid pos=%d for ways=%u\n", __func__, pos, + ways); + return -EINVAL; + } + + return 0; } +EXPORT_SYMBOL_FOR_MODULES(cxl_validate_translation_params, "cxl_translate"); -static bool has_spa_to_hpa(struct cxl_root_decoder *cxlrd) +u64 cxl_calculate_dpa_offset(u64 hpa_offset, u8 eiw, u16 eig) { - return cxlrd->ops && cxlrd->ops->spa_to_hpa; + u64 dpa_offset, bits_lower, bits_upper, temp; + int ret; + + ret = cxl_validate_translation_params(eiw, eig, CXL_POS_ZERO); + if (ret) + return ULLONG_MAX; + + /* + * DPA offset: CXL Spec 3.2 Section 8.2.4.20.13 + * Lower bits [IG+7:0] pass through unchanged + * (eiw < 8) + * Per spec: DPAOffset[51:IG+8] = (HPAOffset[51:IG+IW+8] >> IW) + * Clear the position bits to isolate upper section, then + * reverse the left shift by eiw that occurred during DPA->HPA + * (eiw >= 8) + * Per spec: DPAOffset[51:IG+8] = HPAOffset[51:IG+IW] / 3 + * Extract upper bits from the correct bit range and divide by 3 + * to recover the original DPA upper bits + */ + bits_lower = hpa_offset & GENMASK_ULL(eig + 7, 0); + if (eiw < 8) { + temp = hpa_offset &= ~GENMASK_ULL(eig + eiw + 8 - 1, 0); + dpa_offset = temp >> eiw; + } else { + bits_upper = div64_u64(hpa_offset >> (eig + eiw), 3); + dpa_offset = bits_upper << (eig + 8); + } + dpa_offset |= bits_lower; + + return dpa_offset; } +EXPORT_SYMBOL_FOR_MODULES(cxl_calculate_dpa_offset, "cxl_translate"); -u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, - u64 dpa) +int cxl_calculate_position(u64 hpa_offset, u8 eiw, u16 eig) { - struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); - u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa; - struct cxl_region_params *p = &cxlr->params; - struct cxl_endpoint_decoder *cxled = NULL; - u16 eig = 0; - u8 eiw = 0; - int pos; + unsigned int ways = 0; + u64 shifted, rem; + int pos, ret; - for (int i = 0; i < p->nr_targets; i++) { - cxled = p->targets[i]; - if (cxlmd == cxled_to_memdev(cxled)) - break; + ret = cxl_validate_translation_params(eiw, eig, CXL_POS_ZERO); + if (ret) + return ret; + + if (!eiw) + /* position is 0 if no interleaving */ + return 0; + + /* + * Interleave position: CXL Spec 3.2 Section 8.2.4.20.13 + * eiw < 8 + * Position is in the IW bits at HPA_OFFSET[IG+8+IW-1:IG+8]. + * Per spec "remove IW bits starting with bit position IG+8" + * eiw >= 8 + * Position is not explicitly stored in HPA_OFFSET bits. It is + * derived from the modulo operation of the upper bits using + * the total number of interleave ways. + */ + if (eiw < 8) { + pos = (hpa_offset >> (eig + 8)) & GENMASK(eiw - 1, 0); + } else { + shifted = hpa_offset >> (eig + 8); + eiw_to_ways(eiw, &ways); + div64_u64_rem(shifted, ways, &rem); + pos = rem; } - if (!cxled || cxlmd != cxled_to_memdev(cxled)) - return ULLONG_MAX; - pos = cxled->pos; - ways_to_eiw(p->interleave_ways, &eiw); - granularity_to_eig(p->interleave_granularity, &eig); + return pos; +} +EXPORT_SYMBOL_FOR_MODULES(cxl_calculate_position, "cxl_translate"); + +u64 cxl_calculate_hpa_offset(u64 dpa_offset, int pos, u8 eiw, u16 eig) +{ + u64 mask_upper, hpa_offset, bits_upper; + int ret; + + ret = cxl_validate_translation_params(eiw, eig, pos); + if (ret) + return ULLONG_MAX; /* * The device position in the region interleave set was removed @@ -2967,9 +3091,6 @@ u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, * 8.2.4.19.13 Implementation Note: Device Decode Logic */ - /* Remove the dpa base */ - dpa_offset = dpa - cxl_dpa_resource_start(cxled); - mask_upper = GENMASK_ULL(51, eig + 8); if (eiw < 8) { @@ -2984,12 +3105,43 @@ u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, /* The lower bits remain unchanged */ hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0); + return hpa_offset; +} +EXPORT_SYMBOL_FOR_MODULES(cxl_calculate_hpa_offset, "cxl_translate"); + +u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, + u64 dpa) +{ + struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); + struct cxl_region_params *p = &cxlr->params; + struct cxl_endpoint_decoder *cxled = NULL; + u64 dpa_offset, hpa_offset, hpa; + u16 eig = 0; + u8 eiw = 0; + int pos; + + for (int i = 0; i < p->nr_targets; i++) { + if (cxlmd == cxled_to_memdev(p->targets[i])) { + cxled = p->targets[i]; + break; + } + } + if (!cxled) + return ULLONG_MAX; + + pos = cxled->pos; + ways_to_eiw(p->interleave_ways, &eiw); + granularity_to_eig(p->interleave_granularity, &eig); + + dpa_offset = dpa - cxl_dpa_resource_start(cxled); + hpa_offset = cxl_calculate_hpa_offset(dpa_offset, pos, eiw, eig); + /* Apply the hpa_offset to the region base address */ hpa = hpa_offset + p->res->start + p->cache_size; /* Root decoder translation overrides typical modulo decode */ - if (has_hpa_to_spa(cxlrd)) - hpa = cxlrd->ops->hpa_to_spa(cxlrd, hpa); + if (cxlrd->ops.hpa_to_spa) + hpa = cxlrd->ops.hpa_to_spa(cxlrd, hpa); if (!cxl_resource_contains_addr(p->res, hpa)) { dev_dbg(&cxlr->dev, @@ -2998,7 +3150,7 @@ u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, } /* Simple chunk check, by pos & gran, only applies to modulo decodes */ - if (!has_hpa_to_spa(cxlrd) && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos))) + if (!cxlrd->ops.hpa_to_spa && !cxl_is_hpa_in_chunk(hpa, cxlr, pos)) return ULLONG_MAX; return hpa; @@ -3016,8 +3168,6 @@ static int region_offset_to_dpa_result(struct cxl_region *cxlr, u64 offset, struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); struct cxl_endpoint_decoder *cxled; u64 hpa, hpa_offset, dpa_offset; - u64 bits_upper, bits_lower; - u64 shifted, rem, temp; u16 eig = 0; u8 eiw = 0; int pos; @@ -3033,56 +3183,21 @@ static int region_offset_to_dpa_result(struct cxl_region *cxlr, u64 offset, * If the root decoder has SPA to CXL HPA callback, use it. Otherwise * CXL HPA is assumed to equal SPA. */ - if (has_spa_to_hpa(cxlrd)) { - hpa = cxlrd->ops->spa_to_hpa(cxlrd, p->res->start + offset); + if (cxlrd->ops.spa_to_hpa) { + hpa = cxlrd->ops.spa_to_hpa(cxlrd, p->res->start + offset); hpa_offset = hpa - p->res->start; } else { hpa_offset = offset; } - /* - * Interleave position: CXL Spec 3.2 Section 8.2.4.20.13 - * eiw < 8 - * Position is in the IW bits at HPA_OFFSET[IG+8+IW-1:IG+8]. - * Per spec "remove IW bits starting with bit position IG+8" - * eiw >= 8 - * Position is not explicitly stored in HPA_OFFSET bits. It is - * derived from the modulo operation of the upper bits using - * the total number of interleave ways. - */ - if (eiw < 8) { - pos = (hpa_offset >> (eig + 8)) & GENMASK(eiw - 1, 0); - } else { - shifted = hpa_offset >> (eig + 8); - div64_u64_rem(shifted, p->interleave_ways, &rem); - pos = rem; - } + + pos = cxl_calculate_position(hpa_offset, eiw, eig); if (pos < 0 || pos >= p->nr_targets) { dev_dbg(&cxlr->dev, "Invalid position %d for %d targets\n", pos, p->nr_targets); return -ENXIO; } - /* - * DPA offset: CXL Spec 3.2 Section 8.2.4.20.13 - * Lower bits [IG+7:0] pass through unchanged - * (eiw < 8) - * Per spec: DPAOffset[51:IG+8] = (HPAOffset[51:IG+IW+8] >> IW) - * Clear the position bits to isolate upper section, then - * reverse the left shift by eiw that occurred during DPA->HPA - * (eiw >= 8) - * Per spec: DPAOffset[51:IG+8] = HPAOffset[51:IG+IW] / 3 - * Extract upper bits from the correct bit range and divide by 3 - * to recover the original DPA upper bits - */ - bits_lower = hpa_offset & GENMASK_ULL(eig + 7, 0); - if (eiw < 8) { - temp = hpa_offset &= ~((u64)GENMASK(eig + eiw + 8 - 1, 0)); - dpa_offset = temp >> eiw; - } else { - bits_upper = div64_u64(hpa_offset >> (eig + eiw), 3); - dpa_offset = bits_upper << (eig + 8); - } - dpa_offset |= bits_lower; + dpa_offset = cxl_calculate_dpa_offset(hpa_offset, eiw, eig); /* Look-up and return the result: a memdev and a DPA */ for (int i = 0; i < p->nr_targets; i++) { @@ -3398,7 +3513,7 @@ static int match_region_by_range(struct device *dev, const void *data) p = &cxlr->params; guard(rwsem_read)(&cxl_rwsem.region); - return region_res_match_cxl_range(p, r); + return spa_maps_hpa(p, r); } static int cxl_extended_linear_cache_resize(struct cxl_region *cxlr, @@ -3479,6 +3594,10 @@ static int __construct_region(struct cxl_region *cxlr, "Extended linear cache calculation failed rc:%d\n", rc); } + rc = sysfs_update_group(&cxlr->dev.kobj, &cxl_region_group); + if (rc) + return rc; + rc = insert_resource(cxlrd->res, res); if (rc) { /* diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 231ddccf8977..ba17fa86d249 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -451,7 +451,7 @@ struct cxl_root_decoder { void *platform_data; struct mutex range_lock; int qos_class; - struct cxl_rd_ops *ops; + struct cxl_rd_ops ops; struct cxl_switch_decoder cxlsd; }; @@ -517,6 +517,14 @@ enum cxl_partition_mode { */ #define CXL_REGION_F_NEEDS_RESET 1 +/* + * Indicate whether this region is locked due to 1 or more decoders that have + * been locked. The approach of all or nothing is taken with regard to the + * locked attribute. CXL_REGION_F_NEEDS_RESET should not be set if this flag is + * set. + */ +#define CXL_REGION_F_LOCK 2 + /** * struct cxl_region - CXL region * @dev: This region's device @@ -738,6 +746,25 @@ static inline bool is_cxl_root(struct cxl_port *port) return port->uport_dev == port->dev.parent; } +/* Address translation functions exported to cxl_translate test module only */ +int cxl_validate_translation_params(u8 eiw, u16 eig, int pos); +u64 cxl_calculate_hpa_offset(u64 dpa_offset, int pos, u8 eiw, u16 eig); +u64 cxl_calculate_dpa_offset(u64 hpa_offset, u8 eiw, u16 eig); +int cxl_calculate_position(u64 hpa_offset, u8 eiw, u16 eig); +struct cxl_cxims_data { + int nr_maps; + u64 xormaps[] __counted_by(nr_maps); +}; + +#if IS_ENABLED(CONFIG_CXL_ACPI) +u64 cxl_do_xormap_calc(struct cxl_cxims_data *cximsd, u64 addr, int hbiw); +#else +static inline u64 cxl_do_xormap_calc(struct cxl_cxims_data *cximsd, u64 addr, int hbiw) +{ + return ULLONG_MAX; +} +#endif + int cxl_num_decoders_committed(struct cxl_port *port); bool is_cxl_port(const struct device *dev); struct cxl_port *to_cxl_port(const struct device *dev); diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h index 7ae621e618e7..1d526bea8431 100644 --- a/drivers/cxl/cxlpci.h +++ b/drivers/cxl/cxlpci.h @@ -127,7 +127,6 @@ static inline bool cxl_pci_flit_256(struct pci_dev *pdev) return lnksta2 & PCI_EXP_LNKSTA2_FLIT; } -int devm_cxl_port_enumerate_dports(struct cxl_port *port); struct cxl_dev_state; void read_cdat_data(struct cxl_port *port); void cxl_cor_error_detected(struct pci_dev *pdev); diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index bd100ac31672..0be4e508affe 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -136,7 +136,7 @@ static irqreturn_t cxl_pci_mbox_irq(int irq, void *id) if (opcode == CXL_MBOX_OP_SANITIZE) { mutex_lock(&cxl_mbox->mbox_mutex); if (mds->security.sanitize_node) - mod_delayed_work(system_wq, &mds->security.poll_dwork, 0); + mod_delayed_work(system_percpu_wq, &mds->security.poll_dwork, 0); mutex_unlock(&cxl_mbox->mbox_mutex); } else { /* short-circuit the wait in __cxl_pci_mbox_send_cmd() */ |