summaryrefslogtreecommitdiff
path: root/drivers/ufs
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/ufs')
-rw-r--r--drivers/ufs/core/Makefile1
-rw-r--r--drivers/ufs/core/ufs-mcq.c62
-rw-r--r--drivers/ufs/core/ufs-rpmb.c254
-rw-r--r--drivers/ufs/core/ufs-sysfs.c3
-rw-r--r--drivers/ufs/core/ufs_bsg.c2
-rw-r--r--drivers/ufs/core/ufs_trace.h1
-rw-r--r--drivers/ufs/core/ufs_trace_types.h1
-rw-r--r--drivers/ufs/core/ufshcd-crypto.h18
-rw-r--r--drivers/ufs/core/ufshcd-priv.h54
-rw-r--r--drivers/ufs/core/ufshcd.c934
-rw-r--r--drivers/ufs/host/Kconfig13
-rw-r--r--drivers/ufs/host/Makefile1
-rw-r--r--drivers/ufs/host/ti-j721e-ufs.c37
-rw-r--r--drivers/ufs/host/ufs-amd-versal2.c564
-rw-r--r--drivers/ufs/host/ufs-mediatek.c130
-rw-r--r--drivers/ufs/host/ufs-mediatek.h4
-rw-r--r--drivers/ufs/host/ufs-qcom.c3
-rw-r--r--drivers/ufs/host/ufs-rockchip.c20
-rw-r--r--drivers/ufs/host/ufshcd-dwc.h46
19 files changed, 1644 insertions, 504 deletions
diff --git a/drivers/ufs/core/Makefile b/drivers/ufs/core/Makefile
index cf820fa09a04..51e1867e524e 100644
--- a/drivers/ufs/core/Makefile
+++ b/drivers/ufs/core/Makefile
@@ -2,6 +2,7 @@
obj-$(CONFIG_SCSI_UFSHCD) += ufshcd-core.o
ufshcd-core-y += ufshcd.o ufs-sysfs.o ufs-mcq.o
+ufshcd-core-$(CONFIG_RPMB) += ufs-rpmb.o
ufshcd-core-$(CONFIG_DEBUG_FS) += ufs-debugfs.o
ufshcd-core-$(CONFIG_SCSI_UFS_BSG) += ufs_bsg.o
ufshcd-core-$(CONFIG_SCSI_UFS_CRYPTO) += ufshcd-crypto.o
diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c
index c9bdd4140fd0..9ab91b4c05b0 100644
--- a/drivers/ufs/core/ufs-mcq.c
+++ b/drivers/ufs/core/ufs-mcq.c
@@ -134,17 +134,15 @@ unsigned int ufshcd_mcq_queue_cfg_addr(struct ufs_hba *hba)
EXPORT_SYMBOL_GPL(ufshcd_mcq_queue_cfg_addr);
/**
- * ufshcd_mcq_decide_queue_depth - decide the queue depth
+ * ufshcd_get_hba_mac - Maximum number of commands supported by the host
+ * controller.
* @hba: per adapter instance
*
- * Return: queue-depth on success, non-zero on error
+ * Return: queue depth on success; negative upon error.
*
- * MAC - Max. Active Command of the Host Controller (HC)
- * HC wouldn't send more than this commands to the device.
- * Calculates and adjusts the queue depth based on the depth
- * supported by the HC and ufs device.
+ * MAC = Maximum number of Active Commands supported by the Host Controller.
*/
-int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba)
+int ufshcd_get_hba_mac(struct ufs_hba *hba)
{
int mac;
@@ -162,18 +160,7 @@ int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba)
mac = hba->vops->get_hba_mac(hba);
}
if (mac < 0)
- goto err;
-
- WARN_ON_ONCE(!hba->dev_info.bqueuedepth);
- /*
- * max. value of bqueuedepth = 256, mac is host dependent.
- * It is mandatory for UFS device to define bQueueDepth if
- * shared queuing architecture is enabled.
- */
- return min_t(int, mac, hba->dev_info.bqueuedepth);
-
-err:
- dev_err(hba->dev, "Failed to get mac, err=%d\n", mac);
+ dev_err(hba->dev, "Failed to get mac, err=%d\n", mac);
return mac;
}
@@ -307,9 +294,10 @@ static void ufshcd_mcq_process_cqe(struct ufs_hba *hba,
struct ufs_hw_queue *hwq)
{
struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq);
- int tag = ufshcd_mcq_get_tag(hba, cqe);
if (cqe->command_desc_base_addr) {
+ int tag = ufshcd_mcq_get_tag(hba, cqe);
+
ufshcd_compl_one_cqe(hba, tag, cqe);
/* After processed the cqe, mark it empty (invalid) entry */
cqe->command_desc_base_addr = 0;
@@ -491,9 +479,6 @@ int ufshcd_mcq_init(struct ufs_hba *hba)
mutex_init(&hwq->sq_mutex);
}
- /* The very first HW queue serves device commands */
- hba->dev_cmd_queue = &hba->uhq[0];
-
host->host_tagset = 1;
return 0;
}
@@ -546,8 +531,9 @@ static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
*/
int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)
{
- struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
- struct scsi_cmnd *cmd = lrbp->cmd;
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ struct request *rq = scsi_cmd_to_rq(cmd);
struct ufs_hw_queue *hwq;
void __iomem *reg, *opr_sqd_base;
u32 nexus, id, val;
@@ -556,24 +542,21 @@ int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)
if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
return -ETIMEDOUT;
- if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) {
- if (!cmd)
- return -EINVAL;
- hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
- if (!hwq)
- return 0;
- } else {
- hwq = hba->dev_cmd_queue;
- }
+ if (!cmd)
+ return -EINVAL;
+
+ hwq = ufshcd_mcq_req_to_hwq(hba, rq);
+ if (!hwq)
+ return 0;
id = hwq->id;
- mutex_lock(&hwq->sq_mutex);
+ guard(mutex)(&hwq->sq_mutex);
/* stop the SQ fetching before working on it */
err = ufshcd_mcq_sq_stop(hba, hwq);
if (err)
- goto unlock;
+ return err;
/* SQCTI = EXT_IID, IID, LUN, Task Tag */
nexus = lrbp->lun << 8 | task_tag;
@@ -600,8 +583,6 @@ int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)
if (ufshcd_mcq_sq_start(hba, hwq))
err = -ETIMEDOUT;
-unlock:
- mutex_unlock(&hwq->sq_mutex);
return err;
}
@@ -632,7 +613,8 @@ static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd)
static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,
struct ufs_hw_queue *hwq, int task_tag)
{
- struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct utp_transfer_req_desc *utrd;
__le64 cmd_desc_base_addr;
bool ret = false;
@@ -683,7 +665,7 @@ int ufshcd_mcq_abort(struct scsi_cmnd *cmd)
struct Scsi_Host *host = cmd->device->host;
struct ufs_hba *hba = shost_priv(host);
int tag = scsi_cmd_to_rq(cmd)->tag;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct ufs_hw_queue *hwq;
int err;
diff --git a/drivers/ufs/core/ufs-rpmb.c b/drivers/ufs/core/ufs-rpmb.c
new file mode 100644
index 000000000000..ffad049872b9
--- /dev/null
+++ b/drivers/ufs/core/ufs-rpmb.c
@@ -0,0 +1,254 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * UFS OP-TEE based RPMB Driver
+ *
+ * Copyright (C) 2025 Micron Technology, Inc.
+ * Copyright (C) 2025 Qualcomm Technologies, Inc.
+ *
+ * Authors:
+ * Bean Huo <beanhuo@micron.com>
+ * Can Guo <can.guo@oss.qualcomm.com>
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/rpmb.h>
+#include <linux/string.h>
+#include <linux/list.h>
+#include <ufs/ufshcd.h>
+#include <linux/unaligned.h>
+#include "ufshcd-priv.h"
+
+#define UFS_RPMB_SEC_PROTOCOL 0xEC /* JEDEC UFS application */
+#define UFS_RPMB_SEC_PROTOCOL_ID 0x01 /* JEDEC UFS RPMB protocol ID, CDB byte3 */
+
+static const struct bus_type ufs_rpmb_bus_type = {
+ .name = "ufs_rpmb",
+};
+
+/* UFS RPMB device structure */
+struct ufs_rpmb_dev {
+ u8 region_id;
+ struct device dev;
+ struct rpmb_dev *rdev;
+ struct ufs_hba *hba;
+ struct list_head node;
+};
+
+static int ufs_sec_submit(struct ufs_hba *hba, u16 spsp, void *buffer, size_t len, bool send)
+{
+ struct scsi_device *sdev = hba->ufs_rpmb_wlun;
+ u8 cdb[12] = { };
+
+ cdb[0] = send ? SECURITY_PROTOCOL_OUT : SECURITY_PROTOCOL_IN;
+ cdb[1] = UFS_RPMB_SEC_PROTOCOL;
+ put_unaligned_be16(spsp, &cdb[2]);
+ put_unaligned_be32(len, &cdb[6]);
+
+ return scsi_execute_cmd(sdev, cdb, send ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN,
+ buffer, len, /*timeout=*/30 * HZ, 0, NULL);
+}
+
+/* UFS RPMB route frames implementation */
+static int ufs_rpmb_route_frames(struct device *dev, u8 *req, unsigned int req_len, u8 *resp,
+ unsigned int resp_len)
+{
+ struct ufs_rpmb_dev *ufs_rpmb = dev_get_drvdata(dev);
+ struct rpmb_frame *frm_out = (struct rpmb_frame *)req;
+ bool need_result_read = true;
+ u16 req_type, protocol_id;
+ struct ufs_hba *hba;
+ int ret;
+
+ if (!ufs_rpmb) {
+ dev_err(dev, "Missing driver data\n");
+ return -ENODEV;
+ }
+
+ hba = ufs_rpmb->hba;
+
+ req_type = be16_to_cpu(frm_out->req_resp);
+
+ switch (req_type) {
+ case RPMB_PROGRAM_KEY:
+ if (req_len != sizeof(struct rpmb_frame) || resp_len != sizeof(struct rpmb_frame))
+ return -EINVAL;
+ break;
+ case RPMB_GET_WRITE_COUNTER:
+ if (req_len != sizeof(struct rpmb_frame) || resp_len != sizeof(struct rpmb_frame))
+ return -EINVAL;
+ need_result_read = false;
+ break;
+ case RPMB_WRITE_DATA:
+ if (req_len % sizeof(struct rpmb_frame) || resp_len != sizeof(struct rpmb_frame))
+ return -EINVAL;
+ break;
+ case RPMB_READ_DATA:
+ if (req_len != sizeof(struct rpmb_frame) || resp_len % sizeof(struct rpmb_frame))
+ return -EINVAL;
+ need_result_read = false;
+ break;
+ default:
+ dev_err(dev, "Unknown request type=0x%04x\n", req_type);
+ return -EINVAL;
+ }
+
+ protocol_id = ufs_rpmb->region_id << 8 | UFS_RPMB_SEC_PROTOCOL_ID;
+
+ ret = ufs_sec_submit(hba, protocol_id, req, req_len, true);
+ if (ret) {
+ dev_err(dev, "Command failed with ret=%d\n", ret);
+ return ret;
+ }
+
+ if (need_result_read) {
+ struct rpmb_frame *frm_resp = (struct rpmb_frame *)resp;
+
+ memset(frm_resp, 0, sizeof(*frm_resp));
+ frm_resp->req_resp = cpu_to_be16(RPMB_RESULT_READ);
+ ret = ufs_sec_submit(hba, protocol_id, resp, resp_len, true);
+ if (ret) {
+ dev_err(dev, "Result read request failed with ret=%d\n", ret);
+ return ret;
+ }
+ }
+
+ if (!ret) {
+ ret = ufs_sec_submit(hba, protocol_id, resp, resp_len, false);
+ if (ret)
+ dev_err(dev, "Response read failed with ret=%d\n", ret);
+ }
+
+ return ret;
+}
+
+static void ufs_rpmb_device_release(struct device *dev)
+{
+ struct ufs_rpmb_dev *ufs_rpmb = dev_get_drvdata(dev);
+
+ rpmb_dev_unregister(ufs_rpmb->rdev);
+}
+
+/* UFS RPMB device registration */
+int ufs_rpmb_probe(struct ufs_hba *hba)
+{
+ struct ufs_rpmb_dev *ufs_rpmb, *it, *tmp;
+ struct rpmb_dev *rdev;
+ char *cid = NULL;
+ int region;
+ u32 cap;
+ int ret;
+
+ if (!hba->ufs_rpmb_wlun || hba->dev_info.b_advanced_rpmb_en) {
+ dev_info(hba->dev, "Skip OP-TEE RPMB registration\n");
+ return -ENODEV;
+ }
+
+ /* Check if device_id is available */
+ if (!hba->dev_info.device_id) {
+ dev_err(hba->dev, "UFS Device ID not available\n");
+ return -EINVAL;
+ }
+
+ INIT_LIST_HEAD(&hba->rpmbs);
+
+ struct rpmb_descr descr = {
+ .type = RPMB_TYPE_UFS,
+ .route_frames = ufs_rpmb_route_frames,
+ .reliable_wr_count = hba->dev_info.rpmb_io_size,
+ };
+
+ for (region = 0; region < ARRAY_SIZE(hba->dev_info.rpmb_region_size); region++) {
+ cap = hba->dev_info.rpmb_region_size[region];
+ if (!cap)
+ continue;
+
+ ufs_rpmb = devm_kzalloc(hba->dev, sizeof(*ufs_rpmb), GFP_KERNEL);
+ if (!ufs_rpmb) {
+ ret = -ENOMEM;
+ goto err_out;
+ }
+
+ ufs_rpmb->hba = hba;
+ ufs_rpmb->dev.parent = &hba->ufs_rpmb_wlun->sdev_gendev;
+ ufs_rpmb->dev.bus = &ufs_rpmb_bus_type;
+ ufs_rpmb->dev.release = ufs_rpmb_device_release;
+ dev_set_name(&ufs_rpmb->dev, "ufs_rpmb%d", region);
+
+ /* Set driver data BEFORE device_register */
+ dev_set_drvdata(&ufs_rpmb->dev, ufs_rpmb);
+
+ ret = device_register(&ufs_rpmb->dev);
+ if (ret) {
+ dev_err(hba->dev, "Failed to register UFS RPMB device %d\n", region);
+ put_device(&ufs_rpmb->dev);
+ goto err_out;
+ }
+
+ /* Create unique ID by appending region number to device_id */
+ cid = kasprintf(GFP_KERNEL, "%s-R%d", hba->dev_info.device_id, region);
+ if (!cid) {
+ device_unregister(&ufs_rpmb->dev);
+ ret = -ENOMEM;
+ goto err_out;
+ }
+
+ descr.dev_id = cid;
+ descr.dev_id_len = strlen(cid);
+ descr.capacity = cap;
+
+ /* Register RPMB device */
+ rdev = rpmb_dev_register(&ufs_rpmb->dev, &descr);
+ if (IS_ERR(rdev)) {
+ dev_err(hba->dev, "Failed to register UFS RPMB device.\n");
+ device_unregister(&ufs_rpmb->dev);
+ ret = PTR_ERR(rdev);
+ goto err_out;
+ }
+
+ kfree(cid);
+ cid = NULL;
+
+ ufs_rpmb->rdev = rdev;
+ ufs_rpmb->region_id = region;
+
+ list_add_tail(&ufs_rpmb->node, &hba->rpmbs);
+
+ dev_info(hba->dev, "UFS RPMB region %d registered (capacity=%u)\n", region, cap);
+ }
+
+ return 0;
+err_out:
+ kfree(cid);
+ list_for_each_entry_safe(it, tmp, &hba->rpmbs, node) {
+ list_del(&it->node);
+ device_unregister(&it->dev);
+ }
+
+ return ret;
+}
+
+/* UFS RPMB remove handler */
+void ufs_rpmb_remove(struct ufs_hba *hba)
+{
+ struct ufs_rpmb_dev *ufs_rpmb, *tmp;
+
+ if (list_empty(&hba->rpmbs))
+ return;
+
+ /* Remove all registered RPMB devices */
+ list_for_each_entry_safe(ufs_rpmb, tmp, &hba->rpmbs, node) {
+ dev_info(hba->dev, "Removing UFS RPMB region %d\n", ufs_rpmb->region_id);
+ /* Remove from list first */
+ list_del(&ufs_rpmb->node);
+ /* Unregister device */
+ device_unregister(&ufs_rpmb->dev);
+ }
+
+ dev_info(hba->dev, "All UFS RPMB devices unregistered\n");
+}
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("OP-TEE UFS RPMB driver");
diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c
index 0086816b27cd..b33f8656edb5 100644
--- a/drivers/ufs/core/ufs-sysfs.c
+++ b/drivers/ufs/core/ufs-sysfs.c
@@ -235,7 +235,7 @@ static int ufshcd_ahit_to_us(u32 ahit)
}
/* Convert microseconds to Auto-Hibernate Idle Timer register value */
-static u32 ufshcd_us_to_ahit(unsigned int timer)
+u32 ufshcd_us_to_ahit(unsigned int timer)
{
unsigned int scale;
@@ -245,6 +245,7 @@ static u32 ufshcd_us_to_ahit(unsigned int timer)
return FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, timer) |
FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, scale);
}
+EXPORT_SYMBOL_GPL(ufshcd_us_to_ahit);
static int ufshcd_read_hci_reg(struct ufs_hba *hba, u32 *val, unsigned int reg)
{
diff --git a/drivers/ufs/core/ufs_bsg.c b/drivers/ufs/core/ufs_bsg.c
index 252186124669..58b506eac6dc 100644
--- a/drivers/ufs/core/ufs_bsg.c
+++ b/drivers/ufs/core/ufs_bsg.c
@@ -105,7 +105,7 @@ static int ufs_bsg_exec_advanced_rpmb_req(struct ufs_hba *hba, struct bsg_job *j
if (dir != DMA_NONE) {
payload = &job->request_payload;
- if (!payload || !payload->payload_len || !payload->sg_cnt)
+ if (!payload->payload_len || !payload->sg_cnt)
return -EINVAL;
sg_cnt = dma_map_sg(hba->host->dma_dev, payload->sg_list, payload->sg_cnt, dir);
diff --git a/drivers/ufs/core/ufs_trace.h b/drivers/ufs/core/ufs_trace.h
index 584c2b5c6ad9..309ae51b4906 100644
--- a/drivers/ufs/core/ufs_trace.h
+++ b/drivers/ufs/core/ufs_trace.h
@@ -42,7 +42,6 @@
#define UFS_CMD_TRACE_STRINGS \
EM(UFS_CMD_SEND, "send_req") \
EM(UFS_CMD_COMP, "complete_rsp") \
- EM(UFS_DEV_COMP, "dev_complete") \
EM(UFS_QUERY_SEND, "query_send") \
EM(UFS_QUERY_COMP, "query_complete") \
EM(UFS_QUERY_ERR, "query_complete_err") \
diff --git a/drivers/ufs/core/ufs_trace_types.h b/drivers/ufs/core/ufs_trace_types.h
index f2d5ad1d92b9..bf821970f092 100644
--- a/drivers/ufs/core/ufs_trace_types.h
+++ b/drivers/ufs/core/ufs_trace_types.h
@@ -5,7 +5,6 @@
enum ufs_trace_str_t {
UFS_CMD_SEND,
UFS_CMD_COMP,
- UFS_DEV_COMP,
UFS_QUERY_SEND,
UFS_QUERY_COMP,
UFS_QUERY_ERR,
diff --git a/drivers/ufs/core/ufshcd-crypto.h b/drivers/ufs/core/ufshcd-crypto.h
index 89bb97c14c15..c148a5194378 100644
--- a/drivers/ufs/core/ufshcd-crypto.h
+++ b/drivers/ufs/core/ufshcd-crypto.h
@@ -38,10 +38,10 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp,
}
static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp)
+ struct scsi_cmnd *cmd)
{
- struct scsi_cmnd *cmd = lrbp->cmd;
const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt)
return hba->vops->fill_crypto_prdt(hba, crypt_ctx,
@@ -51,17 +51,19 @@ static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
}
static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp)
+ struct scsi_cmnd *cmd)
{
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT))
return;
- if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx))
+ if (!(scsi_cmd_to_rq(cmd)->crypt_ctx))
return;
/* Zeroize the PRDT because it can contain cryptographic keys. */
memzero_explicit(lrbp->ucd_prdt_ptr,
- ufshcd_sg_entry_size(hba) * scsi_sg_count(lrbp->cmd));
+ ufshcd_sg_entry_size(hba) * scsi_sg_count(cmd));
}
bool ufshcd_crypto_enable(struct ufs_hba *hba);
@@ -82,13 +84,15 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp,
struct request_desc_header *h) { }
static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp)
+ struct scsi_cmnd *cmd)
{
return 0;
}
static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp) { }
+ struct scsi_cmnd *cmd)
+{
+}
static inline bool ufshcd_crypto_enable(struct ufs_hba *hba)
{
diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h
index d0a2c963a27d..4259f499382f 100644
--- a/drivers/ufs/core/ufshcd-priv.h
+++ b/drivers/ufs/core/ufshcd-priv.h
@@ -6,6 +6,8 @@
#include <linux/pm_runtime.h>
#include <ufs/ufshcd.h>
+void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs);
+
static inline bool ufshcd_is_user_access_allowed(struct ufs_hba *hba)
{
return !hba->shutting_down;
@@ -65,7 +67,7 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
struct cq_entry *cqe);
int ufshcd_mcq_init(struct ufs_hba *hba);
void ufshcd_mcq_disable(struct ufs_hba *hba);
-int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba);
+int ufshcd_get_hba_mac(struct ufs_hba *hba);
int ufshcd_mcq_memory_alloc(struct ufs_hba *hba);
struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba,
struct request *req);
@@ -75,14 +77,19 @@ bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd);
int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag);
int ufshcd_mcq_abort(struct scsi_cmnd *cmd);
int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
-void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp);
+void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd);
-#define SD_ASCII_STD true
-#define SD_RAW false
-int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
- u8 **buf, bool ascii);
+/**
+ * enum ufs_descr_fmt - UFS string descriptor format
+ * @SD_RAW: Raw UTF-16 format
+ * @SD_ASCII_STD: Convert to null-terminated ASCII string
+ */
+enum ufs_descr_fmt {
+ SD_RAW = 0,
+ SD_ASCII_STD = 1,
+};
+int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum ufs_descr_fmt fmt);
int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd);
int ufshcd_send_bsg_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd);
@@ -361,6 +368,26 @@ static inline bool ufs_is_valid_unit_desc_lun(struct ufs_dev_info *dev_info, u8
return lun == UFS_UPIU_RPMB_WLUN || (lun < dev_info->max_lu_supported);
}
+/*
+ * Convert a block layer tag into a SCSI command pointer. This function is
+ * called once per I/O completion path and is also called from error paths.
+ */
+static inline struct scsi_cmnd *ufshcd_tag_to_cmd(struct ufs_hba *hba, u32 tag)
+{
+ /*
+ * Host-wide tags are enabled in MCQ mode only. See also the
+ * host->host_tagset assignment in ufs-mcq.c.
+ */
+ struct blk_mq_tags *tags = hba->host->tag_set.shared_tags ?:
+ hba->host->tag_set.tags[0];
+ struct request *rq = blk_mq_tag_to_rq(tags, tag);
+
+ if (WARN_ON_ONCE(!rq))
+ return NULL;
+
+ return blk_mq_rq_to_pdu(rq);
+}
+
static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q)
__must_hold(&q->sq_lock)
{
@@ -411,4 +438,17 @@ static inline u32 ufshcd_mcq_get_sq_head_slot(struct ufs_hw_queue *q)
return val / sizeof(struct utp_transfer_req_desc);
}
+#if IS_ENABLED(CONFIG_RPMB)
+int ufs_rpmb_probe(struct ufs_hba *hba);
+void ufs_rpmb_remove(struct ufs_hba *hba);
+#else
+static inline int ufs_rpmb_probe(struct ufs_hba *hba)
+{
+ return 0;
+}
+static inline void ufs_rpmb_remove(struct ufs_hba *hba)
+{
+}
+#endif
+
#endif /* _UFSHCD_PRIV_H_ */
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index d6a060a72461..040a0ceb170a 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -28,6 +28,7 @@
#include <scsi/scsi_dbg.h>
#include <scsi/scsi_driver.h>
#include <scsi/scsi_eh.h>
+#include <scsi/scsi_tcq.h>
#include "ufshcd-priv.h"
#include <ufs/ufs_quirks.h>
#include <ufs/unipro.h>
@@ -403,10 +404,11 @@ static void ufshcd_configure_wb(struct ufs_hba *hba)
ufshcd_wb_toggle_buf_flush(hba, true);
}
-static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
+static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba,
+ struct ufshcd_lrb *lrb,
enum ufs_trace_str_t str_t)
{
- struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
+ struct utp_upiu_req *rq = lrb->ucd_req_ptr;
struct utp_upiu_header *header;
if (!trace_ufshcd_upiu_enabled())
@@ -415,7 +417,7 @@ static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
if (str_t == UFS_CMD_SEND)
header = &rq->header;
else
- header = &hba->lrb[tag].ucd_rsp_ptr->header;
+ header = &lrb->ucd_rsp_ptr->header;
trace_ufshcd_upiu(hba, str_t, header, &rq->sc.cdb,
UFS_TSF_CDB);
@@ -472,7 +474,7 @@ static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
}
-static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
+static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd,
enum ufs_trace_str_t str_t)
{
u64 lba = 0;
@@ -480,16 +482,13 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
u32 doorbell = 0;
u32 intr;
u32 hwq_id = 0;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
- struct scsi_cmnd *cmd = lrbp->cmd;
struct request *rq = scsi_cmd_to_rq(cmd);
+ unsigned int tag = rq->tag;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int transfer_len = -1;
- if (!cmd)
- return;
-
/* trace UPIU also */
- ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
+ ufshcd_add_cmd_upiu_trace(hba, lrbp, str_t);
if (!trace_ufshcd_command_enabled())
return;
@@ -503,7 +502,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
lba = scsi_get_lba(cmd);
if (opcode == WRITE_10)
- group_id = lrbp->cmd->cmnd[6];
+ group_id = cmd->cmnd[6];
} else if (opcode == UNMAP) {
/*
* The number of Bytes to be unmapped beginning with the lba.
@@ -596,14 +595,13 @@ static void ufshcd_print_evt_hist(struct ufs_hba *hba)
ufshcd_vops_dbg_register_dump(hba);
}
-static
-void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt)
+static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ bool pr_prdt)
{
- const struct ufshcd_lrb *lrbp;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ const int tag = scsi_cmd_to_rq(cmd)->tag;
int prdt_length;
- lrbp = &hba->lrb[tag];
-
if (hba->monitor.enabled) {
dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag,
div_u64(lrbp->issue_time_stamp_local_clock, 1000));
@@ -646,7 +644,8 @@ static bool ufshcd_print_tr_iter(struct request *req, void *priv)
struct Scsi_Host *shost = sdev->host;
struct ufs_hba *hba = shost_priv(shost);
- ufshcd_print_tr(hba, req->tag, *(bool *)priv);
+ if (!blk_mq_is_reserved_rq(req))
+ ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv);
return true;
}
@@ -856,7 +855,7 @@ static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp,
struct cq_entry *cqe)
{
if (cqe)
- return le32_to_cpu(cqe->status) & MASK_OCS;
+ return cqe->overall_status & MASK_OCS;
return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS;
}
@@ -1076,7 +1075,7 @@ void ufshcd_pm_qos_exit(struct ufs_hba *hba)
* @hba: per adapter instance
* @on: If True, vote for perf PM QoS mode otherwise power save mode
*/
-static void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on)
+void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on)
{
guard(mutex)(&hba->pm_qos_mutex);
@@ -1085,6 +1084,7 @@ static void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on)
cpu_latency_qos_update_request(&hba->pm_qos_req, on ? 0 : PM_QOS_DEFAULT_VALUE);
}
+EXPORT_SYMBOL_GPL(ufshcd_pm_qos_update);
/**
* ufshcd_set_clk_freq - set UFS controller clock frequencies
@@ -1290,13 +1290,13 @@ static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
*/
static u32 ufshcd_pending_cmds(struct ufs_hba *hba)
{
- const struct scsi_device *sdev;
+ struct scsi_device *sdev;
unsigned long flags;
u32 pending = 0;
spin_lock_irqsave(hba->host->host_lock, flags);
__shost_for_each_device(sdev, hba->host)
- pending += sbitmap_weight(&sdev->budget_map);
+ pending += scsi_device_busy(sdev);
spin_unlock_irqrestore(hba->host->host_lock, flags);
return pending;
@@ -2294,20 +2294,21 @@ static inline int ufshcd_monitor_opcode2dir(u8 opcode)
return -EINVAL;
}
+/* Must only be called for SCSI commands. */
static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp)
+ struct scsi_cmnd *cmd)
{
const struct ufs_hba_monitor *m = &hba->monitor;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
- return (m->enabled && lrbp && lrbp->cmd &&
- (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
- ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
+ return m->enabled &&
+ (!m->chunk_size || m->chunk_size == cmd->sdb.length) &&
+ ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp);
}
-static void ufshcd_start_monitor(struct ufs_hba *hba,
- const struct ufshcd_lrb *lrbp)
+static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
- int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
+ int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]);
unsigned long flags;
spin_lock_irqsave(hba->host->host_lock, flags);
@@ -2316,14 +2317,15 @@ static void ufshcd_start_monitor(struct ufs_hba *hba,
spin_unlock_irqrestore(hba->host->host_lock, flags);
}
-static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp)
+static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
- int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
+ struct request *req = scsi_cmd_to_rq(cmd);
+ const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]);
unsigned long flags;
spin_lock_irqsave(hba->host->host_lock, flags);
if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
- const struct request *req = scsi_cmd_to_rq(lrbp->cmd);
struct ufs_hba_monitor *m = &hba->monitor;
ktime_t now, inc, lat;
@@ -2348,17 +2350,24 @@ static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *
spin_unlock_irqrestore(hba->host->host_lock, flags);
}
+/* Returns %true for SCSI commands and %false for device management commands. */
+static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd)
+{
+ return !blk_mq_is_reserved_rq(scsi_cmd_to_rq(cmd));
+}
+
/**
* ufshcd_send_command - Send SCSI or device management commands
* @hba: per adapter instance
- * @task_tag: Task tag of the command
+ * @cmd: SCSI command or device management command pointer
* @hwq: pointer to hardware queue instance
*/
-static inline
-void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag,
- struct ufs_hw_queue *hwq)
+static inline void ufshcd_send_command(struct ufs_hba *hba,
+ struct scsi_cmnd *cmd,
+ struct ufs_hw_queue *hwq)
{
- struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ const int tag = scsi_cmd_to_rq(cmd)->tag;
unsigned long flags;
if (hba->monitor.enabled) {
@@ -2367,11 +2376,12 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag,
lrbp->compl_time_stamp = ktime_set(0, 0);
lrbp->compl_time_stamp_local_clock = 0;
}
- ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
- if (lrbp->cmd)
+ if (ufshcd_is_scsi_cmd(cmd)) {
+ ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND);
ufshcd_clk_scaling_start_busy(hba);
- if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
- ufshcd_start_monitor(hba, lrbp);
+ if (unlikely(ufshcd_should_inform_monitor(hba, cmd)))
+ ufshcd_start_monitor(hba, cmd);
+ }
if (hba->mcq_enabled) {
int utrd_size = sizeof(struct utp_transfer_req_desc);
@@ -2386,22 +2396,22 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag,
} else {
spin_lock_irqsave(&hba->outstanding_lock, flags);
if (hba->vops && hba->vops->setup_xfer_req)
- hba->vops->setup_xfer_req(hba, lrbp->task_tag,
- !!lrbp->cmd);
- __set_bit(lrbp->task_tag, &hba->outstanding_reqs);
- ufshcd_writel(hba, 1 << lrbp->task_tag,
- REG_UTP_TRANSFER_REQ_DOOR_BELL);
+ hba->vops->setup_xfer_req(hba, tag,
+ ufshcd_is_scsi_cmd(cmd));
+ __set_bit(tag, &hba->outstanding_reqs);
+ ufshcd_writel(hba, 1 << tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
spin_unlock_irqrestore(&hba->outstanding_lock, flags);
}
}
/**
* ufshcd_copy_sense_data - Copy sense data in case of check condition
- * @lrbp: pointer to local reference block
+ * @cmd: SCSI command
*/
-static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
+static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd)
{
- u8 *const sense_buffer = lrbp->cmd->sense_buffer;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ u8 *const sense_buffer = cmd->sense_buffer;
u16 resp_len;
int len;
@@ -2474,7 +2484,6 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1;
hba->nutmrs =
((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
- hba->reserved_slot = hba->nutrs - 1;
hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1;
@@ -2618,7 +2627,7 @@ __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
init_completion(&uic_cmd->done);
- uic_cmd->cmd_active = 1;
+ uic_cmd->cmd_active = true;
ufshcd_dispatch_uic_cmd(hba, uic_cmd);
return 0;
@@ -2706,13 +2715,13 @@ static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int
/**
* ufshcd_map_sg - Map scatter-gather list to prdt
* @hba: per adapter instance
- * @lrbp: pointer to local reference block
+ * @cmd: SCSI command
*
* Return: 0 in case of success, non-zero value in case of failure.
*/
-static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
- struct scsi_cmnd *cmd = lrbp->cmd;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int sg_segments = scsi_dma_map(cmd);
if (sg_segments < 0)
@@ -2720,7 +2729,7 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd));
- return ufshcd_crypto_fill_prdt(hba, lrbp);
+ return ufshcd_crypto_fill_prdt(hba, cmd);
}
/**
@@ -2779,13 +2788,14 @@ ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
/**
* ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
* for scsi commands
- * @lrbp: local reference block pointer
+ * @cmd: SCSI command
* @upiu_flags: flags
*/
-static
-void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
+static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd,
+ u8 upiu_flags)
{
- struct scsi_cmnd *cmd = lrbp->cmd;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ const int tag = scsi_cmd_to_rq(cmd)->tag;
struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
unsigned short cdb_len;
@@ -2793,11 +2803,11 @@ void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
.transaction_code = UPIU_TRANSACTION_COMMAND,
.flags = upiu_flags,
.lun = lrbp->lun,
- .task_tag = lrbp->task_tag,
+ .task_tag = tag,
.command_set_type = UPIU_COMMAND_SET_TYPE_SCSI,
};
- WARN_ON_ONCE(ucd_req_ptr->header.task_tag != lrbp->task_tag);
+ WARN_ON_ONCE(ucd_req_ptr->header.task_tag != tag);
ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
@@ -2810,13 +2820,15 @@ void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
/**
* ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request
* @hba: UFS hba
- * @lrbp: local reference block pointer
+ * @cmd: SCSI command pointer
* @upiu_flags: flags
*/
static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp, u8 upiu_flags)
+ struct scsi_cmnd *cmd, u8 upiu_flags)
{
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
+ const int tag = scsi_cmd_to_rq(cmd)->tag;
struct ufs_query *query = &hba->dev_cmd.query;
u16 len = be16_to_cpu(query->request.upiu_req.length);
@@ -2825,7 +2837,7 @@ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
.transaction_code = UPIU_TRANSACTION_QUERY_REQ,
.flags = upiu_flags,
.lun = lrbp->lun,
- .task_tag = lrbp->task_tag,
+ .task_tag = tag,
.query_function = query->request.query_func,
/* Data segment length only need for WRITE_DESC */
.data_segment_length =
@@ -2844,15 +2856,17 @@ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
memcpy(ucd_req_ptr + 1, query->descriptor, len);
}
-static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
+static inline void ufshcd_prepare_utp_nop_upiu(struct scsi_cmnd *cmd)
{
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
+ const int tag = scsi_cmd_to_rq(cmd)->tag;
memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
ucd_req_ptr->header = (struct utp_upiu_header){
.transaction_code = UPIU_TRANSACTION_NOP_OUT,
- .task_tag = lrbp->task_tag,
+ .task_tag = tag,
};
}
@@ -2860,22 +2874,23 @@ static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
* ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
* for Device Management Purposes
* @hba: per adapter instance
- * @lrbp: pointer to local reference block
+ * @cmd: SCSI command pointer
*
* Return: 0 upon success; < 0 upon failure.
*/
static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp)
+ struct scsi_cmnd *cmd)
{
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
u8 upiu_flags;
int ret = 0;
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
- ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
+ ufshcd_prepare_utp_query_req_upiu(hba, cmd, upiu_flags);
else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
- ufshcd_prepare_utp_nop_upiu(lrbp);
+ ufshcd_prepare_utp_nop_upiu(cmd);
else
ret = -EINVAL;
@@ -2888,38 +2903,69 @@ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
* ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
* for SCSI Purposes
* @hba: per adapter instance
- * @lrbp: pointer to local reference block
+ * @cmd: SCSI command
*/
-static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
- struct request *rq = scsi_cmd_to_rq(lrbp->cmd);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ struct request *rq = scsi_cmd_to_rq(cmd);
unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
u8 upiu_flags;
- ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0);
+ ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags,
+ cmd->sc_data_direction, 0);
if (ioprio_class == IOPRIO_CLASS_RT)
upiu_flags |= UPIU_CMD_FLAGS_CP;
- ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
+ ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags);
}
-static void __ufshcd_setup_cmd(struct ufshcd_lrb *lrbp, struct scsi_cmnd *cmd, u8 lun, int tag)
+static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
+ const int i = scsi_cmd_to_rq(cmd)->tag;
+ struct utp_transfer_cmd_desc *cmd_descp =
+ (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba);
+ struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
+ dma_addr_t cmd_desc_element_addr =
+ hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba);
+ u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset);
+ u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset);
+ struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd);
+
+ lrb->utr_descriptor_ptr = utrdlp + i;
+ lrb->utrd_dma_addr =
+ hba->utrdl_dma_addr + i * sizeof(struct utp_transfer_req_desc);
+ lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu;
+ lrb->ucd_req_dma_addr = cmd_desc_element_addr;
+ lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu;
+ lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
+ lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table;
+ lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
+}
+
+static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ u8 lun, int tag)
+{
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
+ ufshcd_init_lrb(hba, cmd);
+
memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr));
- lrbp->cmd = cmd;
- lrbp->task_tag = tag;
lrbp->lun = lun;
- ufshcd_prepare_lrbp_crypto(cmd ? scsi_cmd_to_rq(cmd) : NULL, lrbp);
+ ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ?
+ scsi_cmd_to_rq(cmd) : NULL, lrbp);
}
-static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
- struct scsi_cmnd *cmd, u8 lun, int tag)
+static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ u8 lun, int tag)
{
- __ufshcd_setup_cmd(lrbp, cmd, lun, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
+ __ufshcd_setup_cmd(hba, cmd, lun, tag);
lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
lrbp->req_abort_skip = false;
- ufshcd_comp_scsi_upiu(hba, lrbp);
+ ufshcd_comp_scsi_upiu(hba, cmd);
}
/**
@@ -2970,25 +3016,13 @@ static void ufshcd_map_queues(struct Scsi_Host *shost)
}
}
-static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
+/*
+ * The only purpose of this function is to make the SCSI core skip the memset()
+ * call for the private command data.
+ */
+static int ufshcd_init_cmd_priv(struct Scsi_Host *host, struct scsi_cmnd *cmd)
{
- struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr +
- i * ufshcd_get_ucd_size(hba);
- struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
- dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
- i * ufshcd_get_ucd_size(hba);
- u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset);
- u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset);
-
- lrb->utr_descriptor_ptr = utrdlp + i;
- lrb->utrd_dma_addr = hba->utrdl_dma_addr +
- i * sizeof(struct utp_transfer_req_desc);
- lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu;
- lrb->ucd_req_dma_addr = cmd_desc_element_addr;
- lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu;
- lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
- lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table;
- lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
+ return 0;
}
/**
@@ -3002,7 +3036,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
{
struct ufs_hba *hba = shost_priv(host);
int tag = scsi_cmd_to_rq(cmd)->tag;
- struct ufshcd_lrb *lrbp;
int err = 0;
struct ufs_hw_queue *hwq = NULL;
@@ -3053,11 +3086,10 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
ufshcd_hold(hba);
- lrbp = &hba->lrb[tag];
+ ufshcd_setup_scsi_cmd(hba, cmd,
+ ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag);
- ufshcd_setup_scsi_cmd(hba, lrbp, cmd, ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag);
-
- err = ufshcd_map_sg(hba, lrbp);
+ err = ufshcd_map_sg(hba, cmd);
if (err) {
ufshcd_release(hba);
goto out;
@@ -3066,7 +3098,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
if (hba->mcq_enabled)
hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
- ufshcd_send_command(hba, tag, hwq);
+ ufshcd_send_command(hba, cmd, hwq);
out:
if (ufs_trigger_eh(hba)) {
@@ -3080,10 +3112,26 @@ out:
return err;
}
-static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
- enum dev_cmd_type cmd_type, u8 lun, int tag)
+static int ufshcd_queue_reserved_command(struct Scsi_Host *host,
+ struct scsi_cmnd *cmd)
+{
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ struct request *rq = scsi_cmd_to_rq(cmd);
+ struct ufs_hba *hba = shost_priv(host);
+ struct ufs_hw_queue *hwq =
+ hba->mcq_enabled ? ufshcd_mcq_req_to_hwq(hba, rq) : NULL;
+
+ ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
+ ufshcd_send_command(hba, cmd, hwq);
+ return 0;
+}
+
+static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ enum dev_cmd_type cmd_type, u8 lun, int tag)
{
- __ufshcd_setup_cmd(lrbp, NULL, lun, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
+ __ufshcd_setup_cmd(hba, cmd, lun, tag);
lrbp->intr_cmd = true; /* No interrupt aggregation */
hba->dev_cmd.type = cmd_type;
}
@@ -3091,12 +3139,12 @@ static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
/*
* Return: 0 upon success; < 0 upon failure.
*/
-static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
+static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ enum dev_cmd_type cmd_type, int tag)
{
- ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
+ ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag);
- return ufshcd_compose_devman_upiu(hba, lrbp);
+ return ufshcd_compose_devman_upiu(hba, cmd);
}
/*
@@ -3207,87 +3255,6 @@ ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
return err;
}
-/*
- * Return: 0 upon success; > 0 in case the UFS device reported an OCS error;
- * < 0 if another error occurred.
- */
-static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp, int max_timeout)
-{
- unsigned long time_left = msecs_to_jiffies(max_timeout);
- unsigned long flags;
- bool pending;
- int err;
-
-retry:
- time_left = wait_for_completion_timeout(&hba->dev_cmd.complete,
- time_left);
-
- if (likely(time_left)) {
- err = ufshcd_get_tr_ocs(lrbp, NULL);
- if (!err)
- err = ufshcd_dev_cmd_completion(hba, lrbp);
- } else {
- err = -ETIMEDOUT;
- dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
- __func__, lrbp->task_tag);
-
- /* MCQ mode */
- if (hba->mcq_enabled) {
- /* successfully cleared the command, retry if needed */
- if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0)
- err = -EAGAIN;
- return err;
- }
-
- /* SDB mode */
- if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) {
- /* successfully cleared the command, retry if needed */
- err = -EAGAIN;
- /*
- * Since clearing the command succeeded we also need to
- * clear the task tag bit from the outstanding_reqs
- * variable.
- */
- spin_lock_irqsave(&hba->outstanding_lock, flags);
- pending = test_bit(lrbp->task_tag,
- &hba->outstanding_reqs);
- if (pending)
- __clear_bit(lrbp->task_tag,
- &hba->outstanding_reqs);
- spin_unlock_irqrestore(&hba->outstanding_lock, flags);
-
- if (!pending) {
- /*
- * The completion handler ran while we tried to
- * clear the command.
- */
- time_left = 1;
- goto retry;
- }
- } else {
- dev_err(hba->dev, "%s: failed to clear tag %d\n",
- __func__, lrbp->task_tag);
-
- spin_lock_irqsave(&hba->outstanding_lock, flags);
- pending = test_bit(lrbp->task_tag,
- &hba->outstanding_reqs);
- spin_unlock_irqrestore(&hba->outstanding_lock, flags);
-
- if (!pending) {
- /*
- * The completion handler ran while we tried to
- * clear the command.
- */
- time_left = 1;
- goto retry;
- }
- }
- }
-
- return err;
-}
-
static void ufshcd_dev_man_lock(struct ufs_hba *hba)
{
ufshcd_hold(hba);
@@ -3302,23 +3269,40 @@ static void ufshcd_dev_man_unlock(struct ufs_hba *hba)
ufshcd_release(hba);
}
+static struct scsi_cmnd *ufshcd_get_dev_mgmt_cmd(struct ufs_hba *hba)
+{
+ /*
+ * The caller must hold this lock to guarantee that the NOWAIT
+ * allocation will succeed.
+ */
+ lockdep_assert_held(&hba->dev_cmd.lock);
+
+ return scsi_get_internal_cmd(
+ hba->host->pseudo_sdev, DMA_TO_DEVICE,
+ BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
+}
+
+static void ufshcd_put_dev_mgmt_cmd(struct scsi_cmnd *cmd)
+{
+ scsi_put_internal_cmd(cmd);
+}
+
/*
* Return: 0 upon success; > 0 in case the UFS device reported an OCS error;
* < 0 if another error occurred.
*/
-static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
- const u32 tag, int timeout)
+static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ const u32 tag, int timeout)
{
- int err;
-
- ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
- ufshcd_send_command(hba, tag, hba->dev_cmd_queue);
- err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
-
- ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
- (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ struct request *rq = scsi_cmd_to_rq(cmd);
+ blk_status_t sts;
- return err;
+ rq->timeout = timeout;
+ sts = blk_execute_rq(rq, true);
+ if (sts != BLK_STS_OK)
+ return blk_status_to_errno(sts);
+ return lrbp->utr_descriptor_ptr->header.ocs;
}
/**
@@ -3336,18 +3320,31 @@ static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
enum dev_cmd_type cmd_type, int timeout)
{
- const u32 tag = hba->reserved_slot;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ u32 tag;
int err;
- /* Protects use of hba->reserved_slot. */
+ /* Protects use of hba->dev_cmd. */
lockdep_assert_held(&hba->dev_cmd.lock);
- err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
+ if (WARN_ON_ONCE(!cmd))
+ return -ENOMEM;
+
+ tag = scsi_cmd_to_rq(cmd)->tag;
+
+ err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag);
if (unlikely(err))
- return err;
+ goto out;
+
+ err = ufshcd_issue_dev_cmd(hba, cmd, tag, timeout);
+ if (err == 0)
+ err = ufshcd_dev_cmd_completion(hba, lrbp);
+
+out:
+ ufshcd_put_dev_mgmt_cmd(cmd);
- return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout);
+ return err;
}
/**
@@ -3773,16 +3770,14 @@ static inline char ufshcd_remove_non_printable(u8 ch)
* @desc_index: descriptor index
* @buf: pointer to buffer where descriptor would be read,
* the caller should free the memory.
- * @ascii: if true convert from unicode to ascii characters
- * null terminated string.
+ * @fmt: if %SD_ASCII_STD, convert from UTF-16 to ASCII
*
* Return:
* * string size on success.
* * -ENOMEM: on allocation failure
* * -EINVAL: on a wrong parameter
*/
-int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
- u8 **buf, bool ascii)
+int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum ufs_descr_fmt fmt)
{
struct uc_string_id *uc_str;
u8 *str;
@@ -3811,7 +3806,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
goto out;
}
- if (ascii) {
+ if (fmt == SD_ASCII_STD) {
ssize_t ascii_len;
int i;
/* remove header and divide by 2 to move from UTF16 to UTF8 */
@@ -3837,7 +3832,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
str[ret++] = '\0';
} else {
- str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
+ str = kmemdup(uc_str->uc, uc_str->len, GFP_KERNEL);
if (!str) {
ret = -ENOMEM;
goto out;
@@ -3977,14 +3972,6 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba)
}
skip_utmrdl:
- /* Allocate memory for local reference block */
- hba->lrb = devm_kcalloc(hba->dev,
- hba->nutrs, sizeof(struct ufshcd_lrb),
- GFP_KERNEL);
- if (!hba->lrb) {
- dev_err(hba->dev, "LRB Memory allocation failed\n");
- goto out;
- }
return 0;
out:
return -ENOMEM;
@@ -4046,8 +4033,6 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba)
utrdlp[i].response_upiu_length =
cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
}
-
- ufshcd_init_lrb(hba, &hba->lrb[i], i);
}
}
@@ -5253,10 +5238,15 @@ static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev)
desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP)
hba->dev_info.is_lu_power_on_wp = true;
- /* In case of RPMB LU, check if advanced RPMB mode is enabled */
- if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN &&
- desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4))
- hba->dev_info.b_advanced_rpmb_en = true;
+ /* In case of RPMB LU, check if advanced RPMB mode is enabled, and get region size */
+ if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN) {
+ if (desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4))
+ hba->dev_info.b_advanced_rpmb_en = true;
+ hba->dev_info.rpmb_region_size[0] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION0_SIZE];
+ hba->dev_info.rpmb_region_size[1] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION1_SIZE];
+ hba->dev_info.rpmb_region_size[2] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION2_SIZE];
+ hba->dev_info.rpmb_region_size[3] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION3_SIZE];
+ }
kfree(desc_buf);
@@ -5396,19 +5386,18 @@ static void ufshcd_sdev_destroy(struct scsi_device *sdev)
/**
* ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
- * @lrbp: pointer to local reference block of completed command
+ * @cmd: SCSI command
* @scsi_status: SCSI command status
*
* Return: value base on SCSI command status.
*/
-static inline int
-ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
+static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status)
{
int result = 0;
switch (scsi_status) {
case SAM_STAT_CHECK_CONDITION:
- ufshcd_copy_sense_data(lrbp);
+ ufshcd_copy_sense_data(cmd);
fallthrough;
case SAM_STAT_GOOD:
result |= DID_OK << 16 | scsi_status;
@@ -5416,7 +5405,7 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
case SAM_STAT_TASK_SET_FULL:
case SAM_STAT_BUSY:
case SAM_STAT_TASK_ABORTED:
- ufshcd_copy_sense_data(lrbp);
+ ufshcd_copy_sense_data(cmd);
result |= scsi_status;
break;
default:
@@ -5430,15 +5419,17 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
/**
* ufshcd_transfer_rsp_status - Get overall status of the response
* @hba: per adapter instance
- * @lrbp: pointer to local reference block of completed command
+ * @cmd: SCSI command
* @cqe: pointer to the completion queue entry
*
* Return: result of the command to notify SCSI midlayer.
*/
-static inline int
-ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
- struct cq_entry *cqe)
+static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba,
+ struct scsi_cmnd *cmd,
+ struct cq_entry *cqe)
{
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ const int tag = scsi_cmd_to_rq(cmd)->tag;
int result = 0;
int scsi_status;
enum utp_ocs ocs;
@@ -5452,7 +5443,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
* not set either flag.
*/
if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW))
- scsi_set_resid(lrbp->cmd, resid);
+ scsi_set_resid(cmd, resid);
/* overall command status of utrd */
ocs = ufshcd_get_tr_ocs(lrbp, cqe);
@@ -5473,7 +5464,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
* to notify the SCSI midlayer of the command status
*/
scsi_status = lrbp->ucd_rsp_ptr->header.status;
- result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
+ result = ufshcd_scsi_cmd_status(cmd, scsi_status);
/*
* Currently we are only supporting BKOPs exception
@@ -5510,10 +5501,8 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
case OCS_ABORTED:
case OCS_INVALID_COMMAND_STATUS:
result |= DID_REQUEUE << 16;
- dev_warn(hba->dev,
- "OCS %s from controller for tag %d\n",
- (ocs == OCS_ABORTED ? "aborted" : "invalid"),
- lrbp->task_tag);
+ dev_warn(hba->dev, "OCS %s from controller for tag %d\n",
+ ocs == OCS_ABORTED ? "aborted" : "invalid", tag);
break;
case OCS_INVALID_CMD_TABLE_ATTR:
case OCS_INVALID_PRDT_ATTR:
@@ -5526,17 +5515,19 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
case OCS_GENERAL_CRYPTO_ERROR:
default:
result |= DID_ERROR << 16;
- dev_err(hba->dev,
- "OCS error from controller = %x for tag %d\n",
- ocs, lrbp->task_tag);
+ dev_err(hba->dev, "OCS error from controller = %x for tag %d\n",
+ ocs, tag);
ufshcd_print_evt_hist(hba);
ufshcd_print_host_state(hba);
break;
} /* end of switch */
if ((host_byte(result) != DID_OK) &&
- (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
- ufshcd_print_tr(hba, lrbp->task_tag, true);
+ (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) {
+ if (cqe)
+ ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry));
+ ufshcd_print_tr(hba, cmd, true);
+ }
return result;
}
@@ -5575,7 +5566,7 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
guard(spinlock_irqsave)(hba->host->host_lock);
cmd = hba->active_uic_cmd;
if (!cmd)
- goto unlock;
+ return retval;
if (ufshcd_is_auto_hibern8_error(hba, intr_status))
hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
@@ -5584,13 +5575,13 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
cmd->argument2 |= ufshcd_get_uic_cmd_result(hba);
cmd->argument3 = ufshcd_get_dme_attr_val(hba);
if (!hba->uic_async_done)
- cmd->cmd_active = 0;
+ cmd->cmd_active = false;
complete(&cmd->done);
retval = IRQ_HANDLED;
}
if (intr_status & UFSHCD_UIC_PWR_MASK && hba->uic_async_done) {
- cmd->cmd_active = 0;
+ cmd->cmd_active = false;
complete(hba->uic_async_done);
retval = IRQ_HANDLED;
}
@@ -5598,18 +5589,14 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
if (retval == IRQ_HANDLED)
ufshcd_add_uic_command_trace(hba, cmd, UFS_CMD_COMP);
-unlock:
return retval;
}
/* Release the resources allocated for processing a SCSI command. */
-void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp)
+void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
- struct scsi_cmnd *cmd = lrbp->cmd;
-
scsi_dma_unmap(cmd);
- ufshcd_crypto_clear_prdt(hba, lrbp);
+ ufshcd_crypto_clear_prdt(hba, cmd);
ufshcd_release(hba);
ufshcd_clk_scaling_update_busy(hba);
}
@@ -5623,31 +5610,39 @@ void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
struct cq_entry *cqe)
{
- struct ufshcd_lrb *lrbp;
- struct scsi_cmnd *cmd;
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
enum utp_ocs ocs;
- lrbp = &hba->lrb[task_tag];
+ if (WARN_ONCE(!cmd, "cqe->command_desc_base_addr = %#llx\n",
+ le64_to_cpu(cqe->command_desc_base_addr)))
+ return;
+
if (hba->monitor.enabled) {
lrbp->compl_time_stamp = ktime_get();
lrbp->compl_time_stamp_local_clock = local_clock();
}
- cmd = lrbp->cmd;
- if (cmd) {
- if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
- ufshcd_update_monitor(hba, lrbp);
- ufshcd_add_command_trace(hba, task_tag, UFS_CMD_COMP);
- cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe);
- ufshcd_release_scsi_cmd(hba, lrbp);
- /* Do not touch lrbp after scsi done */
- scsi_done(cmd);
+ if (ufshcd_is_scsi_cmd(cmd)) {
+ if (unlikely(ufshcd_should_inform_monitor(hba, cmd)))
+ ufshcd_update_monitor(hba, cmd);
+ ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP);
+ cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe);
+ ufshcd_release_scsi_cmd(hba, cmd);
} else {
if (cqe) {
- ocs = le32_to_cpu(cqe->status) & MASK_OCS;
+ ocs = cqe->overall_status & MASK_OCS;
lrbp->utr_descriptor_ptr->header.ocs = ocs;
+ } else {
+ ocs = lrbp->utr_descriptor_ptr->header.ocs;
}
- complete(&hba->dev_cmd.complete);
+ ufshcd_add_query_upiu_trace(
+ hba,
+ ocs == OCS_SUCCESS ? UFS_QUERY_COMP : UFS_QUERY_ERR,
+ (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
+ cmd->result = 0;
}
+ /* Do not touch lrbp after scsi_done() has been called. */
+ scsi_done(cmd);
}
/**
@@ -5675,7 +5670,7 @@ static void ufshcd_clear_polled(struct ufs_hba *hba,
int tag;
for_each_set_bit(tag, completed_reqs, hba->nutrs) {
- struct scsi_cmnd *cmd = hba->lrb[tag].cmd;
+ struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag);
if (!cmd)
continue;
@@ -5720,6 +5715,47 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
return completed_reqs != 0;
}
+static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv)
+{
+ struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
+ struct scsi_device *sdev = rq->q->queuedata;
+ struct Scsi_Host *shost = sdev->host;
+ struct ufs_hba *hba = shost_priv(shost);
+ struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq);
+
+ if (blk_mq_is_reserved_rq(rq) || !hwq)
+ return true;
+
+ ufshcd_mcq_compl_all_cqes_lock(hba, hwq);
+
+ /*
+ * For those cmds of which the cqes are not present in the cq, complete
+ * them explicitly.
+ */
+ scoped_guard(spinlock_irqsave, &hwq->cq_lock) {
+ if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
+ set_host_byte(cmd, DID_REQUEUE);
+ ufshcd_release_scsi_cmd(hba, cmd);
+ scsi_done(cmd);
+ }
+ }
+
+ return true;
+}
+
+static bool ufshcd_mcq_compl_one(struct request *rq, void *priv)
+{
+ struct scsi_device *sdev = rq->q->queuedata;
+ struct Scsi_Host *shost = sdev->host;
+ struct ufs_hba *hba = shost_priv(shost);
+ struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq);
+
+ if (!blk_mq_is_reserved_rq(rq) && hwq)
+ ufshcd_mcq_poll_cqe_lock(hba, hwq);
+
+ return true;
+}
+
/**
* ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is
* invoked from the error handler context or ufshcd_host_reset_and_restore()
@@ -5734,40 +5770,10 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba,
bool force_compl)
{
- struct ufs_hw_queue *hwq;
- struct ufshcd_lrb *lrbp;
- struct scsi_cmnd *cmd;
- unsigned long flags;
- int tag;
-
- for (tag = 0; tag < hba->nutrs; tag++) {
- lrbp = &hba->lrb[tag];
- cmd = lrbp->cmd;
- if (!ufshcd_cmd_inflight(cmd) ||
- test_bit(SCMD_STATE_COMPLETE, &cmd->state))
- continue;
-
- hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
- if (!hwq)
- continue;
-
- if (force_compl) {
- ufshcd_mcq_compl_all_cqes_lock(hba, hwq);
- /*
- * For those cmds of which the cqes are not present
- * in the cq, complete them explicitly.
- */
- spin_lock_irqsave(&hwq->cq_lock, flags);
- if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
- set_host_byte(cmd, DID_REQUEUE);
- ufshcd_release_scsi_cmd(hba, lrbp);
- scsi_done(cmd);
- }
- spin_unlock_irqrestore(&hwq->cq_lock, flags);
- } else {
- ufshcd_mcq_poll_cqe_lock(hba, hwq);
- }
- }
+ blk_mq_tagset_busy_iter(&hba->host->tag_set,
+ force_compl ? ufshcd_mcq_force_compl_one :
+ ufshcd_mcq_compl_one,
+ NULL);
}
/**
@@ -6612,9 +6618,12 @@ static bool ufshcd_abort_one(struct request *rq, void *priv)
struct Scsi_Host *shost = sdev->host;
struct ufs_hba *hba = shost_priv(shost);
+ if (blk_mq_is_reserved_rq(rq))
+ return true;
+
*ret = ufshcd_try_to_abort_task(hba, tag);
dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
- hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
+ ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1,
*ret ? "failed" : "succeeded");
return *ret == 0;
@@ -7349,15 +7358,21 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
enum dev_cmd_type cmd_type,
enum query_opcode desc_op)
{
- const u32 tag = hba->reserved_slot;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ u32 tag;
int err = 0;
u8 upiu_flags;
- /* Protects use of hba->reserved_slot. */
+ /* Protects use of hba->dev_cmd. */
lockdep_assert_held(&hba->dev_cmd.lock);
- ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
+ if (WARN_ON_ONCE(!cmd))
+ return -ENOMEM;
+
+ tag = scsi_cmd_to_rq(cmd)->tag;
+
+ ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag);
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
@@ -7377,12 +7392,9 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
- /*
- * ignore the returning value here - ufshcd_check_query_response is
- * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
- * read the response directly ignoring all errors.
- */
- ufshcd_issue_dev_cmd(hba, lrbp, tag, dev_cmd_timeout);
+ err = ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout);
+ if (err)
+ goto put_dev_mgmt_cmd;
/* just copy the upiu response as it is */
memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
@@ -7403,6 +7415,9 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
}
}
+put_dev_mgmt_cmd:
+ ufshcd_put_dev_mgmt_cmd(cmd);
+
return err;
}
@@ -7496,8 +7511,9 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list,
enum dma_data_direction dir)
{
- const u32 tag = hba->reserved_slot;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct scsi_cmnd *cmd;
+ struct ufshcd_lrb *lrbp;
+ u32 tag;
int err = 0;
int result;
u8 upiu_flags;
@@ -7505,10 +7521,20 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
u16 ehs_len;
int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0;
- /* Protects use of hba->reserved_slot. */
ufshcd_dev_man_lock(hba);
- ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag);
+ cmd = ufshcd_get_dev_mgmt_cmd(hba);
+
+ if (WARN_ON_ONCE(!cmd)) {
+ err = -ENOMEM;
+ goto unlock;
+ }
+
+ lrbp = scsi_cmd_priv(cmd);
+ tag = scsi_cmd_to_rq(cmd)->tag;
+
+ ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN,
+ tag);
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs);
@@ -7525,8 +7551,11 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
- err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT);
+ err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT);
+ if (err)
+ goto put_dev_mgmt_cmd;
+ err = ufshcd_dev_cmd_completion(hba, lrbp);
if (!err) {
/* Just copy the upiu response as it is */
memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
@@ -7550,11 +7579,45 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
}
}
+put_dev_mgmt_cmd:
+ ufshcd_put_dev_mgmt_cmd(cmd);
+
+unlock:
ufshcd_dev_man_unlock(hba);
return err ? : result;
}
+static bool ufshcd_clear_lu_cmds(struct request *req, void *priv)
+{
+ struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
+ struct scsi_device *sdev = cmd->device;
+ struct Scsi_Host *shost = sdev->host;
+ struct ufs_hba *hba = shost_priv(shost);
+ const u64 lun = *(u64 *)priv;
+ const u32 tag = req->tag;
+
+ if (blk_mq_is_reserved_rq(req) || sdev->lun != lun)
+ return true;
+
+ if (ufshcd_clear_cmd(hba, tag) < 0) {
+ dev_err(hba->dev, "%s: failed to clear request %d\n", __func__,
+ tag);
+ return true;
+ }
+
+ if (hba->mcq_enabled) {
+ struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, req);
+
+ if (hwq)
+ ufshcd_mcq_poll_cqe_lock(hba, hwq);
+ return true;
+ }
+
+ ufshcd_compl_one_cqe(hba, tag, NULL);
+ return true;
+}
+
/**
* ufshcd_eh_device_reset_handler() - Reset a single logical unit.
* @cmd: SCSI command pointer
@@ -7563,12 +7626,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
*/
static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
{
- unsigned long flags, pending_reqs = 0, not_cleared = 0;
struct Scsi_Host *host;
struct ufs_hba *hba;
- struct ufs_hw_queue *hwq;
- struct ufshcd_lrb *lrbp;
- u32 pos, not_cleared_mask = 0;
int err;
u8 resp = 0xF, lun;
@@ -7577,50 +7636,16 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
- if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
- if (!err)
- err = resp;
- goto out;
- }
-
- if (hba->mcq_enabled) {
- for (pos = 0; pos < hba->nutrs; pos++) {
- lrbp = &hba->lrb[pos];
- if (ufshcd_cmd_inflight(lrbp->cmd) &&
- lrbp->lun == lun) {
- ufshcd_clear_cmd(hba, pos);
- hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd));
- ufshcd_mcq_poll_cqe_lock(hba, hwq);
- }
- }
- err = 0;
- goto out;
- }
-
- /* clear the commands that were pending for corresponding LUN */
- spin_lock_irqsave(&hba->outstanding_lock, flags);
- for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs)
- if (hba->lrb[pos].lun == lun)
- __set_bit(pos, &pending_reqs);
- hba->outstanding_reqs &= ~pending_reqs;
- spin_unlock_irqrestore(&hba->outstanding_lock, flags);
-
- for_each_set_bit(pos, &pending_reqs, hba->nutrs) {
- if (ufshcd_clear_cmd(hba, pos) < 0) {
- spin_lock_irqsave(&hba->outstanding_lock, flags);
- not_cleared = 1U << pos &
- ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
- hba->outstanding_reqs |= not_cleared;
- not_cleared_mask |= not_cleared;
- spin_unlock_irqrestore(&hba->outstanding_lock, flags);
-
- dev_err(hba->dev, "%s: failed to clear request %d\n",
- __func__, pos);
- }
+ if (err) {
+ } else if (resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
+ err = resp;
+ } else {
+ /* clear the commands that were pending for corresponding LUN */
+ blk_mq_tagset_busy_iter(&hba->host->tag_set,
+ ufshcd_clear_lu_cmds,
+ &cmd->device->lun);
}
- __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared_mask);
-out:
hba->req_abort_count = 0;
ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err);
if (!err) {
@@ -7634,11 +7659,12 @@ out:
static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
{
- struct ufshcd_lrb *lrbp;
int tag;
for_each_set_bit(tag, &bitmap, hba->nutrs) {
- lrbp = &hba->lrb[tag];
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
lrbp->req_abort_skip = true;
}
}
@@ -7646,7 +7672,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
/**
* ufshcd_try_to_abort_task - abort a specific task
* @hba: Pointer to adapter instance
- * @tag: Task tag/index to be aborted
+ * @tag: Tag of the task to be aborted
*
* Abort the pending command in device by sending UFS_ABORT_TASK task management
* command, and in host controller by clearing the door-bell register. There can
@@ -7658,14 +7684,15 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
*/
int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
{
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err;
int poll_cnt;
u8 resp = 0xF;
for (poll_cnt = 100; poll_cnt; poll_cnt--) {
- err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
- UFS_QUERY_TASK, &resp);
+ err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_QUERY_TASK,
+ &resp);
if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
/* cmd pending in the device */
dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
@@ -7680,7 +7707,7 @@ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
hba->dev,
"%s: cmd with tag %d not pending in the device.\n",
__func__, tag);
- if (!ufshcd_cmd_inflight(lrbp->cmd)) {
+ if (!ufshcd_cmd_inflight(cmd)) {
dev_info(hba->dev,
"%s: cmd with tag=%d completed.\n",
__func__, tag);
@@ -7698,8 +7725,7 @@ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
if (!poll_cnt)
return -EBUSY;
- err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
- UFS_ABORT_TASK, &resp);
+ err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_ABORT_TASK, &resp);
if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
if (!err) {
err = resp; /* service response error */
@@ -7727,8 +7753,9 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
{
struct Scsi_Host *host = cmd->device->host;
struct ufs_hba *hba = shost_priv(host);
- int tag = scsi_cmd_to_rq(cmd)->tag;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct request *rq = scsi_cmd_to_rq(cmd);
+ int tag = rq->tag;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
unsigned long flags;
int err = FAILED;
bool outstanding;
@@ -7757,15 +7784,16 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
* to reduce repeated printouts. For other aborted requests only print
* basic details.
*/
- scsi_print_command(cmd);
+ if (ufshcd_is_scsi_cmd(cmd))
+ scsi_print_command(cmd);
if (!hba->req_abort_count) {
ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
ufshcd_print_evt_hist(hba);
ufshcd_print_host_state(hba);
ufshcd_print_pwr_info(hba);
- ufshcd_print_tr(hba, tag, true);
+ ufshcd_print_tr(hba, cmd, true);
} else {
- ufshcd_print_tr(hba, tag, false);
+ ufshcd_print_tr(hba, cmd, false);
}
hba->req_abort_count++;
@@ -7809,7 +7837,10 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
goto release;
}
- err = ufshcd_try_to_abort_task(hba, tag);
+ if (blk_mq_is_reserved_rq(rq))
+ err = ufshcd_clear_cmd(hba, tag);
+ else
+ err = ufshcd_try_to_abort_task(hba, tag);
if (err) {
dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
@@ -7826,7 +7857,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
spin_unlock_irqrestore(&hba->outstanding_lock, flags);
if (outstanding)
- ufshcd_release_scsi_cmd(hba, lrbp);
+ ufshcd_release_scsi_cmd(hba, cmd);
err = SUCCESS;
@@ -8192,8 +8223,11 @@ static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
if (IS_ERR(sdev_rpmb)) {
ret = PTR_ERR(sdev_rpmb);
+ hba->ufs_rpmb_wlun = NULL;
+ dev_err(hba->dev, "%s: RPMB WLUN not found\n", __func__);
goto remove_ufs_device_wlun;
}
+ hba->ufs_rpmb_wlun = sdev_rpmb;
ufshcd_blk_pm_runtime_init(sdev_rpmb);
scsi_device_put(sdev_rpmb);
@@ -8461,12 +8495,74 @@ static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf)
dev_info->rtc_update_period = 0;
}
+/**
+ * ufshcd_create_device_id - Generate unique device identifier string
+ * @hba: per-adapter instance
+ * @desc_buf: device descriptor buffer
+ *
+ * Creates a unique device ID string combining manufacturer ID, spec version,
+ * model name, serial number (as hex), device version, and manufacture date.
+ *
+ * Returns: Allocated device ID string on success, NULL on failure
+ */
+static char *ufshcd_create_device_id(struct ufs_hba *hba, u8 *desc_buf)
+{
+ struct ufs_dev_info *dev_info = &hba->dev_info;
+ u16 manufacture_date;
+ u16 device_version;
+ u8 *serial_number;
+ char *serial_hex;
+ char *device_id;
+ u8 serial_index;
+ int serial_len;
+ int ret;
+
+ serial_index = desc_buf[DEVICE_DESC_PARAM_SN];
+
+ ret = ufshcd_read_string_desc(hba, serial_index, &serial_number, SD_RAW);
+ if (ret < 0) {
+ dev_err(hba->dev, "Failed reading Serial Number. err = %d\n", ret);
+ return NULL;
+ }
+
+ device_version = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_DEV_VER]);
+ manufacture_date = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_MANF_DATE]);
+
+ serial_len = ret;
+ /* Allocate buffer for hex string: 2 chars per byte + null terminator */
+ serial_hex = kzalloc(serial_len * 2 + 1, GFP_KERNEL);
+ if (!serial_hex) {
+ kfree(serial_number);
+ return NULL;
+ }
+
+ bin2hex(serial_hex, serial_number, serial_len);
+
+ /*
+ * Device ID format is ABI with secure world - do not change without firmware
+ * coordination.
+ */
+ device_id = kasprintf(GFP_KERNEL, "%04X-%04X-%s-%s-%04X-%04X",
+ dev_info->wmanufacturerid, dev_info->wspecversion,
+ dev_info->model, serial_hex, device_version,
+ manufacture_date);
+
+ kfree(serial_hex);
+ kfree(serial_number);
+
+ if (!device_id)
+ dev_warn(hba->dev, "Failed to allocate unique device ID\n");
+
+ return device_id;
+}
+
static int ufs_get_device_desc(struct ufs_hba *hba)
{
+ struct ufs_dev_info *dev_info = &hba->dev_info;
+ struct Scsi_Host *shost = hba->host;
int err;
u8 model_index;
u8 *desc_buf;
- struct ufs_dev_info *dev_info = &hba->dev_info;
desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
if (!desc_buf) {
@@ -8494,6 +8590,18 @@ static int ufs_get_device_desc(struct ufs_hba *hba)
desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH];
+ /*
+ * According to the UFS standard, the UFS device queue depth
+ * (bQueueDepth) must be in the range 1..255 if the shared queueing
+ * architecture is supported. bQueueDepth is zero if the shared queueing
+ * architecture is not supported.
+ */
+ if (dev_info->bqueuedepth)
+ shost->cmd_per_lun = min(hba->nutrs, dev_info->bqueuedepth) -
+ UFSHCD_NUM_RESERVED;
+ else
+ shost->cmd_per_lun = shost->can_queue;
+
dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP];
dev_info->hid_sup = get_unaligned_be32(desc_buf +
@@ -8510,6 +8618,9 @@ static int ufs_get_device_desc(struct ufs_hba *hba)
goto out;
}
+ /* Generate unique device ID */
+ dev_info->device_id = ufshcd_create_device_id(hba, desc_buf);
+
hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] +
desc_buf[DEVICE_DESC_PARAM_NUM_WLU];
@@ -8545,6 +8656,8 @@ static void ufs_put_device_desc(struct ufs_hba *hba)
kfree(dev_info->model);
dev_info->model = NULL;
+ kfree(dev_info->device_id);
+ dev_info->device_id = NULL;
}
/**
@@ -8688,6 +8801,8 @@ static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
hba->dev_info.max_lu_supported = 8;
+ hba->dev_info.rpmb_io_size = desc_buf[GEOMETRY_DESC_PARAM_RPMB_RW_SIZE];
+
out:
kfree(desc_buf);
return err;
@@ -8874,6 +8989,7 @@ static int ufshcd_add_lus(struct ufs_hba *hba)
ufs_bsg_probe(hba);
scsi_scan_host(hba->host);
+ ufs_rpmb_probe(hba);
out:
return ret;
@@ -8891,8 +9007,6 @@ static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs)
utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs;
dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr,
hba->utrdl_dma_addr);
-
- devm_kfree(hba->dev, hba->lrb);
}
static int ufshcd_alloc_mcq(struct ufs_hba *hba)
@@ -8900,7 +9014,7 @@ static int ufshcd_alloc_mcq(struct ufs_hba *hba)
int ret;
int old_nutrs = hba->nutrs;
- ret = ufshcd_mcq_decide_queue_depth(hba);
+ ret = ufshcd_get_hba_mac(hba);
if (ret < 0)
return ret;
@@ -8926,7 +9040,6 @@ static int ufshcd_alloc_mcq(struct ufs_hba *hba)
goto err;
hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
- hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED;
return 0;
err:
@@ -9164,7 +9277,11 @@ static const struct scsi_host_template ufshcd_driver_template = {
.name = UFSHCD,
.proc_name = UFSHCD,
.map_queues = ufshcd_map_queues,
+ .cmd_size = sizeof(struct ufshcd_lrb),
+ .init_cmd_priv = ufshcd_init_cmd_priv,
.queuecommand = ufshcd_queuecommand,
+ .queue_reserved_command = ufshcd_queue_reserved_command,
+ .nr_reserved_cmds = UFSHCD_NUM_RESERVED,
.mq_poll = ufshcd_poll,
.sdev_init = ufshcd_sdev_init,
.sdev_configure = ufshcd_sdev_configure,
@@ -9775,11 +9892,11 @@ static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
}
/*
- * Some UFS devices require delay after VCC power rail is turned-off.
+ * All UFS devices require delay after VCC power rail is turned-off.
*/
- if (vcc_off && hba->vreg_info.vcc &&
- hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
- usleep_range(5000, 5100);
+ if (vcc_off && hba->vreg_info.vcc && !hba->vreg_info.vcc->always_on)
+ usleep_range(hba->vcc_off_delay_us,
+ hba->vcc_off_delay_us + 100);
}
#ifdef CONFIG_PM
@@ -10428,6 +10545,7 @@ void ufshcd_remove(struct ufs_hba *hba)
ufshcd_rpm_get_sync(hba);
ufs_hwmon_remove(hba);
ufs_bsg_remove(hba);
+ ufs_rpmb_remove(hba);
ufs_sysfs_remove_nodes(hba->dev);
cancel_delayed_work_sync(&hba->ufs_rtc_update_work);
blk_mq_destroy_queue(hba->tmf_queue);
@@ -10587,6 +10705,9 @@ static int ufshcd_add_scsi_host(struct ufs_hba *hba)
{
int err;
+ WARN_ON_ONCE(!hba->host->can_queue);
+ WARN_ON_ONCE(!hba->host->cmd_per_lun);
+
if (is_mcq_supported(hba)) {
ufshcd_mcq_enable(hba);
err = ufshcd_alloc_mcq(hba);
@@ -10706,7 +10827,12 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
UFS_SLEEP_PWR_MODE,
UIC_LINK_HIBERN8_STATE);
- init_completion(&hba->dev_cmd.complete);
+ /*
+ * Most ufs devices require 1ms delay after vcc is powered off before
+ * it can be powered on again. Set the default to 2ms. The platform
+ * drivers can override this setting as needed.
+ */
+ hba->vcc_off_delay_us = 2000;
err = ufshcd_hba_init(hba);
if (err)
@@ -10740,7 +10866,11 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
ufshcd_host_memory_configure(hba);
host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
- host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED;
+ /*
+ * Set the queue depth for WLUNs. ufs_get_device_desc() will increase
+ * host->cmd_per_lun to a larger value.
+ */
+ host->cmd_per_lun = 1;
host->max_id = UFSHCD_MAX_ID;
host->max_lun = UFS_MAX_LUNS;
host->max_channel = UFSHCD_MAX_CHANNEL;
@@ -10832,6 +10962,10 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
}
+ err = ufshcd_add_scsi_host(hba);
+ if (err)
+ goto out_disable;
+
/* Hold auto suspend until async scan completes */
pm_runtime_get_sync(dev);
@@ -10882,10 +11016,6 @@ initialized:
if (err)
goto out_disable;
- err = ufshcd_add_scsi_host(hba);
- if (err)
- goto out_disable;
-
ufs_sysfs_add_nodes(hba->dev);
async_schedule(ufshcd_async_scan, hba);
diff --git a/drivers/ufs/host/Kconfig b/drivers/ufs/host/Kconfig
index 191fbd799ec5..7d5117b2dab4 100644
--- a/drivers/ufs/host/Kconfig
+++ b/drivers/ufs/host/Kconfig
@@ -154,3 +154,16 @@ config SCSI_UFS_ROCKCHIP
Select this if you have UFS controller on Rockchip chipset.
If unsure, say N.
+
+config SCSI_UFS_AMD_VERSAL2
+ tristate "AMD Versal Gen 2 UFS controller platform driver"
+ depends on SCSI_UFSHCD_PLATFORM && (ARCH_ZYNQMP || COMPILE_TEST)
+ help
+ This selects the AMD Versal Gen 2 specific additions on top of
+ the UFSHCD DWC and UFSHCD platform driver. UFS host on AMD
+ Versal Gen 2 needs some vendor specific configurations like PHY
+ and vendor specific register accesses before accessing the
+ hardware.
+
+ Select this if you have UFS controller on AMD Versal Gen 2 SoC.
+ If unsure, say N.
diff --git a/drivers/ufs/host/Makefile b/drivers/ufs/host/Makefile
index 2f97feb5db3f..65d8bb23ab7b 100644
--- a/drivers/ufs/host/Makefile
+++ b/drivers/ufs/host/Makefile
@@ -13,3 +13,4 @@ obj-$(CONFIG_SCSI_UFS_RENESAS) += ufs-renesas.o
obj-$(CONFIG_SCSI_UFS_ROCKCHIP) += ufs-rockchip.o
obj-$(CONFIG_SCSI_UFS_SPRD) += ufs-sprd.o
obj-$(CONFIG_SCSI_UFS_TI_J721E) += ti-j721e-ufs.o
+obj-$(CONFIG_SCSI_UFS_AMD_VERSAL2) += ufs-amd-versal2.o ufshcd-dwc.o
diff --git a/drivers/ufs/host/ti-j721e-ufs.c b/drivers/ufs/host/ti-j721e-ufs.c
index 21214e5d5896..43781593b5c1 100644
--- a/drivers/ufs/host/ti-j721e-ufs.c
+++ b/drivers/ufs/host/ti-j721e-ufs.c
@@ -15,18 +15,26 @@
#define TI_UFS_SS_RST_N_PCS BIT(0)
#define TI_UFS_SS_CLK_26MHZ BIT(4)
+struct ti_j721e_ufs {
+ void __iomem *regbase;
+ u32 reg;
+};
+
static int ti_j721e_ufs_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
+ struct ti_j721e_ufs *ufs;
unsigned long clk_rate;
- void __iomem *regbase;
struct clk *clk;
- u32 reg = 0;
int ret;
- regbase = devm_platform_ioremap_resource(pdev, 0);
- if (IS_ERR(regbase))
- return PTR_ERR(regbase);
+ ufs = devm_kzalloc(dev, sizeof(*ufs), GFP_KERNEL);
+ if (!ufs)
+ return -ENOMEM;
+
+ ufs->regbase = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(ufs->regbase))
+ return PTR_ERR(ufs->regbase);
pm_runtime_enable(dev);
ret = pm_runtime_resume_and_get(dev);
@@ -42,12 +50,14 @@ static int ti_j721e_ufs_probe(struct platform_device *pdev)
}
clk_rate = clk_get_rate(clk);
if (clk_rate == 26000000)
- reg |= TI_UFS_SS_CLK_26MHZ;
+ ufs->reg |= TI_UFS_SS_CLK_26MHZ;
devm_clk_put(dev, clk);
/* Take UFS slave device out of reset */
- reg |= TI_UFS_SS_RST_N_PCS;
- writel(reg, regbase + TI_UFS_SS_CTRL);
+ ufs->reg |= TI_UFS_SS_RST_N_PCS;
+ writel(ufs->reg, ufs->regbase + TI_UFS_SS_CTRL);
+
+ dev_set_drvdata(dev, ufs);
ret = of_platform_populate(pdev->dev.of_node, NULL, NULL,
dev);
@@ -72,6 +82,16 @@ static void ti_j721e_ufs_remove(struct platform_device *pdev)
pm_runtime_disable(&pdev->dev);
}
+static int ti_j721e_ufs_resume(struct device *dev)
+{
+ struct ti_j721e_ufs *ufs = dev_get_drvdata(dev);
+
+ writel(ufs->reg, ufs->regbase + TI_UFS_SS_CTRL);
+ return 0;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(ti_j721e_ufs_pm_ops, NULL, ti_j721e_ufs_resume);
+
static const struct of_device_id ti_j721e_ufs_of_match[] = {
{
.compatible = "ti,j721e-ufs",
@@ -87,6 +107,7 @@ static struct platform_driver ti_j721e_ufs_driver = {
.driver = {
.name = "ti-j721e-ufs",
.of_match_table = ti_j721e_ufs_of_match,
+ .pm = pm_sleep_ptr(&ti_j721e_ufs_pm_ops),
},
};
module_platform_driver(ti_j721e_ufs_driver);
diff --git a/drivers/ufs/host/ufs-amd-versal2.c b/drivers/ufs/host/ufs-amd-versal2.c
new file mode 100644
index 000000000000..40543db621a1
--- /dev/null
+++ b/drivers/ufs/host/ufs-amd-versal2.c
@@ -0,0 +1,564 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ *
+ * Authors: Sai Krishna Potthuri <sai.krishna.potthuri@amd.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/firmware/xlnx-zynqmp.h>
+#include <linux/irqreturn.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <ufs/unipro.h>
+
+#include "ufshcd-dwc.h"
+#include "ufshcd-pltfrm.h"
+#include "ufshci-dwc.h"
+
+/* PHY modes */
+#define UFSHCD_DWC_PHY_MODE_ROM 0
+
+#define MPHY_FAST_RX_AFE_CAL BIT(2)
+#define MPHY_FW_CALIB_CFG_VAL BIT(8)
+
+#define MPHY_RX_OVRD_EN BIT(3)
+#define MPHY_RX_OVRD_VAL BIT(2)
+#define MPHY_RX_ACK_MASK BIT(0)
+
+#define TIMEOUT_MICROSEC 1000000
+
+struct ufs_versal2_host {
+ struct ufs_hba *hba;
+ struct reset_control *rstc;
+ struct reset_control *rstphy;
+ u32 phy_mode;
+ unsigned long host_clk;
+ u8 attcompval0;
+ u8 attcompval1;
+ u8 ctlecompval0;
+ u8 ctlecompval1;
+};
+
+static int ufs_versal2_phy_reg_write(struct ufs_hba *hba, u32 addr, u32 val)
+{
+ static struct ufshcd_dme_attr_val phy_write_attrs[] = {
+ { UIC_ARG_MIB(CBCREGADDRLSB), 0, DME_LOCAL },
+ { UIC_ARG_MIB(CBCREGADDRMSB), 0, DME_LOCAL },
+ { UIC_ARG_MIB(CBCREGWRLSB), 0, DME_LOCAL },
+ { UIC_ARG_MIB(CBCREGWRMSB), 0, DME_LOCAL },
+ { UIC_ARG_MIB(CBCREGRDWRSEL), 1, DME_LOCAL },
+ { UIC_ARG_MIB(VS_MPHYCFGUPDT), 1, DME_LOCAL }
+ };
+
+ phy_write_attrs[0].mib_val = (u8)addr;
+ phy_write_attrs[1].mib_val = (u8)(addr >> 8);
+ phy_write_attrs[2].mib_val = (u8)val;
+ phy_write_attrs[3].mib_val = (u8)(val >> 8);
+
+ return ufshcd_dwc_dme_set_attrs(hba, phy_write_attrs, ARRAY_SIZE(phy_write_attrs));
+}
+
+static int ufs_versal2_phy_reg_read(struct ufs_hba *hba, u32 addr, u32 *val)
+{
+ u32 mib_val;
+ int ret;
+ static struct ufshcd_dme_attr_val phy_read_attrs[] = {
+ { UIC_ARG_MIB(CBCREGADDRLSB), 0, DME_LOCAL },
+ { UIC_ARG_MIB(CBCREGADDRMSB), 0, DME_LOCAL },
+ { UIC_ARG_MIB(CBCREGRDWRSEL), 0, DME_LOCAL },
+ { UIC_ARG_MIB(VS_MPHYCFGUPDT), 1, DME_LOCAL }
+ };
+
+ phy_read_attrs[0].mib_val = (u8)addr;
+ phy_read_attrs[1].mib_val = (u8)(addr >> 8);
+
+ ret = ufshcd_dwc_dme_set_attrs(hba, phy_read_attrs, ARRAY_SIZE(phy_read_attrs));
+ if (ret)
+ return ret;
+
+ ret = ufshcd_dme_get(hba, UIC_ARG_MIB(CBCREGRDLSB), &mib_val);
+ if (ret)
+ return ret;
+
+ *val = mib_val;
+ ret = ufshcd_dme_get(hba, UIC_ARG_MIB(CBCREGRDMSB), &mib_val);
+ if (ret)
+ return ret;
+
+ *val |= (mib_val << 8);
+
+ return 0;
+}
+
+static int ufs_versal2_enable_phy(struct ufs_hba *hba)
+{
+ u32 offset, reg;
+ int ret;
+
+ ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYDISABLE), 0);
+ if (ret)
+ return ret;
+
+ ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYCFGUPDT), 1);
+ if (ret)
+ return ret;
+
+ /* Check Tx/Rx FSM states */
+ for (offset = 0; offset < 2; offset++) {
+ u32 time_left, mibsel;
+
+ time_left = TIMEOUT_MICROSEC;
+ mibsel = UIC_ARG_MIB_SEL(MTX_FSM_STATE, UIC_ARG_MPHY_TX_GEN_SEL_INDEX(offset));
+ do {
+ ret = ufshcd_dme_get(hba, mibsel, &reg);
+ if (ret)
+ return ret;
+
+ if (reg == TX_STATE_HIBERN8 || reg == TX_STATE_SLEEP ||
+ reg == TX_STATE_LSBURST)
+ break;
+
+ time_left--;
+ usleep_range(1, 5);
+ } while (time_left);
+
+ if (!time_left) {
+ dev_err(hba->dev, "Invalid Tx FSM state.\n");
+ return -ETIMEDOUT;
+ }
+
+ time_left = TIMEOUT_MICROSEC;
+ mibsel = UIC_ARG_MIB_SEL(MRX_FSM_STATE, UIC_ARG_MPHY_RX_GEN_SEL_INDEX(offset));
+ do {
+ ret = ufshcd_dme_get(hba, mibsel, &reg);
+ if (ret)
+ return ret;
+
+ if (reg == RX_STATE_HIBERN8 || reg == RX_STATE_SLEEP ||
+ reg == RX_STATE_LSBURST)
+ break;
+
+ time_left--;
+ usleep_range(1, 5);
+ } while (time_left);
+
+ if (!time_left) {
+ dev_err(hba->dev, "Invalid Rx FSM state.\n");
+ return -ETIMEDOUT;
+ }
+ }
+
+ return 0;
+}
+
+static int ufs_versal2_setup_phy(struct ufs_hba *hba)
+{
+ struct ufs_versal2_host *host = ufshcd_get_variant(hba);
+ int ret;
+ u32 reg;
+
+ /* Bypass RX-AFE offset calibrations (ATT/CTLE) */
+ ret = ufs_versal2_phy_reg_read(hba, FAST_FLAGS(0), &reg);
+ if (ret)
+ return ret;
+
+ reg |= MPHY_FAST_RX_AFE_CAL;
+ ret = ufs_versal2_phy_reg_write(hba, FAST_FLAGS(0), reg);
+ if (ret)
+ return ret;
+
+ ret = ufs_versal2_phy_reg_read(hba, FAST_FLAGS(1), &reg);
+ if (ret)
+ return ret;
+
+ reg |= MPHY_FAST_RX_AFE_CAL;
+ ret = ufs_versal2_phy_reg_write(hba, FAST_FLAGS(1), reg);
+ if (ret)
+ return ret;
+
+ /* Program ATT and CTLE compensation values */
+ if (host->attcompval0) {
+ ret = ufs_versal2_phy_reg_write(hba, RX_AFE_ATT_IDAC(0), host->attcompval0);
+ if (ret)
+ return ret;
+ }
+
+ if (host->attcompval1) {
+ ret = ufs_versal2_phy_reg_write(hba, RX_AFE_ATT_IDAC(1), host->attcompval1);
+ if (ret)
+ return ret;
+ }
+
+ if (host->ctlecompval0) {
+ ret = ufs_versal2_phy_reg_write(hba, RX_AFE_CTLE_IDAC(0), host->ctlecompval0);
+ if (ret)
+ return ret;
+ }
+
+ if (host->ctlecompval1) {
+ ret = ufs_versal2_phy_reg_write(hba, RX_AFE_CTLE_IDAC(1), host->ctlecompval1);
+ if (ret)
+ return ret;
+ }
+
+ ret = ufs_versal2_phy_reg_read(hba, FW_CALIB_CCFG(0), &reg);
+ if (ret)
+ return ret;
+
+ reg |= MPHY_FW_CALIB_CFG_VAL;
+ ret = ufs_versal2_phy_reg_write(hba, FW_CALIB_CCFG(0), reg);
+ if (ret)
+ return ret;
+
+ ret = ufs_versal2_phy_reg_read(hba, FW_CALIB_CCFG(1), &reg);
+ if (ret)
+ return ret;
+
+ reg |= MPHY_FW_CALIB_CFG_VAL;
+ return ufs_versal2_phy_reg_write(hba, FW_CALIB_CCFG(1), reg);
+}
+
+static int ufs_versal2_phy_init(struct ufs_hba *hba)
+{
+ struct ufs_versal2_host *host = ufshcd_get_variant(hba);
+ u32 time_left;
+ bool is_ready;
+ int ret;
+ static const struct ufshcd_dme_attr_val rmmi_attrs[] = {
+ { UIC_ARG_MIB(CBREFCLKCTRL2), CBREFREFCLK_GATE_OVR_EN, DME_LOCAL },
+ { UIC_ARG_MIB(CBCRCTRL), 1, DME_LOCAL },
+ { UIC_ARG_MIB(CBC10DIRECTCONF2), 1, DME_LOCAL },
+ { UIC_ARG_MIB(VS_MPHYCFGUPDT), 1, DME_LOCAL }
+ };
+
+ /* Wait for Tx/Rx config_rdy */
+ time_left = TIMEOUT_MICROSEC;
+ do {
+ time_left--;
+ ret = zynqmp_pm_is_mphy_tx_rx_config_ready(&is_ready);
+ if (ret)
+ return ret;
+
+ if (!is_ready)
+ break;
+
+ usleep_range(1, 5);
+ } while (time_left);
+
+ if (!time_left) {
+ dev_err(hba->dev, "Tx/Rx configuration signal busy.\n");
+ return -ETIMEDOUT;
+ }
+
+ ret = ufshcd_dwc_dme_set_attrs(hba, rmmi_attrs, ARRAY_SIZE(rmmi_attrs));
+ if (ret)
+ return ret;
+
+ ret = reset_control_deassert(host->rstphy);
+ if (ret) {
+ dev_err(hba->dev, "ufsphy reset deassert failed, err = %d\n", ret);
+ return ret;
+ }
+
+ /* Wait for SRAM init done */
+ time_left = TIMEOUT_MICROSEC;
+ do {
+ time_left--;
+ ret = zynqmp_pm_is_sram_init_done(&is_ready);
+ if (ret)
+ return ret;
+
+ if (is_ready)
+ break;
+
+ usleep_range(1, 5);
+ } while (time_left);
+
+ if (!time_left) {
+ dev_err(hba->dev, "SRAM initialization failed.\n");
+ return -ETIMEDOUT;
+ }
+
+ ret = ufs_versal2_setup_phy(hba);
+ if (ret)
+ return ret;
+
+ return ufs_versal2_enable_phy(hba);
+}
+
+static int ufs_versal2_init(struct ufs_hba *hba)
+{
+ struct ufs_versal2_host *host;
+ struct device *dev = hba->dev;
+ struct ufs_clk_info *clki;
+ int ret;
+ u32 cal;
+
+ host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
+ if (!host)
+ return -ENOMEM;
+
+ host->hba = hba;
+ ufshcd_set_variant(hba, host);
+
+ host->phy_mode = UFSHCD_DWC_PHY_MODE_ROM;
+
+ list_for_each_entry(clki, &hba->clk_list_head, list) {
+ if (!strcmp(clki->name, "core"))
+ host->host_clk = clk_get_rate(clki->clk);
+ }
+
+ host->rstc = devm_reset_control_get_exclusive(dev, "host");
+ if (IS_ERR(host->rstc)) {
+ dev_err(dev, "failed to get reset ctrl: host\n");
+ return PTR_ERR(host->rstc);
+ }
+
+ host->rstphy = devm_reset_control_get_exclusive(dev, "phy");
+ if (IS_ERR(host->rstphy)) {
+ dev_err(dev, "failed to get reset ctrl: phy\n");
+ return PTR_ERR(host->rstphy);
+ }
+
+ ret = reset_control_assert(host->rstc);
+ if (ret) {
+ dev_err(hba->dev, "host reset assert failed, err = %d\n", ret);
+ return ret;
+ }
+
+ ret = reset_control_assert(host->rstphy);
+ if (ret) {
+ dev_err(hba->dev, "phy reset assert failed, err = %d\n", ret);
+ return ret;
+ }
+
+ ret = zynqmp_pm_set_sram_bypass();
+ if (ret) {
+ dev_err(dev, "Bypass SRAM interface failed, err = %d\n", ret);
+ return ret;
+ }
+
+ ret = reset_control_deassert(host->rstc);
+ if (ret)
+ dev_err(hba->dev, "host reset deassert failed, err = %d\n", ret);
+
+ ret = zynqmp_pm_get_ufs_calibration_values(&cal);
+ if (ret) {
+ dev_err(dev, "failed to read calibration values\n");
+ return ret;
+ }
+
+ host->attcompval0 = (u8)cal;
+ host->attcompval1 = (u8)(cal >> 8);
+ host->ctlecompval0 = (u8)(cal >> 16);
+ host->ctlecompval1 = (u8)(cal >> 24);
+
+ hba->quirks |= UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING;
+
+ return 0;
+}
+
+static int ufs_versal2_hce_enable_notify(struct ufs_hba *hba,
+ enum ufs_notify_change_status status)
+{
+ int ret = 0;
+
+ if (status == PRE_CHANGE) {
+ ret = ufs_versal2_phy_init(hba);
+ if (ret)
+ dev_err(hba->dev, "Phy init failed (%d)\n", ret);
+ }
+
+ return ret;
+}
+
+static int ufs_versal2_link_startup_notify(struct ufs_hba *hba,
+ enum ufs_notify_change_status status)
+{
+ struct ufs_versal2_host *host = ufshcd_get_variant(hba);
+ int ret = 0;
+
+ switch (status) {
+ case PRE_CHANGE:
+ if (host->host_clk)
+ ufshcd_writel(hba, host->host_clk / 1000000, DWC_UFS_REG_HCLKDIV);
+
+ break;
+ case POST_CHANGE:
+ ret = ufshcd_dwc_link_startup_notify(hba, status);
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+ return ret;
+}
+
+static int ufs_versal2_phy_ratesel(struct ufs_hba *hba, u32 activelanes, u32 rx_req)
+{
+ u32 time_left, reg, lane;
+ int ret;
+
+ for (lane = 0; lane < activelanes; lane++) {
+ time_left = TIMEOUT_MICROSEC;
+ ret = ufs_versal2_phy_reg_read(hba, RX_OVRD_IN_1(lane), &reg);
+ if (ret)
+ return ret;
+
+ reg |= MPHY_RX_OVRD_EN;
+ if (rx_req)
+ reg |= MPHY_RX_OVRD_VAL;
+ else
+ reg &= ~MPHY_RX_OVRD_VAL;
+
+ ret = ufs_versal2_phy_reg_write(hba, RX_OVRD_IN_1(lane), reg);
+ if (ret)
+ return ret;
+
+ do {
+ ret = ufs_versal2_phy_reg_read(hba, RX_PCS_OUT(lane), &reg);
+ if (ret)
+ return ret;
+
+ reg &= MPHY_RX_ACK_MASK;
+ if (reg == rx_req)
+ break;
+
+ time_left--;
+ usleep_range(1, 5);
+ } while (time_left);
+
+ if (!time_left) {
+ dev_err(hba->dev, "Invalid Rx Ack value.\n");
+ return -ETIMEDOUT;
+ }
+ }
+
+ return 0;
+}
+
+static int ufs_versal2_pwr_change_notify(struct ufs_hba *hba, enum ufs_notify_change_status status,
+ const struct ufs_pa_layer_attr *dev_max_params,
+ struct ufs_pa_layer_attr *dev_req_params)
+{
+ struct ufs_versal2_host *host = ufshcd_get_variant(hba);
+ u32 lane, reg, rate = 0;
+ int ret = 0;
+
+ if (status == PRE_CHANGE) {
+ memcpy(dev_req_params, dev_max_params, sizeof(struct ufs_pa_layer_attr));
+
+ /* If it is not a calibrated part, switch PWRMODE to SLOW_MODE */
+ if (!host->attcompval0 && !host->attcompval1 && !host->ctlecompval0 &&
+ !host->ctlecompval1) {
+ dev_req_params->pwr_rx = SLOW_MODE;
+ dev_req_params->pwr_tx = SLOW_MODE;
+ return 0;
+ }
+
+ if (dev_req_params->pwr_rx == SLOW_MODE || dev_req_params->pwr_rx == SLOWAUTO_MODE)
+ return 0;
+
+ if (dev_req_params->hs_rate == PA_HS_MODE_B)
+ rate = 1;
+
+ /* Select the rate */
+ ret = ufshcd_dme_set(hba, UIC_ARG_MIB(CBRATESEL), rate);
+ if (ret)
+ return ret;
+
+ ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYCFGUPDT), 1);
+ if (ret)
+ return ret;
+
+ ret = ufs_versal2_phy_ratesel(hba, dev_req_params->lane_tx, 1);
+ if (ret)
+ return ret;
+
+ ret = ufs_versal2_phy_ratesel(hba, dev_req_params->lane_tx, 0);
+ if (ret)
+ return ret;
+
+ /* Remove rx_req override */
+ for (lane = 0; lane < dev_req_params->lane_tx; lane++) {
+ ret = ufs_versal2_phy_reg_read(hba, RX_OVRD_IN_1(lane), &reg);
+ if (ret)
+ return ret;
+
+ reg &= ~MPHY_RX_OVRD_EN;
+ ret = ufs_versal2_phy_reg_write(hba, RX_OVRD_IN_1(lane), reg);
+ if (ret)
+ return ret;
+ }
+
+ if (dev_req_params->lane_tx == UFS_LANE_2 && dev_req_params->lane_rx == UFS_LANE_2)
+ ret = ufshcd_dme_configure_adapt(hba, dev_req_params->gear_tx,
+ PA_INITIAL_ADAPT);
+ }
+
+ return ret;
+}
+
+static struct ufs_hba_variant_ops ufs_versal2_hba_vops = {
+ .name = "ufs-versal2-pltfm",
+ .init = ufs_versal2_init,
+ .link_startup_notify = ufs_versal2_link_startup_notify,
+ .hce_enable_notify = ufs_versal2_hce_enable_notify,
+ .pwr_change_notify = ufs_versal2_pwr_change_notify,
+};
+
+static const struct of_device_id ufs_versal2_pltfm_match[] = {
+ {
+ .compatible = "amd,versal2-ufs",
+ .data = &ufs_versal2_hba_vops,
+ },
+ { },
+};
+MODULE_DEVICE_TABLE(of, ufs_versal2_pltfm_match);
+
+static int ufs_versal2_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ int ret;
+
+ /* Perform generic probe */
+ ret = ufshcd_pltfrm_init(pdev, &ufs_versal2_hba_vops);
+ if (ret)
+ dev_err(dev, "ufshcd_pltfrm_init() failed %d\n", ret);
+
+ return ret;
+}
+
+static void ufs_versal2_remove(struct platform_device *pdev)
+{
+ struct ufs_hba *hba = platform_get_drvdata(pdev);
+
+ pm_runtime_get_sync(&(pdev)->dev);
+ ufshcd_remove(hba);
+}
+
+static const struct dev_pm_ops ufs_versal2_pm_ops = {
+ SET_SYSTEM_SLEEP_PM_OPS(ufshcd_system_suspend, ufshcd_system_resume)
+ SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL)
+};
+
+static struct platform_driver ufs_versal2_pltfm = {
+ .probe = ufs_versal2_probe,
+ .remove = ufs_versal2_remove,
+ .driver = {
+ .name = "ufshcd-versal2",
+ .pm = &ufs_versal2_pm_ops,
+ .of_match_table = of_match_ptr(ufs_versal2_pltfm_match),
+ },
+};
+
+module_platform_driver(ufs_versal2_pltfm);
+
+MODULE_AUTHOR("Sai Krishna Potthuri <sai.krishna.potthuri@amd.com>");
+MODULE_DESCRIPTION("AMD Versal Gen 2 UFS Host Controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c
index 758a393a9de1..ecbbf52bf734 100644
--- a/drivers/ufs/host/ufs-mediatek.c
+++ b/drivers/ufs/host/ufs-mediatek.c
@@ -41,8 +41,7 @@ static void _ufs_mtk_clk_scale(struct ufs_hba *hba, bool scale_up);
static const struct ufs_dev_quirk ufs_mtk_dev_fixups[] = {
{ .wmanufacturerid = UFS_ANY_VENDOR,
.model = UFS_ANY_MODEL,
- .quirk = UFS_DEVICE_QUIRK_DELAY_AFTER_LPM |
- UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
+ .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
{ .wmanufacturerid = UFS_VENDOR_SKHYNIX,
.model = "H9HQ21AFAMZDAR",
.quirk = UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES },
@@ -280,12 +279,21 @@ static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba,
ufshcd_readl(hba, REG_UFS_XOUFS_CTRL) | 0x80,
REG_UFS_XOUFS_CTRL);
+ if (host->legacy_ip_ver)
+ return 0;
+
/* DDR_EN setting */
if (host->ip_ver >= IP_VER_MT6989) {
ufshcd_rmwl(hba, UFS_MASK(0x7FFF, 8),
0x453000, REG_UFS_MMIO_OPT_CTRL_0);
}
+ if (host->ip_ver >= IP_VER_MT6991_A0) {
+ /* Enable multi-rtt */
+ ufshcd_rmwl(hba, MRTT_EN, MRTT_EN, REG_UFS_MMIO_OPT_CTRL_0);
+ /* Enable random performance improvement */
+ ufshcd_rmwl(hba, RDN_PFM_IMPV_DIS, 0, REG_UFS_MMIO_OPT_CTRL_0);
+ }
}
return 0;
@@ -405,7 +413,7 @@ static void ufs_mtk_dbg_sel(struct ufs_hba *hba)
{
struct ufs_mtk_host *host = ufshcd_get_variant(hba);
- if (((host->ip_ver >> 16) & 0xFF) >= 0x36) {
+ if (!host->legacy_ip_ver && host->ip_ver >= IP_VER_MT6983) {
ufshcd_writel(hba, 0x820820, REG_UFS_DEBUG_SEL);
ufshcd_writel(hba, 0x0, REG_UFS_DEBUG_SEL_B0);
ufshcd_writel(hba, 0x55555555, REG_UFS_DEBUG_SEL_B1);
@@ -422,6 +430,7 @@ static int ufs_mtk_wait_idle_state(struct ufs_hba *hba,
u64 timeout, time_checked;
u32 val, sm;
bool wait_idle;
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
/* cannot use plain ktime_get() in suspend */
timeout = ktime_get_mono_fast_ns() + retry_ms * 1000000UL;
@@ -432,8 +441,13 @@ static int ufs_mtk_wait_idle_state(struct ufs_hba *hba,
do {
time_checked = ktime_get_mono_fast_ns();
- ufs_mtk_dbg_sel(hba);
- val = ufshcd_readl(hba, REG_UFS_PROBE);
+ if (host->legacy_ip_ver || host->ip_ver < IP_VER_MT6899) {
+ ufs_mtk_dbg_sel(hba);
+ val = ufshcd_readl(hba, REG_UFS_PROBE);
+ } else {
+ val = ufshcd_readl(hba, REG_UFS_UFS_MMIO_OTSD_CTRL);
+ val = val >> 16;
+ }
sm = val & 0x1f;
@@ -465,13 +479,20 @@ static int ufs_mtk_wait_link_state(struct ufs_hba *hba, u32 state,
{
ktime_t timeout, time_checked;
u32 val;
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
timeout = ktime_add_ms(ktime_get(), max_wait_ms);
do {
time_checked = ktime_get();
- ufs_mtk_dbg_sel(hba);
- val = ufshcd_readl(hba, REG_UFS_PROBE);
- val = val >> 28;
+
+ if (host->legacy_ip_ver || host->ip_ver < IP_VER_MT6899) {
+ ufs_mtk_dbg_sel(hba);
+ val = ufshcd_readl(hba, REG_UFS_PROBE);
+ val = val >> 28;
+ } else {
+ val = ufshcd_readl(hba, REG_UFS_UFS_MMIO_OTSD_CTRL);
+ val = val >> 24;
+ }
if (val == state)
return 0;
@@ -1109,18 +1130,6 @@ static void ufs_mtk_setup_clk_gating(struct ufs_hba *hba)
}
}
-/* Convert microseconds to Auto-Hibernate Idle Timer register value */
-static u32 ufs_mtk_us_to_ahit(unsigned int timer)
-{
- unsigned int scale;
-
- for (scale = 0; timer > UFSHCI_AHIBERN8_TIMER_MASK; ++scale)
- timer /= UFSHCI_AHIBERN8_SCALE_FACTOR;
-
- return FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, timer) |
- FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, scale);
-}
-
static void ufs_mtk_fix_ahit(struct ufs_hba *hba)
{
unsigned int us;
@@ -1143,7 +1152,7 @@ static void ufs_mtk_fix_ahit(struct ufs_hba *hba)
break;
}
- hba->ahit = ufs_mtk_us_to_ahit(us);
+ hba->ahit = ufshcd_us_to_ahit(us);
}
ufs_mtk_setup_clk_gating(hba);
@@ -1332,6 +1341,36 @@ static bool ufs_mtk_pmc_via_fastauto(struct ufs_hba *hba,
return true;
}
+static void ufs_mtk_adjust_sync_length(struct ufs_hba *hba)
+{
+ int i;
+ u32 value;
+ u32 cnt, att, min;
+ struct attr_min {
+ u32 attr;
+ u32 min_value;
+ } pa_min_sync_length[] = {
+ {PA_TXHSG1SYNCLENGTH, 0x48},
+ {PA_TXHSG2SYNCLENGTH, 0x48},
+ {PA_TXHSG3SYNCLENGTH, 0x48},
+ {PA_TXHSG4SYNCLENGTH, 0x48},
+ {PA_TXHSG5SYNCLENGTH, 0x48}
+ };
+
+ cnt = sizeof(pa_min_sync_length) / sizeof(struct attr_min);
+ for (i = 0; i < cnt; i++) {
+ att = pa_min_sync_length[i].attr;
+ min = pa_min_sync_length[i].min_value;
+ ufshcd_dme_get(hba, UIC_ARG_MIB(att), &value);
+ if (value < min)
+ ufshcd_dme_set(hba, UIC_ARG_MIB(att), min);
+
+ ufshcd_dme_peer_get(hba, UIC_ARG_MIB(att), &value);
+ if (value < min)
+ ufshcd_dme_peer_set(hba, UIC_ARG_MIB(att), min);
+ }
+}
+
static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba,
const struct ufs_pa_layer_attr *dev_max_params,
struct ufs_pa_layer_attr *dev_req_params)
@@ -1355,6 +1394,8 @@ static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba,
}
if (ufs_mtk_pmc_via_fastauto(hba, dev_req_params)) {
+ ufs_mtk_adjust_sync_length(hba);
+
ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true);
ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), UFS_HS_G1);
@@ -1619,14 +1660,26 @@ static int ufs_mtk_device_reset(struct ufs_hba *hba)
static int ufs_mtk_link_set_hpm(struct ufs_hba *hba)
{
int err;
+ u32 val;
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
err = ufshcd_hba_enable(hba);
if (err)
return err;
err = ufs_mtk_unipro_set_lpm(hba, false);
- if (err)
+ if (err) {
+ if (host->ip_ver < IP_VER_MT6899) {
+ ufs_mtk_dbg_sel(hba);
+ val = ufshcd_readl(hba, REG_UFS_PROBE);
+ } else {
+ val = ufshcd_readl(hba, REG_UFS_UFS_MMIO_OTSD_CTRL);
+ }
+ ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)val);
+ val = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
+ ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)val);
return err;
+ }
err = ufshcd_uic_hibern8_exit(hba);
if (err)
@@ -1744,6 +1797,7 @@ static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
{
int err;
struct arm_smccc_res res;
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
if (status == PRE_CHANGE) {
if (ufshcd_is_auto_hibern8_supported(hba))
@@ -1773,6 +1827,15 @@ static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
ufs_mtk_sram_pwr_ctrl(false, res);
+ /* Release pm_qos/clk if in scale-up mode during suspend */
+ if (ufshcd_is_clkscaling_supported(hba) && (host->clk_scale_up)) {
+ ufshcd_pm_qos_update(hba, false);
+ _ufs_mtk_clk_scale(hba, false);
+ } else if ((!ufshcd_is_clkscaling_supported(hba) &&
+ hba->pwr_info.gear_rx >= UFS_HS_G5)) {
+ _ufs_mtk_clk_scale(hba, false);
+ }
+
return 0;
fail:
/*
@@ -1788,6 +1851,7 @@ static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
{
int err;
struct arm_smccc_res res;
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)
ufs_mtk_dev_vreg_set_lpm(hba, false);
@@ -1798,6 +1862,15 @@ static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
if (err)
goto fail;
+ /* Request pm_qos/clk if in scale-up mode after resume */
+ if (ufshcd_is_clkscaling_supported(hba) && (host->clk_scale_up)) {
+ ufshcd_pm_qos_update(hba, true);
+ _ufs_mtk_clk_scale(hba, true);
+ } else if ((!ufshcd_is_clkscaling_supported(hba) &&
+ hba->pwr_info.gear_rx >= UFS_HS_G5)) {
+ _ufs_mtk_clk_scale(hba, true);
+ }
+
if (ufshcd_is_link_hibern8(hba)) {
err = ufs_mtk_link_set_hpm(hba);
if (err)
@@ -1889,15 +1962,13 @@ static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba)
{
ufshcd_fixup_dev_quirks(hba, ufs_mtk_dev_fixups);
- if (ufs_mtk_is_broken_vcc(hba) && hba->vreg_info.vcc &&
- (hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)) {
+ if (ufs_mtk_is_broken_vcc(hba) && hba->vreg_info.vcc) {
hba->vreg_info.vcc->always_on = true;
/*
* VCC will be kept always-on thus we don't
- * need any delay during regulator operations
+ * need any delay before putting device's VCC in LPM mode.
*/
- hba->dev_quirks &= ~(UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
- UFS_DEVICE_QUIRK_DELAY_AFTER_LPM);
+ hba->dev_quirks &= ~UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM;
}
ufs_mtk_vreg_fix_vcc(hba);
@@ -2373,6 +2444,11 @@ static int ufs_mtk_system_suspend(struct device *dev)
struct arm_smccc_res res;
int ret;
+ if (hba->shutting_down) {
+ ret = -EBUSY;
+ goto out;
+ }
+
ret = ufshcd_system_suspend(dev);
if (ret)
goto out;
diff --git a/drivers/ufs/host/ufs-mediatek.h b/drivers/ufs/host/ufs-mediatek.h
index dfbf78bd8664..9747277f11e8 100644
--- a/drivers/ufs/host/ufs-mediatek.h
+++ b/drivers/ufs/host/ufs-mediatek.h
@@ -20,6 +20,9 @@
#define MCQ_MULTI_INTR_EN BIT(2)
#define MCQ_CMB_INTR_EN BIT(3)
#define MCQ_AH8 BIT(4)
+#define MON_EN BIT(5)
+#define MRTT_EN BIT(25)
+#define RDN_PFM_IMPV_DIS BIT(28)
#define MCQ_INTR_EN_MSK (MCQ_MULTI_INTR_EN | MCQ_CMB_INTR_EN)
@@ -28,6 +31,7 @@
*/
#define REG_UFS_XOUFS_CTRL 0x140
#define REG_UFS_REFCLK_CTRL 0x144
+#define REG_UFS_UFS_MMIO_OTSD_CTRL 0x14C
#define REG_UFS_MMIO_OPT_CTRL_0 0x160
#define REG_UFS_EXTREG 0x2100
#define REG_UFS_MPHYCTRL 0x2200
diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c
index eba0e6617483..8d119b3223cb 100644
--- a/drivers/ufs/host/ufs-qcom.c
+++ b/drivers/ufs/host/ufs-qcom.c
@@ -1037,9 +1037,6 @@ static struct ufs_dev_quirk ufs_qcom_dev_fixups[] = {
{ .wmanufacturerid = UFS_VENDOR_SKHYNIX,
.model = UFS_ANY_MODEL,
.quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
- { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
- .model = UFS_ANY_MODEL,
- .quirk = UFS_DEVICE_QUIRK_DELAY_AFTER_LPM },
{ .wmanufacturerid = UFS_VENDOR_WDC,
.model = UFS_ANY_MODEL,
.quirk = UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE },
diff --git a/drivers/ufs/host/ufs-rockchip.c b/drivers/ufs/host/ufs-rockchip.c
index 8754085dd0cc..7fff34513a60 100644
--- a/drivers/ufs/host/ufs-rockchip.c
+++ b/drivers/ufs/host/ufs-rockchip.c
@@ -7,6 +7,7 @@
#include <linux/clk.h>
#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
#include <linux/mfd/syscon.h>
#include <linux/of.h>
#include <linux/platform_device.h>
@@ -20,9 +21,17 @@
#include "ufshcd-pltfrm.h"
#include "ufs-rockchip.h"
+static void ufs_rockchip_controller_reset(struct ufs_rockchip_host *host)
+{
+ reset_control_assert(host->rst);
+ udelay(1);
+ reset_control_deassert(host->rst);
+}
+
static int ufs_rockchip_hce_enable_notify(struct ufs_hba *hba,
enum ufs_notify_change_status status)
{
+ struct ufs_rockchip_host *host = ufshcd_get_variant(hba);
int err = 0;
if (status == POST_CHANGE) {
@@ -37,6 +46,9 @@ static int ufs_rockchip_hce_enable_notify(struct ufs_hba *hba,
return ufshcd_vops_phy_initialization(hba);
}
+ /* PRE_CHANGE */
+ ufs_rockchip_controller_reset(host);
+
return 0;
}
@@ -156,9 +168,7 @@ static int ufs_rockchip_common_init(struct ufs_hba *hba)
return dev_err_probe(dev, PTR_ERR(host->rst),
"failed to get reset control\n");
- reset_control_assert(host->rst);
- udelay(1);
- reset_control_deassert(host->rst);
+ ufs_rockchip_controller_reset(host);
host->ref_out_clk = devm_clk_get_enabled(dev, "ref_out");
if (IS_ERR(host->ref_out_clk))
@@ -282,9 +292,7 @@ static int ufs_rockchip_runtime_resume(struct device *dev)
return err;
}
- reset_control_assert(host->rst);
- udelay(1);
- reset_control_deassert(host->rst);
+ ufs_rockchip_controller_reset(host);
return ufshcd_runtime_resume(dev);
}
diff --git a/drivers/ufs/host/ufshcd-dwc.h b/drivers/ufs/host/ufshcd-dwc.h
index ad91ea56662c..c618bb914904 100644
--- a/drivers/ufs/host/ufshcd-dwc.h
+++ b/drivers/ufs/host/ufshcd-dwc.h
@@ -12,6 +12,52 @@
#include <ufs/ufshcd.h>
+/* RMMI Attributes */
+#define CBREFCLKCTRL2 0x8132
+#define CBCRCTRL 0x811F
+#define CBC10DIRECTCONF2 0x810E
+#define CBRATESEL 0x8114
+#define CBCREGADDRLSB 0x8116
+#define CBCREGADDRMSB 0x8117
+#define CBCREGWRLSB 0x8118
+#define CBCREGWRMSB 0x8119
+#define CBCREGRDLSB 0x811A
+#define CBCREGRDMSB 0x811B
+#define CBCREGRDWRSEL 0x811C
+
+#define CBREFREFCLK_GATE_OVR_EN BIT(7)
+
+/* M-PHY Attributes */
+#define MTX_FSM_STATE 0x41
+#define MRX_FSM_STATE 0xC1
+
+/* M-PHY registers */
+#define RX_OVRD_IN_1(n) (0x3006 + ((n) * 0x100))
+#define RX_PCS_OUT(n) (0x300F + ((n) * 0x100))
+#define FAST_FLAGS(n) (0x401C + ((n) * 0x100))
+#define RX_AFE_ATT_IDAC(n) (0x4000 + ((n) * 0x100))
+#define RX_AFE_CTLE_IDAC(n) (0x4001 + ((n) * 0x100))
+#define FW_CALIB_CCFG(n) (0x404D + ((n) * 0x100))
+
+/* Tx/Rx FSM state */
+enum rx_fsm_state {
+ RX_STATE_DISABLED = 0,
+ RX_STATE_HIBERN8 = 1,
+ RX_STATE_SLEEP = 2,
+ RX_STATE_STALL = 3,
+ RX_STATE_LSBURST = 4,
+ RX_STATE_HSBURST = 5,
+};
+
+enum tx_fsm_state {
+ TX_STATE_DISABLED = 0,
+ TX_STATE_HIBERN8 = 1,
+ TX_STATE_SLEEP = 2,
+ TX_STATE_STALL = 3,
+ TX_STATE_LSBURST = 4,
+ TX_STATE_HSBURST = 5,
+};
+
struct ufshcd_dme_attr_val {
u32 attr_sel;
u32 mib_val;
generated by cgit (git 2.34.1) at 2025-12-06 15:45:54 +0000