From 0ab2d7532b05a3e7c06fd3b0c8bd6b46c1dfb508 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Fri, 25 Mar 2022 14:55:30 -0400 Subject: drm/amdkfd: prepare per-process debug enable and disable The ROCm debugger will attach to a process to debug by PTRACE and will expect the KFD to prepare a process for the target PID, whether the target PID has opened the KFD device or not. This patch is to explicity handle this requirement. Further HW mode setting and runtime coordination requirements will be handled in following patches. In the case where the target process has not opened the KFD device, a new KFD process must be created for the target PID. The debugger as well as the target process for this case will have not acquired any VMs so handle process restoration to correctly account for this. To coordinate with HSA runtime, the debugger must be aware of the target process' runtime enablement status and will copy the runtime status information into the debugged KFD process for later query. On enablement, the debugger will subscribe to a set of exceptions where each exception events will notify the debugger through a pollable FIFO file descriptor that the debugger provides to the KFD to manage. Finally on process termination of either the debugger or the target, debugging must be disabled if it has not been done so. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 80 ++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_debug.c (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c new file mode 100644 index 000000000000..898cc1fe3d13 --- /dev/null +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -0,0 +1,80 @@ +/* + * Copyright 2023 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "kfd_debug.h" +#include + +int kfd_dbg_trap_disable(struct kfd_process *target) +{ + if (!target->debug_trap_enabled) + return 0; + + fput(target->dbg_ev_file); + target->dbg_ev_file = NULL; + + if (target->debugger_process) { + atomic_dec(&target->debugger_process->debugged_process_count); + target->debugger_process = NULL; + } + + target->debug_trap_enabled = false; + kfd_unref_process(target); + + return 0; +} + +int kfd_dbg_trap_enable(struct kfd_process *target, uint32_t fd, + void __user *runtime_info, uint32_t *runtime_size) +{ + struct file *f; + uint32_t copy_size; + int r = 0; + + if (target->debug_trap_enabled) + return -EALREADY; + + copy_size = min((size_t)(*runtime_size), sizeof(target->runtime_info)); + + f = fget(fd); + if (!f) { + pr_err("Failed to get file for (%i)\n", fd); + return -EBADF; + } + + target->dbg_ev_file = f; + + /* We already hold the process reference but hold another one for the + * debug session. + */ + kref_get(&target->ref); + target->debug_trap_enabled = true; + + if (target->debugger_process) + atomic_inc(&target->debugger_process->debugged_process_count); + + if (copy_to_user(runtime_info, (void *)&target->runtime_info, copy_size)) + r = -EFAULT; + + *runtime_size = sizeof(target->runtime_info); + + return r; +} -- cgit From 218895820e6fccade42a7c3ab9c0a44dec0a1ebc Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Tue, 5 Apr 2022 12:34:55 -0400 Subject: drm/amdkfd: add per process hw trap enable and disable functions To enable HW debug mode per process, all devices must be debug enabled successfully. If a failure occures, rewind the enablement of debug mode on the enabled devices. A power management scenario that needs to be considered is HW debug mode setting during GFXOFF. During GFXOFF, these registers will be unreachable so we have to transiently disable GFXOFF when setting. Also, some devices don't support the RLC save restore function for these debug registers so we have to disable GFXOFF completely during a debug session. Cooperative launch also has debugging restriction based on HW/FW bugs. If such bugs exists, the debugger cannot attach to a process that uses GWS resources nor can GWS resources be requested if a process is being debugged. Multi-process debug devices can only enable trap temporaries based on certain runtime scenerios, which will be explained when the runtime enable functions are implemented in a follow up patch. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 148 ++++++++++++++++++++++++++++++++- 1 file changed, 146 insertions(+), 2 deletions(-) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 898cc1fe3d13..73b07b5f17f1 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -21,13 +21,78 @@ */ #include "kfd_debug.h" +#include "kfd_device_queue_manager.h" #include +static int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd) +{ + uint32_t spi_dbg_cntl = pdd->spi_dbg_override | pdd->spi_dbg_launch_mode; + uint32_t flags = pdd->process->dbg_flags; + + if (!kfd_dbg_is_per_vmid_supported(pdd->dev)) + return 0; + + return amdgpu_mes_set_shader_debugger(pdd->dev->adev, pdd->proc_ctx_gpu_addr, spi_dbg_cntl, + pdd->watch_points, flags); +} + +/* kfd_dbg_trap_deactivate: + * target: target process + * unwind: If this is unwinding a failed kfd_dbg_trap_enable() + * unwind_count: + * If unwind == true, how far down the pdd list we need + * to unwind + * else: ignored + */ +static void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind_count) +{ + int i; + + for (i = 0; i < target->n_pdds; i++) { + struct kfd_process_device *pdd = target->pdds[i]; + + /* If this is an unwind, and we have unwound the required + * enable calls on the pdd list, we need to stop now + * otherwise we may mess up another debugger session. + */ + if (unwind && i == unwind_count) + break; + + /* GFX off is already disabled by debug activate if not RLC restore supported. */ + if (kfd_dbg_is_rlc_restore_supported(pdd->dev)) + amdgpu_gfx_off_ctrl(pdd->dev->adev, false); + pdd->spi_dbg_override = + pdd->dev->kfd2kgd->disable_debug_trap( + pdd->dev->adev, + target->runtime_info.ttmp_setup, + pdd->dev->vm_info.last_vmid_kfd); + amdgpu_gfx_off_ctrl(pdd->dev->adev, true); + + if (!kfd_dbg_is_per_vmid_supported(pdd->dev) && + release_debug_trap_vmid(pdd->dev->dqm, &pdd->qpd)) + pr_err("Failed to release debug vmid on [%i]\n", pdd->dev->id); + + if (!pdd->dev->kfd->shared_resources.enable_mes) + debug_refresh_runlist(pdd->dev->dqm); + else + kfd_dbg_set_mes_debug_mode(pdd); + } +} + int kfd_dbg_trap_disable(struct kfd_process *target) { if (!target->debug_trap_enabled) return 0; + /* + * Defer deactivation to runtime if runtime not enabled otherwise reset + * attached running target runtime state to enable for re-attach. + */ + if (target->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED) + kfd_dbg_trap_deactivate(target, false, 0); + else if (target->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_DISABLED) + target->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_ENABLED; + fput(target->dbg_ev_file); target->dbg_ev_file = NULL; @@ -42,16 +107,89 @@ int kfd_dbg_trap_disable(struct kfd_process *target) return 0; } +static int kfd_dbg_trap_activate(struct kfd_process *target) +{ + int i, r = 0; + + for (i = 0; i < target->n_pdds; i++) { + struct kfd_process_device *pdd = target->pdds[i]; + + if (!kfd_dbg_is_per_vmid_supported(pdd->dev)) { + r = reserve_debug_trap_vmid(pdd->dev->dqm, &pdd->qpd); + + if (r) { + target->runtime_info.runtime_state = (r == -EBUSY) ? + DEBUG_RUNTIME_STATE_ENABLED_BUSY : + DEBUG_RUNTIME_STATE_ENABLED_ERROR; + + goto unwind_err; + } + } + + /* Disable GFX OFF to prevent garbage read/writes to debug registers. + * If RLC restore of debug registers is not supported and runtime enable + * hasn't done so already on ttmp setup request, restore the trap config registers. + * + * If RLC restore of debug registers is not supported, keep gfx off disabled for + * the debug session. + */ + amdgpu_gfx_off_ctrl(pdd->dev->adev, false); + if (!(kfd_dbg_is_rlc_restore_supported(pdd->dev) || + target->runtime_info.ttmp_setup)) + pdd->dev->kfd2kgd->enable_debug_trap(pdd->dev->adev, true, + pdd->dev->vm_info.last_vmid_kfd); + + pdd->spi_dbg_override = pdd->dev->kfd2kgd->enable_debug_trap( + pdd->dev->adev, + false, + pdd->dev->vm_info.last_vmid_kfd); + + if (kfd_dbg_is_rlc_restore_supported(pdd->dev)) + amdgpu_gfx_off_ctrl(pdd->dev->adev, true); + + if (!pdd->dev->kfd->shared_resources.enable_mes) + r = debug_refresh_runlist(pdd->dev->dqm); + else + r = kfd_dbg_set_mes_debug_mode(pdd); + + if (r) { + target->runtime_info.runtime_state = + DEBUG_RUNTIME_STATE_ENABLED_ERROR; + goto unwind_err; + } + } + + return 0; + +unwind_err: + /* Enabling debug failed, we need to disable on + * all GPUs so the enable is all or nothing. + */ + kfd_dbg_trap_deactivate(target, true, i); + return r; +} + int kfd_dbg_trap_enable(struct kfd_process *target, uint32_t fd, void __user *runtime_info, uint32_t *runtime_size) { struct file *f; uint32_t copy_size; - int r = 0; + int i, r = 0; if (target->debug_trap_enabled) return -EALREADY; + /* Enable pre-checks */ + for (i = 0; i < target->n_pdds; i++) { + struct kfd_process_device *pdd = target->pdds[i]; + + if (!KFD_IS_SOC15(pdd->dev)) + return -ENODEV; + + if (!kfd_dbg_has_gws_support(pdd->dev) && pdd->qpd.num_gws) + return -EBUSY; + } + copy_size = min((size_t)(*runtime_size), sizeof(target->runtime_info)); f = fget(fd); @@ -62,6 +200,10 @@ int kfd_dbg_trap_enable(struct kfd_process *target, uint32_t fd, target->dbg_ev_file = f; + /* defer activation to runtime if not runtime enabled */ + if (target->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED) + kfd_dbg_trap_activate(target); + /* We already hold the process reference but hold another one for the * debug session. */ @@ -71,8 +213,10 @@ int kfd_dbg_trap_enable(struct kfd_process *target, uint32_t fd, if (target->debugger_process) atomic_inc(&target->debugger_process->debugged_process_count); - if (copy_to_user(runtime_info, (void *)&target->runtime_info, copy_size)) + if (copy_to_user(runtime_info, (void *)&target->runtime_info, copy_size)) { + kfd_dbg_trap_deactivate(target, false, 0); r = -EFAULT; + } *runtime_size = sizeof(target->runtime_info); -- cgit From 69a8c3ae2dea84a6d571e4c1aad306f630f3ccfd Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Thu, 1 Sep 2022 11:27:15 -0400 Subject: drm/amdkfd: apply trap workaround for gfx11 Due to a HW bug, waves in only half the shader arrays can enter trap. When starting a debug session, relocate all waves to the first shader array of each shader engine and mask off the 2nd shader array as unavailable. When ending a debug session, re-enable the 2nd shader array per shader engine. User CU masking per queue cannot be guaranteed to remain functional if requested during debugging (e.g. user cu mask requests only 2nd shader array as an available resource leading to zero HW resources available) nor can runtime be alerted of any of these changes during execution. Make user CU masking and debugging mutual exclusive with respect to availability. If the debugger tries to attach to a process with a user cu masked queue, return the runtime status as enabled but busy. If the debugger tries to attach and fails to reallocate queue waves to the first shader array of each shader engine, return the runtime status as enabled but with an error. In addition, like any other mutli-process debug supported devices, disable trap temporary setup per-process to avoid performance impact from setup overhead. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 57 ++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 73b07b5f17f1..5e2ee2d1acc4 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -24,6 +24,57 @@ #include "kfd_device_queue_manager.h" #include +static int kfd_dbg_set_queue_workaround(struct queue *q, bool enable) +{ + struct mqd_update_info minfo = {0}; + int err; + + if (!q) + return 0; + + if (KFD_GC_VERSION(q->device) < IP_VERSION(11, 0, 0) || + KFD_GC_VERSION(q->device) >= IP_VERSION(12, 0, 0)) + return 0; + + if (enable && q->properties.is_user_cu_masked) + return -EBUSY; + + minfo.update_flag = enable ? UPDATE_FLAG_DBG_WA_ENABLE : UPDATE_FLAG_DBG_WA_DISABLE; + + q->properties.is_dbg_wa = enable; + err = q->device->dqm->ops.update_queue(q->device->dqm, q, &minfo); + if (err) + q->properties.is_dbg_wa = false; + + return err; +} + +static int kfd_dbg_set_workaround(struct kfd_process *target, bool enable) +{ + struct process_queue_manager *pqm = &target->pqm; + struct process_queue_node *pqn; + int r = 0; + + list_for_each_entry(pqn, &pqm->queues, process_queue_list) { + r = kfd_dbg_set_queue_workaround(pqn->q, enable); + if (enable && r) + goto unwind; + } + + return 0; + +unwind: + list_for_each_entry(pqn, &pqm->queues, process_queue_list) + kfd_dbg_set_queue_workaround(pqn->q, false); + + if (enable) + target->runtime_info.runtime_state = r == -EBUSY ? + DEBUG_RUNTIME_STATE_ENABLED_BUSY : + DEBUG_RUNTIME_STATE_ENABLED_ERROR; + + return r; +} + static int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd) { uint32_t spi_dbg_cntl = pdd->spi_dbg_override | pdd->spi_dbg_launch_mode; @@ -77,6 +128,8 @@ static void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int else kfd_dbg_set_mes_debug_mode(pdd); } + + kfd_dbg_set_workaround(target, false); } int kfd_dbg_trap_disable(struct kfd_process *target) @@ -111,6 +164,10 @@ static int kfd_dbg_trap_activate(struct kfd_process *target) { int i, r = 0; + r = kfd_dbg_set_workaround(target, true); + if (r) + return r; + for (i = 0; i < target->n_pdds; i++) { struct kfd_process_device *pdd = target->pdds[i]; -- cgit From 44b87bb0836c65d1b9d21b01503eb6e9b9297771 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 6 Apr 2022 12:03:31 -0400 Subject: drm/amdkfd: add raise exception event function Exception events can be generated from interrupts or queue activitity. The raise event function will save exception status of a queue, device or process then notify the debugger of the status change by writing to a debugger polled file descriptor that the debugger provides during debug attach. For memory violation exceptions, extra exception data will be saved. The debugger will be able to query the saved exception states by query operation that will be provided by follow up patches. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 104 +++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 5e2ee2d1acc4..dccb27fc764b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -24,6 +24,107 @@ #include "kfd_device_queue_manager.h" #include +void debug_event_write_work_handler(struct work_struct *work) +{ + struct kfd_process *process; + + static const char write_data = '.'; + loff_t pos = 0; + + process = container_of(work, + struct kfd_process, + debug_event_workarea); + + kernel_write(process->dbg_ev_file, &write_data, 1, &pos); +} + +/* update process/device/queue exception status, write to descriptor + * only if exception_status is enabled. + */ +bool kfd_dbg_ev_raise(uint64_t event_mask, + struct kfd_process *process, struct kfd_node *dev, + unsigned int source_id, bool use_worker, + void *exception_data, size_t exception_data_size) +{ + struct process_queue_manager *pqm; + struct process_queue_node *pqn; + int i; + static const char write_data = '.'; + loff_t pos = 0; + bool is_subscribed = true; + + if (!(process && process->debug_trap_enabled)) + return false; + + mutex_lock(&process->event_mutex); + + if (event_mask & KFD_EC_MASK_DEVICE) { + for (i = 0; i < process->n_pdds; i++) { + struct kfd_process_device *pdd = process->pdds[i]; + + if (pdd->dev != dev) + continue; + + pdd->exception_status |= event_mask & KFD_EC_MASK_DEVICE; + + if (event_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) { + if (!pdd->vm_fault_exc_data) { + pdd->vm_fault_exc_data = kmemdup( + exception_data, + exception_data_size, + GFP_KERNEL); + if (!pdd->vm_fault_exc_data) + pr_debug("Failed to allocate exception data memory"); + } else { + pr_debug("Debugger exception data not saved\n"); + print_hex_dump_bytes("exception data: ", + DUMP_PREFIX_OFFSET, + exception_data, + exception_data_size); + } + } + break; + } + } else if (event_mask & KFD_EC_MASK_PROCESS) { + process->exception_status |= event_mask & KFD_EC_MASK_PROCESS; + } else { + pqm = &process->pqm; + list_for_each_entry(pqn, &pqm->queues, + process_queue_list) { + int target_id; + + if (!pqn->q) + continue; + + target_id = event_mask & KFD_EC_MASK(EC_QUEUE_NEW) ? + pqn->q->properties.queue_id : + pqn->q->doorbell_id; + + if (pqn->q->device != dev || target_id != source_id) + continue; + + pqn->q->properties.exception_status |= event_mask; + break; + } + } + + if (process->exception_enable_mask & event_mask) { + if (use_worker) + schedule_work(&process->debug_event_workarea); + else + kernel_write(process->dbg_ev_file, + &write_data, + 1, + &pos); + } else { + is_subscribed = false; + } + + mutex_unlock(&process->event_mutex); + + return is_subscribed; +} + static int kfd_dbg_set_queue_workaround(struct queue *q, bool enable) { struct mqd_update_info minfo = {0}; @@ -99,6 +200,9 @@ static void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int { int i; + if (!unwind) + cancel_work_sync(&target->debug_event_workarea); + for (i = 0; i < target->n_pdds; i++) { struct kfd_process_device *pdd = target->pdds[i]; -- cgit From c2d2588c702364ff53916ddd97e2b26fd4f4a317 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Fri, 8 Apr 2022 12:49:48 -0400 Subject: drm/amdkfd: add send exception operation Add a debug operation that allows the debugger to send an exception directly to runtime through a payload address. For memory violations, normal vmfault signals will be applied to notify runtime instead after passing in the saved exception data when a memory violation was raised to the debugger. For runtime exceptions, this will unblock the runtime enable function which will be explained and implemented in a follow up patch. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index dccb27fc764b..61098975bb0e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -125,6 +125,49 @@ bool kfd_dbg_ev_raise(uint64_t event_mask, return is_subscribed; } +int kfd_dbg_send_exception_to_runtime(struct kfd_process *p, + unsigned int dev_id, + unsigned int queue_id, + uint64_t error_reason) +{ + if (error_reason & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) { + struct kfd_process_device *pdd = NULL; + struct kfd_hsa_memory_exception_data *data; + int i; + + for (i = 0; i < p->n_pdds; i++) { + if (p->pdds[i]->dev->id == dev_id) { + pdd = p->pdds[i]; + break; + } + } + + if (!pdd) + return -ENODEV; + + data = (struct kfd_hsa_memory_exception_data *) + pdd->vm_fault_exc_data; + + kfd_dqm_evict_pasid(pdd->dev->dqm, p->pasid); + kfd_signal_vm_fault_event(pdd->dev, p->pasid, NULL, data); + error_reason &= ~KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION); + } + + if (error_reason & (KFD_EC_MASK(EC_PROCESS_RUNTIME))) { + /* + * block should only happen after the debugger receives runtime + * enable notice. + */ + up(&p->runtime_enable_sema); + error_reason &= ~KFD_EC_MASK(EC_PROCESS_RUNTIME); + } + + if (error_reason) + return kfd_send_exception_to_runtime(p, queue_id, error_reason); + + return 0; +} + static int kfd_dbg_set_queue_workaround(struct queue *q, bool enable) { struct mqd_update_info minfo = {0}; -- cgit From 455227c4642c5e1867213cea73a527e431779060 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Fri, 8 Apr 2022 13:12:24 -0400 Subject: drm/amdkfd: add runtime enable operation The debugger can attach to a process prior to HSA enablement (i.e. inferior is spawned by the debugger and attached to immediately before target process has been enabled for HSA dispatches) or it can attach to a running target that is already HSA enabled. Either way, the debugger needs to know the enablement status to know when it can inspect queues. For the scenario where the debugger spawns the target process, it will have to wait for ROCr's runtime enable request from the target. The runtime enable request will be able to see that its process has been debug attached. ROCr raises an EC_PROCESS_RUNTIME signal to the debugger then blocks the target process while waiting the debugger's response. Once the debugger has received the runtime signal, it will unblock the target process. For the scenario where the debugger attaches to a running target process, ROCr will set the target process' runtime status as enabled so that on an attach request, the debugger will be able to see this status and will continue with debug enablement as normal. A secondary requirement is to conditionally enable the trap tempories only if the user requests it (env var HSA_ENABLE_DEBUG=1) or if the debugger attaches with HSA runtime enabled. This is because setting up the trap temporaries incurs a performance overhead that is unacceptable for microbench performance in normal mode for certain customers. In the scenario where the debugger spawns the target process, when ROCr detects that the debugger has attached during the runtime enable request, it will enable the trap temporaries before it blocks the target process while waiting for the debugger to respond. In the scenario where the debugger attaches to a running target process, it will enable to trap temporaries itself. Finally, there is an additional restriction that is required to be enforced with runtime enable and HW debug mode setting. The debugger must first ensure that HW debug mode has been enabled before permitting HW debug mode operations. With single process debug devices, allowing the debugger to set debug HW modes prior to trap activation means that debug HW mode setting can occur before the KFD has reserved the debug VMID (0xf) from the hardware scheduler's VMID allocation resource pool. This can result in the hardware scheduler assigning VMID 0xf to a non-debugged process and having that process inherit debug HW mode settings intended for the debugged target process instead, which is both incorrect and potentially fatal for normal mode operation. With multi process debug devices, allowing the debugger to set debug HW modes prior to trap activation means that non-debugged processes migrating to a new VMID could inherit unintended debug settings. All debug operations that touch HW settings must require trap activation where trap activation is triggered by both debug attach and runtime enablement (target has KFD opened and is ready to dispatch work). Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 61098975bb0e..a19c21d04438 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -219,7 +219,7 @@ unwind: return r; } -static int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd) +int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd) { uint32_t spi_dbg_cntl = pdd->spi_dbg_override | pdd->spi_dbg_launch_mode; uint32_t flags = pdd->process->dbg_flags; @@ -239,7 +239,7 @@ static int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd) * to unwind * else: ignored */ -static void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind_count) +void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind_count) { int i; @@ -307,7 +307,7 @@ int kfd_dbg_trap_disable(struct kfd_process *target) return 0; } -static int kfd_dbg_trap_activate(struct kfd_process *target) +int kfd_dbg_trap_activate(struct kfd_process *target) { int i, r = 0; -- cgit From 50cff45e274896235d371f16eab67a180e12a732 Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Mon, 1 Mar 2021 18:34:39 -0600 Subject: drm/amdkfd: add debug trap enabled flag to tma Trap handler behavior will differ when a debugger is attached. Make the debug trap flag available in the trap handler TMA. Update it when the debug trap ioctl is invoked. Signed-off-by: Jay Cornwall Reviewed-by: Felix Kuehling Signed-off-by: Jonathan Kim Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index a19c21d04438..17e8e9edccbf 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -256,6 +256,8 @@ void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind if (unwind && i == unwind_count) break; + kfd_process_set_trap_debug_flag(&pdd->qpd, false); + /* GFX off is already disabled by debug activate if not RLC restore supported. */ if (kfd_dbg_is_rlc_restore_supported(pdd->dev)) amdgpu_gfx_off_ctrl(pdd->dev->adev, false); @@ -351,6 +353,15 @@ int kfd_dbg_trap_activate(struct kfd_process *target) if (kfd_dbg_is_rlc_restore_supported(pdd->dev)) amdgpu_gfx_off_ctrl(pdd->dev->adev, true); + /* + * Setting the debug flag in the trap handler requires that the TMA has been + * allocated, which occurs during CWSR initialization. + * In the event that CWSR has not been initialized at this point, setting the + * flag will be called again during CWSR initialization if the target process + * is still debug enabled. + */ + kfd_process_set_trap_debug_flag(&pdd->qpd, true); + if (!pdd->dev->kfd->shared_resources.enable_mes) r = debug_refresh_runlist(pdd->dev->dqm); else -- cgit From 12fb1ad70d65edc3405884792d044fa79df7244f Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Fri, 22 Apr 2022 12:26:18 -0400 Subject: drm/amdkfd: update process interrupt handling for debug events The debugger must be notified by any debugger subscribed exception that comes from hardware interrupts. If a debugger session exits, any exceptions it subscribed to may still have interrupts in the interrupt ring buffer or KGD/KFD pipeline. To prevent a new session from inheriting stale interrupts, when a new queue is created, open an interrupt drain and allow the IH ring to drain from a timestamped checkpoint. Then inject a custom IV so that once the custom IV is picked up by the KFD, it's safe to close the drain and proceed with queue creation. The drain must also be on debug disable as SW interrupts may still be processed. Drain at this time and clear all the exception status. The debugger may also not be attached nor subscibed to certain exceptions so forward them directly to the runtime. GFX10 also requires its own IV processing, hence the creation of kfd_int_process_v10.c. This is because the IV from SQ interrupts are packed into a new continguous format unlike GFX9. To make this clear, a separate interrupting handling code file was created. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 84 ++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 17e8e9edccbf..68b657398d41 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -125,6 +125,64 @@ bool kfd_dbg_ev_raise(uint64_t event_mask, return is_subscribed; } +/* set pending event queue entry from ring entry */ +bool kfd_set_dbg_ev_from_interrupt(struct kfd_node *dev, + unsigned int pasid, + uint32_t doorbell_id, + uint64_t trap_mask, + void *exception_data, + size_t exception_data_size) +{ + struct kfd_process *p; + bool signaled_to_debugger_or_runtime = false; + + p = kfd_lookup_process_by_pasid(pasid); + + if (!p) + return false; + + if (!kfd_dbg_ev_raise(trap_mask, p, dev, doorbell_id, true, + exception_data, exception_data_size)) { + struct process_queue_manager *pqm; + struct process_queue_node *pqn; + + if (!!(trap_mask & KFD_EC_MASK_QUEUE) && + p->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED) { + mutex_lock(&p->mutex); + + pqm = &p->pqm; + list_for_each_entry(pqn, &pqm->queues, + process_queue_list) { + + if (!(pqn->q && pqn->q->device == dev && + pqn->q->doorbell_id == doorbell_id)) + continue; + + kfd_send_exception_to_runtime(p, pqn->q->properties.queue_id, + trap_mask); + + signaled_to_debugger_or_runtime = true; + + break; + } + + mutex_unlock(&p->mutex); + } else if (trap_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) { + kfd_dqm_evict_pasid(dev->dqm, p->pasid); + kfd_signal_vm_fault_event(dev, p->pasid, NULL, + exception_data); + + signaled_to_debugger_or_runtime = true; + } + } else { + signaled_to_debugger_or_runtime = true; + } + + kfd_unref_process(p); + + return signaled_to_debugger_or_runtime; +} + int kfd_dbg_send_exception_to_runtime(struct kfd_process *p, unsigned int dev_id, unsigned int queue_id, @@ -281,6 +339,31 @@ void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind kfd_dbg_set_workaround(target, false); } +static void kfd_dbg_clean_exception_status(struct kfd_process *target) +{ + struct process_queue_manager *pqm; + struct process_queue_node *pqn; + int i; + + for (i = 0; i < target->n_pdds; i++) { + struct kfd_process_device *pdd = target->pdds[i]; + + kfd_process_drain_interrupts(pdd); + + pdd->exception_status = 0; + } + + pqm = &target->pqm; + list_for_each_entry(pqn, &pqm->queues, process_queue_list) { + if (!pqn->q) + continue; + + pqn->q->properties.exception_status = 0; + } + + target->exception_status = 0; +} + int kfd_dbg_trap_disable(struct kfd_process *target) { if (!target->debug_trap_enabled) @@ -304,6 +387,7 @@ int kfd_dbg_trap_disable(struct kfd_process *target) } target->debug_trap_enabled = false; + kfd_dbg_clean_exception_status(target); kfd_unref_process(target); return 0; -- cgit From e90bf919f714ae2a658cdfd03238e7be9ce9185c Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 27 Apr 2022 10:24:37 -0400 Subject: drm/amdkfd: add debug set exceptions enabled operation The debugger subscibes to nofication for requested exceptions on attach. Allow the debugger to change its subsciption later on. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 68b657398d41..48a4e3cc2234 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -521,3 +521,39 @@ int kfd_dbg_trap_enable(struct kfd_process *target, uint32_t fd, return r; } + +void kfd_dbg_set_enabled_debug_exception_mask(struct kfd_process *target, + uint64_t exception_set_mask) +{ + uint64_t found_mask = 0; + struct process_queue_manager *pqm; + struct process_queue_node *pqn; + static const char write_data = '.'; + loff_t pos = 0; + int i; + + mutex_lock(&target->event_mutex); + + found_mask |= target->exception_status; + + pqm = &target->pqm; + list_for_each_entry(pqn, &pqm->queues, process_queue_list) { + if (!pqn) + continue; + + found_mask |= pqn->q->properties.exception_status; + } + + for (i = 0; i < target->n_pdds; i++) { + struct kfd_process_device *pdd = target->pdds[i]; + + found_mask |= pdd->exception_status; + } + + if (exception_set_mask & found_mask) + kernel_write(target->dbg_ev_file, &write_data, 1, &pos); + + target->exception_enable_mask = exception_set_mask; + + mutex_unlock(&target->event_mutex); +} -- cgit From 101827e13026a981e887527620fe9710adc0e481 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 27 Apr 2022 13:18:10 -0400 Subject: drm/amdkfd: add debug wave launch override operation This operation allows the debugger to override the enabled HW exceptions on the device. On debug devices that only support the debugging of a single process, the HW exceptions are global and set through the SPI_GDBG_TRAP_MASK register. Because they are global, only address watch exceptions are allowed to be enabled. In other words, the debugger must preserve all non-address watch exception states in normal mode operation by barring a full replacement override or a non-address watch override request. For multi-process debugging, all HW exception overrides are per-VMID so all exceptions can be overridden or fully replaced. In order for the debugger to know what is permissible, returned the supported override mask back to the debugger along with the previously enable overrides. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 69 ++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 48a4e3cc2234..733390fb2459 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -522,6 +522,75 @@ int kfd_dbg_trap_enable(struct kfd_process *target, uint32_t fd, return r; } +static int kfd_dbg_validate_trap_override_request(struct kfd_process *p, + uint32_t trap_override, + uint32_t trap_mask_request, + uint32_t *trap_mask_supported) +{ + int i = 0; + + *trap_mask_supported = 0xffffffff; + + for (i = 0; i < p->n_pdds; i++) { + struct kfd_process_device *pdd = p->pdds[i]; + int err = pdd->dev->kfd2kgd->validate_trap_override_request( + pdd->dev->adev, + trap_override, + trap_mask_supported); + + if (err) + return err; + } + + if (trap_mask_request & ~*trap_mask_supported) + return -EACCES; + + return 0; +} + +int kfd_dbg_trap_set_wave_launch_override(struct kfd_process *target, + uint32_t trap_override, + uint32_t trap_mask_bits, + uint32_t trap_mask_request, + uint32_t *trap_mask_prev, + uint32_t *trap_mask_supported) +{ + int r = 0, i; + + r = kfd_dbg_validate_trap_override_request(target, + trap_override, + trap_mask_request, + trap_mask_supported); + + if (r) + return r; + + for (i = 0; i < target->n_pdds; i++) { + struct kfd_process_device *pdd = target->pdds[i]; + + amdgpu_gfx_off_ctrl(pdd->dev->adev, false); + pdd->spi_dbg_override = pdd->dev->kfd2kgd->set_wave_launch_trap_override( + pdd->dev->adev, + pdd->dev->vm_info.last_vmid_kfd, + trap_override, + trap_mask_bits, + trap_mask_request, + trap_mask_prev, + pdd->spi_dbg_override); + amdgpu_gfx_off_ctrl(pdd->dev->adev, true); + + if (!pdd->dev->kfd->shared_resources.enable_mes) + r = debug_refresh_runlist(pdd->dev->dqm); + else + r = kfd_dbg_set_mes_debug_mode(pdd); + + if (r) + break; + } + + return r; +} + void kfd_dbg_set_enabled_debug_exception_mask(struct kfd_process *target, uint64_t exception_set_mask) { -- cgit From aea1b4738bebd8092bd437ce0b03aa9587fc20a7 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 2 May 2022 11:45:05 -0400 Subject: drm/amdkfd: add debug wave launch mode operation Allow the debugger to set wave behaviour on to either normally operate, halt at launch, trap on every instruction, terminate immediately or stall on allocation. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 36 +++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 733390fb2459..53c3418562d4 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -301,8 +301,10 @@ void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind { int i; - if (!unwind) + if (!unwind) { cancel_work_sync(&target->debug_event_workarea); + kfd_dbg_trap_set_wave_launch_mode(target, 0); + } for (i = 0; i < target->n_pdds; i++) { struct kfd_process_device *pdd = target->pdds[i]; @@ -591,6 +593,38 @@ int kfd_dbg_trap_set_wave_launch_override(struct kfd_process *target, return r; } +int kfd_dbg_trap_set_wave_launch_mode(struct kfd_process *target, + uint8_t wave_launch_mode) +{ + int r = 0, i; + + if (wave_launch_mode != KFD_DBG_TRAP_WAVE_LAUNCH_MODE_NORMAL && + wave_launch_mode != KFD_DBG_TRAP_WAVE_LAUNCH_MODE_HALT && + wave_launch_mode != KFD_DBG_TRAP_WAVE_LAUNCH_MODE_DEBUG) + return -EINVAL; + + for (i = 0; i < target->n_pdds; i++) { + struct kfd_process_device *pdd = target->pdds[i]; + + amdgpu_gfx_off_ctrl(pdd->dev->adev, false); + pdd->spi_dbg_launch_mode = pdd->dev->kfd2kgd->set_wave_launch_mode( + pdd->dev->adev, + wave_launch_mode, + pdd->dev->vm_info.last_vmid_kfd); + amdgpu_gfx_off_ctrl(pdd->dev->adev, true); + + if (!pdd->dev->kfd->shared_resources.enable_mes) + r = debug_refresh_runlist(pdd->dev->dqm); + else + r = kfd_dbg_set_mes_debug_mode(pdd); + + if (r) + break; + } + + return r; +} + void kfd_dbg_set_enabled_debug_exception_mask(struct kfd_process *target, uint64_t exception_set_mask) { -- cgit From a70a93fa568b4f05aba548dadb673703eccf5480 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Thu, 5 May 2022 16:15:37 -0400 Subject: drm/amdkfd: add debug suspend and resume process queues operation In order to inspect waves from the saved context at any point during a debug session, the debugger must be able to preempt queues to trigger context save by suspending them. On queue suspend, the KFD will copy the context save header information so that the debugger can correctly crawl the appropriate size of the saved context. The debugger must then also be allowed to resume suspended queues. A queue that is newly created cannot be suspended because queue ids are recycled after destruction so the debugger needs to know that this has occurred. Query functions will be later added that will clear a given queue of its new queue status. A queue cannot be destroyed while it is suspended to preserve its saved context during debugger inspection. Have queue destruction block while a queue is suspended and unblocked when it is resumed. Likewise, if a queue is about to be destroyed, it cannot be suspended. Return the number of queues successfully suspended or resumed along with a per queue status array where the upper bits per queue status show that the request was invalid (new/destroyed queue suspend request, missing queue) or an error occurred (HWS in a fatal state so it can't suspend or resume queues). Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 53c3418562d4..f4d3dfb35cb3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -339,6 +339,13 @@ void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind } kfd_dbg_set_workaround(target, false); + + if (!unwind) { + int resume_count = resume_queues(target, 0, NULL); + + if (resume_count) + pr_debug("Resumed %d queues\n", resume_count); + } } static void kfd_dbg_clean_exception_status(struct kfd_process *target) -- cgit From e0f85f4690d089cc1a60337decafb1acf7eec45e Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Fri, 6 May 2022 14:58:55 -0400 Subject: drm/amdkfd: add debug set and clear address watch points operation Shader read, write and atomic memory operations can be alerted to the debugger as an address watch exception. Allow the debugger to pass in a watch point to a particular memory address per device. Note that there exists only 4 watch points per devices to date, so have the KFD keep track of what watch points are allocated or not. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 136 +++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index f4d3dfb35cb3..4b36cc8b5fb7 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -24,6 +24,8 @@ #include "kfd_device_queue_manager.h" #include +#define MAX_WATCH_ADDRESSES 4 + void debug_event_write_work_handler(struct work_struct *work) { struct kfd_process *process; @@ -289,6 +291,139 @@ int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd) pdd->watch_points, flags); } +#define KFD_DEBUGGER_INVALID_WATCH_POINT_ID -1 +static int kfd_dbg_get_dev_watch_id(struct kfd_process_device *pdd, int *watch_id) +{ + int i; + + *watch_id = KFD_DEBUGGER_INVALID_WATCH_POINT_ID; + + spin_lock(&pdd->dev->kfd->watch_points_lock); + + for (i = 0; i < MAX_WATCH_ADDRESSES; i++) { + /* device watchpoint in use so skip */ + if ((pdd->dev->kfd->alloc_watch_ids >> i) & 0x1) + continue; + + pdd->alloc_watch_ids |= 0x1 << i; + pdd->dev->kfd->alloc_watch_ids |= 0x1 << i; + *watch_id = i; + spin_unlock(&pdd->dev->kfd->watch_points_lock); + return 0; + } + + spin_unlock(&pdd->dev->kfd->watch_points_lock); + + return -ENOMEM; +} + +static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, int watch_id) +{ + spin_lock(&pdd->dev->kfd->watch_points_lock); + + /* process owns device watch point so safe to clear */ + if ((pdd->alloc_watch_ids >> watch_id) & 0x1) { + pdd->alloc_watch_ids &= ~(0x1 << watch_id); + pdd->dev->kfd->alloc_watch_ids &= ~(0x1 << watch_id); + } + + spin_unlock(&pdd->dev->kfd->watch_points_lock); +} + +static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, int watch_id) +{ + bool owns_watch_id = false; + + spin_lock(&pdd->dev->kfd->watch_points_lock); + owns_watch_id = watch_id < MAX_WATCH_ADDRESSES && + ((pdd->alloc_watch_ids >> watch_id) & 0x1); + + spin_unlock(&pdd->dev->kfd->watch_points_lock); + + return owns_watch_id; +} + +int kfd_dbg_trap_clear_dev_address_watch(struct kfd_process_device *pdd, + uint32_t watch_id) +{ + int r; + + if (!kfd_dbg_owns_dev_watch_id(pdd, watch_id)) + return -EINVAL; + + if (!pdd->dev->kfd->shared_resources.enable_mes) { + r = debug_lock_and_unmap(pdd->dev->dqm); + if (r) + return r; + } + + amdgpu_gfx_off_ctrl(pdd->dev->adev, false); + pdd->watch_points[watch_id] = pdd->dev->kfd2kgd->clear_address_watch( + pdd->dev->adev, + watch_id); + amdgpu_gfx_off_ctrl(pdd->dev->adev, true); + + if (!pdd->dev->kfd->shared_resources.enable_mes) + r = debug_map_and_unlock(pdd->dev->dqm); + else + r = kfd_dbg_set_mes_debug_mode(pdd); + + kfd_dbg_clear_dev_watch_id(pdd, watch_id); + + return r; +} + +int kfd_dbg_trap_set_dev_address_watch(struct kfd_process_device *pdd, + uint64_t watch_address, + uint32_t watch_address_mask, + uint32_t *watch_id, + uint32_t watch_mode) +{ + int r = kfd_dbg_get_dev_watch_id(pdd, watch_id); + + if (r) + return r; + + if (!pdd->dev->kfd->shared_resources.enable_mes) { + r = debug_lock_and_unmap(pdd->dev->dqm); + if (r) { + kfd_dbg_clear_dev_watch_id(pdd, *watch_id); + return r; + } + } + + amdgpu_gfx_off_ctrl(pdd->dev->adev, false); + pdd->watch_points[*watch_id] = pdd->dev->kfd2kgd->set_address_watch( + pdd->dev->adev, + watch_address, + watch_address_mask, + *watch_id, + watch_mode, + pdd->dev->vm_info.last_vmid_kfd); + amdgpu_gfx_off_ctrl(pdd->dev->adev, true); + + if (!pdd->dev->kfd->shared_resources.enable_mes) + r = debug_map_and_unlock(pdd->dev->dqm); + else + r = kfd_dbg_set_mes_debug_mode(pdd); + + /* HWS is broken so no point in HW rollback but release the watchpoint anyways */ + if (r) + kfd_dbg_clear_dev_watch_id(pdd, *watch_id); + + return 0; +} + +static void kfd_dbg_clear_process_address_watch(struct kfd_process *target) +{ + int i, j; + + for (i = 0; i < target->n_pdds; i++) + for (j = 0; j < MAX_WATCH_ADDRESSES; j++) + kfd_dbg_trap_clear_dev_address_watch(target->pdds[i], j); +} + + /* kfd_dbg_trap_deactivate: * target: target process * unwind: If this is unwinding a failed kfd_dbg_trap_enable() @@ -303,6 +438,7 @@ void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind if (!unwind) { cancel_work_sync(&target->debug_event_workarea); + kfd_dbg_clear_process_address_watch(target); kfd_dbg_trap_set_wave_launch_mode(target, 0); } -- cgit From 103d5f08ff42b666c61c350be2c3e724c1646918 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 9 May 2022 10:51:56 -0400 Subject: drm/amdkfd: add debug set flags operation Allow the debugger to set single memory and single ALU operations. Some exceptions are imprecise (memory violations, address watch) in the sense that a trap occurs only when the exception interrupt occurs and not at the non-halting faulty instruction. Trap temporaries 0 & 1 save the program counter address, which means that these values will not point to the faulty instruction address but to whenever the interrupt was raised. Setting the Single Memory Operations flag will inject an automatic wait on every memory operation instruction forcing imprecise memory exceptions to become precise at the cost of performance. This setting is not permitted on debug devices that support only a global setting of this option. Return the previous set flags to the debugger as well. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 58 ++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 4b36cc8b5fb7..43c3170998d3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -23,6 +23,7 @@ #include "kfd_debug.h" #include "kfd_device_queue_manager.h" #include +#include #define MAX_WATCH_ADDRESSES 4 @@ -423,6 +424,59 @@ static void kfd_dbg_clear_process_address_watch(struct kfd_process *target) kfd_dbg_trap_clear_dev_address_watch(target->pdds[i], j); } +int kfd_dbg_trap_set_flags(struct kfd_process *target, uint32_t *flags) +{ + uint32_t prev_flags = target->dbg_flags; + int i, r = 0, rewind_count = 0; + + for (i = 0; i < target->n_pdds; i++) { + if (!kfd_dbg_is_per_vmid_supported(target->pdds[i]->dev) && + (*flags & KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP)) { + *flags = prev_flags; + return -EACCES; + } + } + + target->dbg_flags = *flags & KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP; + *flags = prev_flags; + for (i = 0; i < target->n_pdds; i++) { + struct kfd_process_device *pdd = target->pdds[i]; + + if (!kfd_dbg_is_per_vmid_supported(pdd->dev)) + continue; + + if (!pdd->dev->kfd->shared_resources.enable_mes) + r = debug_refresh_runlist(pdd->dev->dqm); + else + r = kfd_dbg_set_mes_debug_mode(pdd); + + if (r) { + target->dbg_flags = prev_flags; + break; + } + + rewind_count++; + } + + /* Rewind flags */ + if (r) { + target->dbg_flags = prev_flags; + + for (i = 0; i < rewind_count; i++) { + struct kfd_process_device *pdd = target->pdds[i]; + + if (!kfd_dbg_is_per_vmid_supported(pdd->dev)) + continue; + + if (!pdd->dev->kfd->shared_resources.enable_mes) + debug_refresh_runlist(pdd->dev->dqm); + else + kfd_dbg_set_mes_debug_mode(pdd); + } + } + + return r; +} /* kfd_dbg_trap_deactivate: * target: target process @@ -437,9 +491,13 @@ void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind int i; if (!unwind) { + uint32_t flags = 0; + cancel_work_sync(&target->debug_event_workarea); kfd_dbg_clear_process_address_watch(target); kfd_dbg_trap_set_wave_launch_mode(target, 0); + + kfd_dbg_trap_set_flags(target, &flags); } for (i = 0; i < target->n_pdds; i++) { -- cgit From 5bc20c224bcb863571e8831cdbba23cd61b10ac3 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 9 May 2022 11:10:32 -0400 Subject: drm/amdkfd: add debug query event operation Allow the debugger to query a single queue, device and process exception. The KFD should also return the GPU or Queue id of the exception. The debugger also has the option of clearing exceptions after being queried. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 43c3170998d3..e9530e682e85 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -27,6 +27,70 @@ #define MAX_WATCH_ADDRESSES 4 +int kfd_dbg_ev_query_debug_event(struct kfd_process *process, + unsigned int *queue_id, + unsigned int *gpu_id, + uint64_t exception_clear_mask, + uint64_t *event_status) +{ + struct process_queue_manager *pqm; + struct process_queue_node *pqn; + int i; + + if (!(process && process->debug_trap_enabled)) + return -ENODATA; + + mutex_lock(&process->event_mutex); + *event_status = 0; + *queue_id = 0; + *gpu_id = 0; + + /* find and report queue events */ + pqm = &process->pqm; + list_for_each_entry(pqn, &pqm->queues, process_queue_list) { + uint64_t tmp = process->exception_enable_mask; + + if (!pqn->q) + continue; + + tmp &= pqn->q->properties.exception_status; + + if (!tmp) + continue; + + *event_status = pqn->q->properties.exception_status; + *queue_id = pqn->q->properties.queue_id; + *gpu_id = pqn->q->device->id; + pqn->q->properties.exception_status &= ~exception_clear_mask; + goto out; + } + + /* find and report device events */ + for (i = 0; i < process->n_pdds; i++) { + struct kfd_process_device *pdd = process->pdds[i]; + uint64_t tmp = process->exception_enable_mask + & pdd->exception_status; + + if (!tmp) + continue; + + *event_status = pdd->exception_status; + *gpu_id = pdd->dev->id; + pdd->exception_status &= ~exception_clear_mask; + goto out; + } + + /* report process events */ + if (process->exception_enable_mask & process->exception_status) { + *event_status = process->exception_status; + process->exception_status &= ~exception_clear_mask; + } + +out: + mutex_unlock(&process->event_mutex); + return *event_status ? 0 : -EAGAIN; +} + void debug_event_write_work_handler(struct work_struct *work) { struct kfd_process *process; -- cgit From 2b36de971d25daa2ad287114ae3ca11a8f8d49d7 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 9 May 2022 13:37:36 -0400 Subject: drm/amdkfd: add debug query exception info operation Allow the debugger to query additional info based on an exception code. For device exceptions, it's currently only memory violation information. For process exceptions, it's currently only runtime information. Queue exception only report the queue exception status. The debugger has the option of clearing the target exception on query. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 120 +++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index e9530e682e85..24e2b285448a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -890,6 +890,126 @@ int kfd_dbg_trap_set_wave_launch_mode(struct kfd_process *target, return r; } +int kfd_dbg_trap_query_exception_info(struct kfd_process *target, + uint32_t source_id, + uint32_t exception_code, + bool clear_exception, + void __user *info, + uint32_t *info_size) +{ + bool found = false; + int r = 0; + uint32_t copy_size, actual_info_size = 0; + uint64_t *exception_status_ptr = NULL; + + if (!target) + return -EINVAL; + + if (!info || !info_size) + return -EINVAL; + + mutex_lock(&target->event_mutex); + + if (KFD_DBG_EC_TYPE_IS_QUEUE(exception_code)) { + /* Per queue exceptions */ + struct queue *queue = NULL; + int i; + + for (i = 0; i < target->n_pdds; i++) { + struct kfd_process_device *pdd = target->pdds[i]; + struct qcm_process_device *qpd = &pdd->qpd; + + list_for_each_entry(queue, &qpd->queues_list, list) { + if (!found && queue->properties.queue_id == source_id) { + found = true; + break; + } + } + if (found) + break; + } + + if (!found) { + r = -EINVAL; + goto out; + } + + if (!(queue->properties.exception_status & KFD_EC_MASK(exception_code))) { + r = -ENODATA; + goto out; + } + exception_status_ptr = &queue->properties.exception_status; + } else if (KFD_DBG_EC_TYPE_IS_DEVICE(exception_code)) { + /* Per device exceptions */ + struct kfd_process_device *pdd = NULL; + int i; + + for (i = 0; i < target->n_pdds; i++) { + pdd = target->pdds[i]; + if (pdd->dev->id == source_id) { + found = true; + break; + } + } + + if (!found) { + r = -EINVAL; + goto out; + } + + if (!(pdd->exception_status & KFD_EC_MASK(exception_code))) { + r = -ENODATA; + goto out; + } + + if (exception_code == EC_DEVICE_MEMORY_VIOLATION) { + copy_size = min((size_t)(*info_size), pdd->vm_fault_exc_data_size); + + if (copy_to_user(info, pdd->vm_fault_exc_data, copy_size)) { + r = -EFAULT; + goto out; + } + actual_info_size = pdd->vm_fault_exc_data_size; + if (clear_exception) { + kfree(pdd->vm_fault_exc_data); + pdd->vm_fault_exc_data = NULL; + pdd->vm_fault_exc_data_size = 0; + } + } + exception_status_ptr = &pdd->exception_status; + } else if (KFD_DBG_EC_TYPE_IS_PROCESS(exception_code)) { + /* Per process exceptions */ + if (!(target->exception_status & KFD_EC_MASK(exception_code))) { + r = -ENODATA; + goto out; + } + + if (exception_code == EC_PROCESS_RUNTIME) { + copy_size = min((size_t)(*info_size), sizeof(target->runtime_info)); + + if (copy_to_user(info, (void *)&target->runtime_info, copy_size)) { + r = -EFAULT; + goto out; + } + + actual_info_size = sizeof(target->runtime_info); + } + + exception_status_ptr = &target->exception_status; + } else { + pr_debug("Bad exception type [%i]\n", exception_code); + r = -EINVAL; + goto out; + } + + *info_size = actual_info_size; + if (clear_exception) + *exception_status_ptr &= ~KFD_EC_MASK(exception_code); +out: + mutex_unlock(&target->event_mutex); + return r; +} + void kfd_dbg_set_enabled_debug_exception_mask(struct kfd_process *target, uint64_t exception_set_mask) { -- cgit From 12976e6a5ab8fc3766c0304d72f7eec81a109b55 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Tue, 10 May 2022 12:47:45 -0400 Subject: drm/amdkfd: add debug device snapshot operation Similar to queue snapshot, return an array of device information using an entry_size check and return. Unlike queue snapshots, the debugger needs to pass to correct number of devices that exist. If it fails to do so, the KFD will return the number of actual devices so that the debugger can make a subsequent successful call. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 24e2b285448a..125274445f43 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -22,6 +22,7 @@ #include "kfd_debug.h" #include "kfd_device_queue_manager.h" +#include "kfd_topology.h" #include #include @@ -1010,6 +1011,78 @@ out: return r; } +int kfd_dbg_trap_device_snapshot(struct kfd_process *target, + uint64_t exception_clear_mask, + void __user *user_info, + uint32_t *number_of_device_infos, + uint32_t *entry_size) +{ + struct kfd_dbg_device_info_entry device_info; + uint32_t tmp_entry_size = *entry_size, tmp_num_devices; + int i, r = 0; + + if (!(target && user_info && number_of_device_infos && entry_size)) + return -EINVAL; + + tmp_num_devices = min_t(size_t, *number_of_device_infos, target->n_pdds); + *number_of_device_infos = target->n_pdds; + *entry_size = min_t(size_t, *entry_size, sizeof(device_info)); + + if (!tmp_num_devices) + return 0; + + memset(&device_info, 0, sizeof(device_info)); + + mutex_lock(&target->event_mutex); + + /* Run over all pdd of the process */ + for (i = 0; i < tmp_num_devices; i++) { + struct kfd_process_device *pdd = target->pdds[i]; + struct kfd_topology_device *topo_dev = kfd_topology_device_by_id(pdd->dev->id); + + device_info.gpu_id = pdd->dev->id; + device_info.exception_status = pdd->exception_status; + device_info.lds_base = pdd->lds_base; + device_info.lds_limit = pdd->lds_limit; + device_info.scratch_base = pdd->scratch_base; + device_info.scratch_limit = pdd->scratch_limit; + device_info.gpuvm_base = pdd->gpuvm_base; + device_info.gpuvm_limit = pdd->gpuvm_limit; + device_info.location_id = topo_dev->node_props.location_id; + device_info.vendor_id = topo_dev->node_props.vendor_id; + device_info.device_id = topo_dev->node_props.device_id; + device_info.revision_id = pdd->dev->adev->pdev->revision; + device_info.subsystem_vendor_id = pdd->dev->adev->pdev->subsystem_vendor; + device_info.subsystem_device_id = pdd->dev->adev->pdev->subsystem_device; + device_info.fw_version = pdd->dev->kfd->mec_fw_version; + device_info.gfx_target_version = + topo_dev->node_props.gfx_target_version; + device_info.simd_count = topo_dev->node_props.simd_count; + device_info.max_waves_per_simd = + topo_dev->node_props.max_waves_per_simd; + device_info.array_count = topo_dev->node_props.array_count; + device_info.simd_arrays_per_engine = + topo_dev->node_props.simd_arrays_per_engine; + device_info.num_xcc = NUM_XCC(pdd->dev->xcc_mask); + device_info.capability = topo_dev->node_props.capability; + device_info.debug_prop = topo_dev->node_props.debug_prop; + + if (exception_clear_mask) + pdd->exception_status &= ~exception_clear_mask; + + if (copy_to_user(user_info, &device_info, *entry_size)) { + r = -EFAULT; + break; + } + + user_info += tmp_entry_size; + } + + mutex_unlock(&target->event_mutex); + + return r; +} + void kfd_dbg_set_enabled_debug_exception_mask(struct kfd_process *target, uint64_t exception_set_mask) { -- cgit From 09d49e14ea6fd125a21f89b80f888c09be32a174 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Tue, 23 May 2023 11:57:27 -0400 Subject: drm/amdkfd: fix and enable debugging for gfx11 There are a couple of fixes required to enable gfx11 debugging. First, ADD_QUEUE.trap_en is an inappropriate place to toggle a per-process register so move it to SET_SHADER_DEBUGGER.trap_en. When ADD_QUEUE.skip_process_ctx_clear is set, MES will prioritize the SET_SHADER_DEBUGGER.trap_en setting. Second, to preserve correct save/restore priviledged wave states in coordination with the trap enablement setting, resume suspended waves early in the disable call. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_debug.c') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 125274445f43..cd34e7aaead4 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -349,12 +349,13 @@ int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd) { uint32_t spi_dbg_cntl = pdd->spi_dbg_override | pdd->spi_dbg_launch_mode; uint32_t flags = pdd->process->dbg_flags; + bool sq_trap_en = !!spi_dbg_cntl; if (!kfd_dbg_is_per_vmid_supported(pdd->dev)) return 0; return amdgpu_mes_set_shader_debugger(pdd->dev->adev, pdd->proc_ctx_gpu_addr, spi_dbg_cntl, - pdd->watch_points, flags); + pdd->watch_points, flags, sq_trap_en); } #define KFD_DEBUGGER_INVALID_WATCH_POINT_ID -1 @@ -557,6 +558,10 @@ void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind if (!unwind) { uint32_t flags = 0; + int resume_count = resume_queues(target, 0, NULL); + + if (resume_count) + pr_debug("Resumed %d queues\n", resume_count); cancel_work_sync(&target->debug_event_workarea); kfd_dbg_clear_process_address_watch(target); @@ -598,13 +603,6 @@ void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind } kfd_dbg_set_workaround(target, false); - - if (!unwind) { - int resume_count = resume_queues(target, 0, NULL); - - if (resume_count) - pr_debug("Resumed %d queues\n", resume_count); - } } static void kfd_dbg_clean_exception_status(struct kfd_process *target) -- cgit