From a957cbc02531a23beeac6dd9e751f8d4dadaf7a9 Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Tue, 6 Jun 2023 18:12:15 +0200 Subject: rtla: Add -C cgroup support The -C option sets a cgroup to the tracer's threads. If the -C option is passed without arguments, the tracer's thread will inherit rtla's cgroup. Otherwise, the threads will be placed on the cgroup passed to the option. Link: https://lkml.kernel.org/r/cb051477331d292f17c08bf1d66f0e0384bbe5a5.1686066600.git.bristot@kernel.org Cc: William White Cc: Jonathan Corbet Tested-by: Juri Lelli Signed-off-by: Daniel Bristot de Oliveira Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/utils.c | 185 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) (limited to 'tools/tracing/rtla/src/utils.c') diff --git a/tools/tracing/rtla/src/utils.c b/tools/tracing/rtla/src/utils.c index 663a047f794d..bcc0a9f39cfe 100644 --- a/tools/tracing/rtla/src/utils.c +++ b/tools/tracing/rtla/src/utils.c @@ -529,3 +529,188 @@ int set_cpu_dma_latency(int32_t latency) return fd; } + +#define _STR(x) #x +#define STR(x) _STR(x) + +/* + * find_mount - find a the mount point of a given fs + * + * Returns 0 if mount is not found, otherwise return 1 and fill mp + * with the mount point. + */ +static const int find_mount(const char *fs, char *mp, int sizeof_mp) +{ + char mount_point[MAX_PATH]; + char type[100]; + int found; + FILE *fp; + + fp = fopen("/proc/mounts", "r"); + if (!fp) + return 0; + + while (fscanf(fp, "%*s %" STR(MAX_PATH) "s %99s %*s %*d %*d\n", mount_point, type) == 2) { + if (strcmp(type, fs) == 0) { + found = 1; + break; + } + } + fclose(fp); + + if (!found) + return 0; + + memset(mp, 0, sizeof_mp); + strncpy(mp, mount_point, sizeof_mp - 1); + + debug_msg("Fs %s found at %s\n", fs, mp); + return 1; +} + +/* + * get_self_cgroup - get the current thread cgroup path + * + * Parse /proc/$$/cgroup file to get the thread's cgroup. As an example of line to parse: + * + * 0::/user.slice/user-0.slice/session-3.scope'\n' + * + * This function is interested in the content after the second : and before the '\n'. + * + * Returns 1 if a string was found, 0 otherwise. + */ +static int get_self_cgroup(char *self_cg, int sizeof_self_cg) +{ + char path[MAX_PATH], *start; + int fd, retval; + + snprintf(path, MAX_PATH, "/proc/%d/cgroup", getpid()); + + fd = open(path, O_RDONLY); + if (fd < 0) + return 0; + + retval = read(fd, path, MAX_PATH); + + close(fd); + + if (retval <= 0) + return 0; + + start = path; + + start = strstr(start, ":"); + if (!start) + return 0; + + /* skip ":" */ + start++; + + start = strstr(start, ":"); + if (!start) + return 0; + + /* skip ":" */ + start++; + + if (strlen(start) >= sizeof_self_cg) + return 0; + + snprintf(self_cg, sizeof_self_cg, "%s", start); + + /* Swap '\n' with '\0' */ + start = strstr(self_cg, "\n"); + + /* there must be '\n' */ + if (!start) + return 0; + + /* ok, it found a string after the second : and before the \n */ + *start = '\0'; + + return 1; +} + +/** + * set_comm_cgroup - Set cgroup to threads starting with char *comm_prefix + * + * If cgroup argument is not NULL, the threads will move to the given cgroup. + * Otherwise, the cgroup of the calling, i.e., rtla, thread will be used. + * + * Supports cgroup v2. + * + * Returns 1 on success, 0 otherwise. + */ +int set_comm_cgroup(const char *comm_prefix, const char *cgroup) +{ + char cgroup_path[MAX_PATH - strlen("/cgroup.procs")]; + char cgroup_procs[MAX_PATH]; + struct dirent *proc_entry; + DIR *procfs; + int retval; + int cg_fd; + + if (strlen(comm_prefix) >= MAX_PATH) { + err_msg("Command prefix is too long: %d < strlen(%s)\n", + MAX_PATH, comm_prefix); + return 0; + } + + retval = find_mount("cgroup2", cgroup_path, sizeof(cgroup_path)); + if (!retval) { + err_msg("Did not find cgroupv2 mount point\n"); + return 0; + } + + if (!cgroup) { + retval = get_self_cgroup(&cgroup_path[strlen(cgroup_path)], + sizeof(cgroup_path) - strlen(cgroup_path)); + if (!retval) { + err_msg("Did not find self cgroup\n"); + return 0; + } + } else { + snprintf(&cgroup_path[strlen(cgroup_path)], + sizeof(cgroup_path) - strlen(cgroup_path), "%s/", cgroup); + } + + snprintf(cgroup_procs, MAX_PATH, "%s/cgroup.procs", cgroup_path); + + debug_msg("Using cgroup path at: %s\n", cgroup_procs); + + cg_fd = open(cgroup_procs, O_RDWR); + if (cg_fd < 0) + return 0; + + procfs = opendir("/proc"); + if (!procfs) { + err_msg("Could not open procfs\n"); + goto out_cg; + } + + while ((proc_entry = readdir(procfs))) { + + retval = procfs_is_workload_pid(comm_prefix, proc_entry); + if (!retval) + continue; + + retval = write(cg_fd, proc_entry->d_name, strlen(proc_entry->d_name)); + if (retval < 0) { + err_msg("Error setting cgroup attributes for pid:%s - %s\n", + proc_entry->d_name, strerror(errno)); + goto out_procfs; + } + + debug_msg("Set cgroup attributes for pid:%s\n", proc_entry->d_name); + } + + closedir(procfs); + close(cg_fd); + return 1; + +out_procfs: + closedir(procfs); +out_cg: + close(cg_fd); + return 0; +} -- cgit From 272ced2556e63943113a54c113f8c11aeb53a5c3 Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Tue, 6 Jun 2023 18:12:16 +0200 Subject: rtla: Add --house-keeping option To avoid having rtla interfering with the measurement threads, add an option for the user to set the CPUs in which rtla should run. For instance: # rtla timerlat top -H 0 -c 1-7 Will place rtla in the CPU 0, while running the measurement threads in the CPU 1-7. Link: https://lkml.kernel.org/r/6a6c78a579a96ba8b02ae67ee1e0ba2cb5e03c4a.1686066600.git.bristot@kernel.org Cc: William White Cc: Jonathan Corbet Tested-by: Juri Lelli Suggested-by: Juri Lelli Signed-off-by: Daniel Bristot de Oliveira Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/utils.c | 58 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) (limited to 'tools/tracing/rtla/src/utils.c') diff --git a/tools/tracing/rtla/src/utils.c b/tools/tracing/rtla/src/utils.c index bcc0a9f39cfe..ee6fab09acae 100644 --- a/tools/tracing/rtla/src/utils.c +++ b/tools/tracing/rtla/src/utils.c @@ -3,6 +3,7 @@ * Copyright (C) 2021 Red Hat Inc, Daniel Bristot de Oliveira */ +#define _GNU_SOURCE #include #include #include @@ -150,6 +151,63 @@ err: return 1; } +/* + * parse_cpu_set - parse a cpu_list filling cpu_set_t argument + * + * Receives a cpu list, like 1-3,5 (cpus 1, 2, 3, 5), and then set + * filling cpu_set_t argument. + * + * Returns 1 on success, 0 otherwise. + */ +int parse_cpu_set(char *cpu_list, cpu_set_t *set) +{ + const char *p; + int end_cpu; + int nr_cpus; + int cpu; + int i; + + CPU_ZERO(set); + + nr_cpus = sysconf(_SC_NPROCESSORS_CONF); + + for (p = cpu_list; *p; ) { + cpu = atoi(p); + if (cpu < 0 || (!cpu && *p != '0') || cpu >= nr_cpus) + goto err; + + while (isdigit(*p)) + p++; + if (*p == '-') { + p++; + end_cpu = atoi(p); + if (end_cpu < cpu || (!end_cpu && *p != '0') || end_cpu >= nr_cpus) + goto err; + while (isdigit(*p)) + p++; + } else + end_cpu = cpu; + + if (cpu == end_cpu) { + debug_msg("cpu_set: adding cpu %d\n", cpu); + CPU_SET(cpu, set); + } else { + for (i = cpu; i <= end_cpu; i++) { + debug_msg("cpu_set: adding cpu %d\n", i); + CPU_SET(i, set); + } + } + + if (*p == ',') + p++; + } + + return 0; +err: + debug_msg("Error parsing the cpu set %s\n", cpu_list); + return 1; +} + /* * parse_duration - parse duration with s/m/h/d suffix converting it to seconds */ -- cgit From 894c29c76b2b4c2cfbe1482dab42e4f03b49cf18 Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Tue, 6 Jun 2023 18:12:17 +0200 Subject: rtla: Change monitored_cpus from char * to cpu_set_t Use a cpumask instead of a char *, reducing memory footprint and code. No functional change, and in preparation for auto house-keeping. Link: https://lkml.kernel.org/r/54c46293261d13cb1042d0314486539eeb45fe5d.1686066600.git.bristot@kernel.org Cc: William White Cc: Jonathan Corbet Tested-by: Juri Lelli Signed-off-by: Daniel Bristot de Oliveira Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/utils.c | 63 ------------------------------------------ 1 file changed, 63 deletions(-) (limited to 'tools/tracing/rtla/src/utils.c') diff --git a/tools/tracing/rtla/src/utils.c b/tools/tracing/rtla/src/utils.c index ee6fab09acae..8f9ad8f01e0f 100644 --- a/tools/tracing/rtla/src/utils.c +++ b/tools/tracing/rtla/src/utils.c @@ -88,69 +88,6 @@ void get_duration(time_t start_time, char *output, int output_size) tm_info->tm_sec); } -/* - * parse_cpu_list - parse a cpu_list filling a char vector with cpus set - * - * Receives a cpu list, like 1-3,5 (cpus 1, 2, 3, 5), and then set the char - * in the monitored_cpus. - * - * XXX: convert to a bitmask. - */ -int parse_cpu_list(char *cpu_list, char **monitored_cpus) -{ - char *mon_cpus; - const char *p; - int end_cpu; - int nr_cpus; - int cpu; - int i; - - nr_cpus = sysconf(_SC_NPROCESSORS_CONF); - - mon_cpus = calloc(nr_cpus, sizeof(char)); - if (!mon_cpus) - goto err; - - for (p = cpu_list; *p; ) { - cpu = atoi(p); - if (cpu < 0 || (!cpu && *p != '0') || cpu >= nr_cpus) - goto err; - - while (isdigit(*p)) - p++; - if (*p == '-') { - p++; - end_cpu = atoi(p); - if (end_cpu < cpu || (!end_cpu && *p != '0') || end_cpu >= nr_cpus) - goto err; - while (isdigit(*p)) - p++; - } else - end_cpu = cpu; - - if (cpu == end_cpu) { - debug_msg("cpu_list: adding cpu %d\n", cpu); - mon_cpus[cpu] = 1; - } else { - for (i = cpu; i <= end_cpu; i++) { - debug_msg("cpu_list: adding cpu %d\n", i); - mon_cpus[i] = 1; - } - } - - if (*p == ',') - p++; - } - - *monitored_cpus = mon_cpus; - - return 0; - -err: - debug_msg("Error parsing the cpu list %s", cpu_list); - return 1; -} - /* * parse_cpu_set - parse a cpu_list filling cpu_set_t argument * -- cgit From c58a3f8c7f974d171d1b6897a71a078a3bc7afd3 Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Tue, 6 Jun 2023 18:12:18 +0200 Subject: rtla: Automatically move rtla to a house-keeping cpu When the user sets -c try to move rtla out of the , even without an -H option. This is useful to avoid having rtla interfering with the workload. This works by removing from rtla's current affinity. If rtla fails to move itself away it is not that of a problem as this is an automatic measure. Link: https://lkml.kernel.org/r/c54304d90c777310fb85a3e658d1449173759aab.1686066600.git.bristot@kernel.org Cc: William White Cc: Jonathan Corbet Tested-by: Juri Lelli Signed-off-by: Daniel Bristot de Oliveira Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/utils.c | 50 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) (limited to 'tools/tracing/rtla/src/utils.c') diff --git a/tools/tracing/rtla/src/utils.c b/tools/tracing/rtla/src/utils.c index 8f9ad8f01e0f..3e25f0277fb9 100644 --- a/tools/tracing/rtla/src/utils.c +++ b/tools/tracing/rtla/src/utils.c @@ -709,3 +709,53 @@ out_cg: close(cg_fd); return 0; } + +/** + * auto_house_keeping - Automatically move rtla out of measurement threads + * + * Try to move rtla away from the tracer, if possible. + * + * Returns 1 on success, 0 otherwise. + */ +int auto_house_keeping(cpu_set_t *monitored_cpus) +{ + cpu_set_t rtla_cpus, house_keeping_cpus; + int retval; + + /* first get the CPUs in which rtla can actually run. */ + retval = sched_getaffinity(getpid(), sizeof(rtla_cpus), &rtla_cpus); + if (retval == -1) { + debug_msg("Could not get rtla affinity, rtla might run with the threads!\n"); + return 0; + } + + /* then check if the existing setup is already good. */ + CPU_AND(&house_keeping_cpus, &rtla_cpus, monitored_cpus); + if (!CPU_COUNT(&house_keeping_cpus)) { + debug_msg("rtla and the monitored CPUs do not share CPUs."); + debug_msg("Skipping auto house-keeping\n"); + return 1; + } + + /* remove the intersection */ + CPU_XOR(&house_keeping_cpus, &rtla_cpus, monitored_cpus); + + /* get only those that rtla can run */ + CPU_AND(&house_keeping_cpus, &house_keeping_cpus, &rtla_cpus); + + /* is there any cpu left? */ + if (!CPU_COUNT(&house_keeping_cpus)) { + debug_msg("Could not find any CPU for auto house-keeping\n"); + return 0; + } + + retval = sched_setaffinity(getpid(), sizeof(house_keeping_cpus), &house_keeping_cpus); + if (retval == -1) { + debug_msg("Could not set affinity for auto house-keeping\n"); + return 0; + } + + debug_msg("rtla automatically moved to an auto house-keeping cpu set\n"); + + return 1; +} -- cgit From cdca4f4e5e8ea1c21417d86a0b2ed6af282cbb6e Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Tue, 6 Jun 2023 18:12:23 +0200 Subject: rtla/timerlat_top: Add timerlat user-space support Add the support for running timerlat threads in user-space. In this mode, enabled with -u/--user-threads, timerlat dispatches user-space processes that will loop in the timerlat_fd, measuring the overhead for going to user-space and then returning to the kernel - in addition to the existing measurements. Here is one example of the tool's output with -u enabled: $ sudo timerlat top -u -d 600 -q Timer Latency 0 00:10:01 | IRQ Timer Latency (us) | Thread Timer Latency (us) | Ret user Timer Latency (us) CPU COUNT | cur min avg max | cur min avg max | cur min avg max 0 #600001 | 0 0 0 3 | 2 1 2 9 | 3 2 3 15 1 #600001 | 0 0 0 2 | 2 1 2 13 | 2 2 3 18 2 #600001 | 0 0 0 10 | 2 1 2 16 | 3 2 3 20 3 #600001 | 0 0 0 7 | 2 1 2 10 | 3 2 3 11 4 #600000 | 0 0 0 16 | 2 1 2 41 | 3 2 3 58 5 #600000 | 0 0 0 3 | 2 1 2 10 | 3 2 3 13 6 #600000 | 0 0 0 5 | 2 1 2 7 | 3 2 3 10 7 #600000 | 0 0 0 1 | 2 1 2 7 | 3 2 3 10 The tuning setup like -p or -C work for the user-space threads as well. Link: https://lkml.kernel.org/r/758ad2292a0a1d884138d08219e1a0f572d257a2.1686066600.git.bristot@kernel.org Cc: William White Cc: Jonathan Corbet Tested-by: Juri Lelli Signed-off-by: Daniel Bristot de Oliveira Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/utils.c | 58 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) (limited to 'tools/tracing/rtla/src/utils.c') diff --git a/tools/tracing/rtla/src/utils.c b/tools/tracing/rtla/src/utils.c index 3e25f0277fb9..623a38908ed5 100644 --- a/tools/tracing/rtla/src/utils.c +++ b/tools/tracing/rtla/src/utils.c @@ -626,6 +626,64 @@ static int get_self_cgroup(char *self_cg, int sizeof_self_cg) return 1; } +/* + * set_comm_cgroup - Set cgroup to pid_t pid + * + * If cgroup argument is not NULL, the threads will move to the given cgroup. + * Otherwise, the cgroup of the calling, i.e., rtla, thread will be used. + * + * Supports cgroup v2. + * + * Returns 1 on success, 0 otherwise. + */ +int set_pid_cgroup(pid_t pid, const char *cgroup) +{ + char cgroup_path[MAX_PATH - strlen("/cgroup.procs")]; + char cgroup_procs[MAX_PATH]; + char pid_str[24]; + int retval; + int cg_fd; + + retval = find_mount("cgroup2", cgroup_path, sizeof(cgroup_path)); + if (!retval) { + err_msg("Did not find cgroupv2 mount point\n"); + return 0; + } + + if (!cgroup) { + retval = get_self_cgroup(&cgroup_path[strlen(cgroup_path)], + sizeof(cgroup_path) - strlen(cgroup_path)); + if (!retval) { + err_msg("Did not find self cgroup\n"); + return 0; + } + } else { + snprintf(&cgroup_path[strlen(cgroup_path)], + sizeof(cgroup_path) - strlen(cgroup_path), "%s/", cgroup); + } + + snprintf(cgroup_procs, MAX_PATH, "%s/cgroup.procs", cgroup_path); + + debug_msg("Using cgroup path at: %s\n", cgroup_procs); + + cg_fd = open(cgroup_procs, O_RDWR); + if (cg_fd < 0) + return 0; + + snprintf(pid_str, sizeof(pid_str), "%d\n", pid); + + retval = write(cg_fd, pid_str, strlen(pid_str)); + if (retval < 0) + err_msg("Error setting cgroup attributes for pid:%s - %s\n", + pid_str, strerror(errno)); + else + debug_msg("Set cgroup attributes for pid:%s\n", pid_str); + + close(cg_fd); + + return (retval >= 0); +} + /** * set_comm_cgroup - Set cgroup to threads starting with char *comm_prefix * -- cgit