From 1699d3efe111e33e275ca7d4163c8b1470ba79b3 Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:22:28 +0530 Subject: perf scripts python: Add initial script file with usage information Added necessary modules, including the Perf-Trace-Util library, and defines the required functions and variables for using perf script python. The perf_trace_context and Core modules for tracing and processing events has been also imported. Added usage information. Signed-off-by: Anup Sharma Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/f2f1a62f1cc69f44a5414da46a26a4cf124d2744.1689961706.git.anupnewsmail@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tools/perf/scripts/python/gecko.py (limited to 'tools/perf/scripts/python/gecko.py') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py new file mode 100644 index 000000000000..7a62c1b411d9 --- /dev/null +++ b/tools/perf/scripts/python/gecko.py @@ -0,0 +1,31 @@ +# firefox-gecko-converter.py - Convert perf record output to Firefox's gecko profile format +# SPDX-License-Identifier: GPL-2.0 +# +# The script converts perf.data to Gecko Profile Format, +# which can be read by https://profiler.firefox.com/. +# +# Usage: +# +# perf record -a -g -F 99 sleep 60 +# perf script report gecko > output.json + +import os +import sys +from typing import Dict + +# Add the Perf-Trace-Util library to the Python path +sys.path.append(os.environ['PERF_EXEC_PATH'] + \ + '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') + +from perf_trace_context import * +from Core import * + +# Uses perf script python interface to parse each +# event and store the data in the thread builder. +def process_event(param_dict: Dict) -> None: + pass + +# Trace_end runs at the end and will be used to aggregate +# the data into the final json object and print it out to stdout. +def trace_end() -> None: + pass -- cgit From 0a02e44cc2fe1657af1f2740cb9a1dcd8a9338cc Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:22:56 +0530 Subject: perf scripts python: Extact necessary information from process event The script takes in a sample event dictionary(param_dict) and retrieves relevant data such as time stamp, PID, TID, and comm for each event. Also start time is defined as a global variable as it need to be passed to trace_end for gecko meta information field creation. Signed-off-by: Anup Sharma Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/19910fefcfe4be03cd5c2aa3fec11d3f86c0381b.1689961706.git.anupnewsmail@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'tools/perf/scripts/python/gecko.py') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index 7a62c1b411d9..a02b1e04ff52 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -20,10 +20,22 @@ sys.path.append(os.environ['PERF_EXEC_PATH'] + \ from perf_trace_context import * from Core import * +# start_time is intialiazed only once for the all event traces. +start_time = None + # Uses perf script python interface to parse each # event and store the data in the thread builder. def process_event(param_dict: Dict) -> None: - pass + global start_time + global tid_to_thread + time_stamp = (param_dict['sample']['time'] // 1000) / 1000 + pid = param_dict['sample']['pid'] + tid = param_dict['sample']['tid'] + comm = param_dict['comm'] + + # Start time is the time of the first sample + if not start_time: + start_time = time_stamp # Trace_end runs at the end and will be used to aggregate # the data into the final json object and print it out to stdout. -- cgit From 5aacd7f08a3276f4fad729a600d51a1cc5d5191a Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:23:19 +0530 Subject: perf scripts python: Add classes and conversion functions This commit introduces new classes and conversion functions to facilitate the representation of Gecko profile information. The new classes Frame, Stack, Sample, and Thread are added to handle specific components of the profile data, also link to the origin docs has been commented out. Additionally, Inside the Thread class _to_json_dict() method has been created that converts the current thread data into the corresponding format expected by the GeckoThread JSON schema, as per the Gecko profile format specification. Signed-off-by: Anup Sharma Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/ab7b40bd32df7101a6f8b4a3aa41570b63b831ac.1689961706.git.anupnewsmail@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 133 ++++++++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 1 deletion(-) (limited to 'tools/perf/scripts/python/gecko.py') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index a02b1e04ff52..2cd5cd6e31e3 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -11,7 +11,8 @@ import os import sys -from typing import Dict +from dataclasses import dataclass, field +from typing import List, Dict, Optional, NamedTuple, Set, Tuple, Any # Add the Perf-Trace-Util library to the Python path sys.path.append(os.environ['PERF_EXEC_PATH'] + \ @@ -20,9 +21,139 @@ sys.path.append(os.environ['PERF_EXEC_PATH'] + \ from perf_trace_context import * from Core import * +StringID = int +StackID = int +FrameID = int +CategoryID = int +Milliseconds = float + # start_time is intialiazed only once for the all event traces. start_time = None +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 +class Frame(NamedTuple): + string_id: StringID + relevantForJS: bool + innerWindowID: int + implementation: None + optimizations: None + line: None + column: None + category: CategoryID + subcategory: int + +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 +class Stack(NamedTuple): + prefix_id: Optional[StackID] + frame_id: FrameID + +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 +class Sample(NamedTuple): + stack_id: Optional[StackID] + time_ms: Milliseconds + responsiveness: int + +@dataclass +class Thread: + """A builder for a profile of the thread. + + Attributes: + comm: Thread command-line (name). + pid: process ID of containing process. + tid: thread ID. + samples: Timeline of profile samples. + frameTable: interned stack frame ID -> stack frame. + stringTable: interned string ID -> string. + stringMap: interned string -> string ID. + stackTable: interned stack ID -> stack. + stackMap: (stack prefix ID, leaf stack frame ID) -> interned Stack ID. + frameMap: Stack Frame string -> interned Frame ID. + comm: str + pid: int + tid: int + samples: List[Sample] = field(default_factory=list) + frameTable: List[Frame] = field(default_factory=list) + stringTable: List[str] = field(default_factory=list) + stringMap: Dict[str, int] = field(default_factory=dict) + stackTable: List[Stack] = field(default_factory=list) + stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) + frameMap: Dict[str, int] = field(default_factory=dict) + """ + comm: str + pid: int + tid: int + samples: List[Sample] = field(default_factory=list) + frameTable: List[Frame] = field(default_factory=list) + stringTable: List[str] = field(default_factory=list) + stringMap: Dict[str, int] = field(default_factory=dict) + stackTable: List[Stack] = field(default_factory=list) + stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) + frameMap: Dict[str, int] = field(default_factory=dict) + + def _to_json_dict(self) -> Dict: + """Converts current Thread to GeckoThread JSON format.""" + # Gecko profile format is row-oriented data as List[List], + # And a schema for interpreting each index. + # Schema: + # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/gecko-profile-format.md + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L230 + return { + "tid": self.tid, + "pid": self.pid, + "name": self.comm, + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L51 + "markers": { + "schema": { + "name": 0, + "startTime": 1, + "endTime": 2, + "phase": 3, + "category": 4, + "data": 5, + }, + "data": [], + }, + + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 + "samples": { + "schema": { + "stack": 0, + "time": 1, + "responsiveness": 2, + }, + "data": self.samples + }, + + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 + "frameTable": { + "schema": { + "location": 0, + "relevantForJS": 1, + "innerWindowID": 2, + "implementation": 3, + "optimizations": 4, + "line": 5, + "column": 6, + "category": 7, + "subcategory": 8, + }, + "data": self.frameTable, + }, + + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 + "stackTable": { + "schema": { + "prefix": 0, + "frame": 1, + }, + "data": self.stackTable, + }, + "stringTable": self.stringTable, + "registerTime": 0, + "unregisterTime": None, + "processType": "default", + } + # Uses perf script python interface to parse each # event and store the data in the thread builder. def process_event(param_dict: Dict) -> None: -- cgit From 833daec7e6cfda0d6c30a94c21b6706ff094cd45 Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:24:42 +0530 Subject: perf scripts python: Add trace end processing and PRODUCT and CATEGORIES information The final output will now be presented in JSON format following the Gecko profile structure. Additionally, the inclusion of PRODUCT allows easy retrieval of header information for UI. Furthermore, CATEGORIES have been introduced to enable customization of kernel and user colors using input arguments. To facilitate this functionality, an argparse-based parser has been implemented. Note: The implementation of threads will be addressed in subsequent commits for now I have commented it out. Signed-off-by: Anup Sharma Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/fa6d027e4134c48e8a2ea45dd8f6b21e6a3418e4.1689961706.git.anupnewsmail@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 65 +++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) (limited to 'tools/perf/scripts/python/gecko.py') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index 2cd5cd6e31e3..794a91bec464 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -11,6 +11,8 @@ import os import sys +import json +import argparse from dataclasses import dataclass, field from typing import List, Dict, Optional, NamedTuple, Set, Tuple, Any @@ -30,6 +32,13 @@ Milliseconds = float # start_time is intialiazed only once for the all event traces. start_time = None +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/profile.js#L425 +# Follow Brendan Gregg's Flamegraph convention: orange for kernel and yellow for user space by default. +CATEGORIES = None + +# The product name is used by the profiler UI to show the Operating system and Processor. +PRODUCT = os.popen('uname -op').read().strip() + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 class Frame(NamedTuple): string_id: StringID @@ -171,4 +180,58 @@ def process_event(param_dict: Dict) -> None: # Trace_end runs at the end and will be used to aggregate # the data into the final json object and print it out to stdout. def trace_end() -> None: - pass + # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L305 + gecko_profile_with_meta = { + "meta": { + "interval": 1, + "processType": 0, + "product": PRODUCT, + "stackwalk": 1, + "debug": 0, + "gcpoison": 0, + "asyncstack": 1, + "startTime": start_time, + "shutdownTime": None, + "version": 24, + "presymbolicated": True, + "categories": CATEGORIES, + "markerSchema": [], + }, + "libs": [], + # threads will be implemented in later commits. + # "threads": threads, + "processes": [], + "pausedRanges": [], + } + json.dump(gecko_profile_with_meta, sys.stdout, indent=2) + +def main() -> None: + global CATEGORIES + parser = argparse.ArgumentParser(description="Convert perf.data to Firefox\'s Gecko Profile format") + + # Add the command-line options + # Colors must be defined according to this: + # https://github.com/firefox-devtools/profiler/blob/50124adbfa488adba6e2674a8f2618cf34b59cd2/res/css/categories.css + parser.add_argument('--user-color', default='yellow', help='Color for the User category') + parser.add_argument('--kernel-color', default='orange', help='Color for the Kernel category') + # Parse the command-line arguments + args = parser.parse_args() + # Access the values provided by the user + user_color = args.user_color + kernel_color = args.kernel_color + + CATEGORIES = [ + { + "name": 'User', + "color": user_color, + "subcategories": ['Other'] + }, + { + "name": 'Kernel', + "color": kernel_color, + "subcategories": ['Other'] + }, + ] + +if __name__ == '__main__': + main() -- cgit From 258dfd41c1df2030772ceab378f7bd6f0f78b938 Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:25:38 +0530 Subject: perf scripts python: Implement add sample function and thread processing The intern_stack function is responsible for retrieving or creating a stack_id based on the provided frame_id and prefix_id. It first generates a key using the frame_id and prefix_id values. If the stack corresponding to the key is found in the stackMap, it is returned. Otherwise, a new stack is created by appending the prefix_id and frame_id to the stackTable. The key and the index of the newly created stack are added to the stackMap for future reference. The _intern_frame function is responsible for retrieving or creating a frame_id based on the provided frame string. If the frame_id corresponding to the frameString is found in the frameMap, it is returned. Otherwise, a new frame is created by appending relevant information to the frameTable and adding the frameString to the string_id through _intern_string. The _intern_string function will gets a matching string, or saves the new string and returns a String ID. Signed-off-by: Anup Sharma Link: https://lore.kernel.org/r/4442f4b1ab4c7317cf940560a3a285fcdfbeeb08.1689961706.git.anupnewsmail@gmail.com Cc: Mark Rutland Cc: Ian Rogers Cc: Peter Zijlstra Cc: Adrian Hunter Cc: Arnaldo Carvalho de Melo Cc: Jiri Olsa Cc: Namhyung Kim Cc: Alexander Shishkin Cc: Ingo Molnar Cc: linux-kernel@vger.kernel.org Cc: linux-perf-users@vger.kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 54 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) (limited to 'tools/perf/scripts/python/gecko.py') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index 794a91bec464..97949249a3c0 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -13,6 +13,7 @@ import os import sys import json import argparse +from functools import reduce from dataclasses import dataclass, field from typing import List, Dict, Optional, NamedTuple, Set, Tuple, Any @@ -39,6 +40,10 @@ CATEGORIES = None # The product name is used by the profiler UI to show the Operating system and Processor. PRODUCT = os.popen('uname -op').read().strip() +# The category index is used by the profiler UI to show the color of the flame graph. +USER_CATEGORY_INDEX = 0 +KERNEL_CATEGORY_INDEX = 1 + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 class Frame(NamedTuple): string_id: StringID @@ -99,6 +104,55 @@ class Thread: stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) frameMap: Dict[str, int] = field(default_factory=dict) + def _intern_stack(self, frame_id: int, prefix_id: Optional[int]) -> int: + """Gets a matching stack, or saves the new stack. Returns a Stack ID.""" + key = f"{frame_id}" if prefix_id is None else f"{frame_id},{prefix_id}" + # key = (prefix_id, frame_id) + stack_id = self.stackMap.get(key) + if stack_id is None: + # return stack_id + stack_id = len(self.stackTable) + self.stackTable.append(Stack(prefix_id=prefix_id, frame_id=frame_id)) + self.stackMap[key] = stack_id + return stack_id + + def _intern_string(self, string: str) -> int: + """Gets a matching string, or saves the new string. Returns a String ID.""" + string_id = self.stringMap.get(string) + if string_id is not None: + return string_id + string_id = len(self.stringTable) + self.stringTable.append(string) + self.stringMap[string] = string_id + return string_id + + def _intern_frame(self, frame_str: str) -> int: + """Gets a matching stack frame, or saves the new frame. Returns a Frame ID.""" + frame_id = self.frameMap.get(frame_str) + if frame_id is not None: + return frame_id + frame_id = len(self.frameTable) + self.frameMap[frame_str] = frame_id + string_id = self._intern_string(frame_str) + + symbol_name_to_category = KERNEL_CATEGORY_INDEX if frame_str.find('kallsyms') != -1 \ + or frame_str.find('/vmlinux') != -1 \ + or frame_str.endswith('.ko)') \ + else USER_CATEGORY_INDEX + + self.frameTable.append(Frame( + string_id=string_id, + relevantForJS=False, + innerWindowID=0, + implementation=None, + optimizations=None, + line=None, + column=None, + category=symbol_name_to_category, + subcategory=None, + )) + return frame_id + def _to_json_dict(self) -> Dict: """Converts current Thread to GeckoThread JSON format.""" # Gecko profile format is row-oriented data as List[List], -- cgit From 2d889c6af1cc125380b03b6efdaed0a4b4611aed Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:26:24 +0530 Subject: perf scripts python: Implement add sample function and thread processing The stack has been created for storing func and dso from the callchain. The sample has been added to a specific thread. It first checks if the thread exists in the Thread class. Then it call _add_sample function which is responsible for appending a new entry to the samples list. Also callchain parsing and storing part is implemented. Moreover removed the comment from thread. Signed-off-by: Anup Sharma Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/5a112be85ccdcdcd611e343f6a7a7482d01f6299.1689961706.git.anupnewsmail@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 52 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) (limited to 'tools/perf/scripts/python/gecko.py') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index 97949249a3c0..278c3aed282a 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -40,6 +40,9 @@ CATEGORIES = None # The product name is used by the profiler UI to show the Operating system and Processor. PRODUCT = os.popen('uname -op').read().strip() +# Here key = tid, value = Thread +tid_to_thread = dict() + # The category index is used by the profiler UI to show the color of the flame graph. USER_CATEGORY_INDEX = 0 KERNEL_CATEGORY_INDEX = 1 @@ -153,6 +156,25 @@ class Thread: )) return frame_id + def _add_sample(self, comm: str, stack: List[str], time_ms: Milliseconds) -> None: + """Add a timestamped stack trace sample to the thread builder. + Args: + comm: command-line (name) of the thread at this sample + stack: sampled stack frames. Root first, leaf last. + time_ms: timestamp of sample in milliseconds. + """ + # Ihreads may not set their names right after they are created. + # Instead, they might do it later. In such situations, to use the latest name they have set. + if self.comm != comm: + self.comm = comm + + prefix_stack_id = reduce(lambda prefix_id, frame: self._intern_stack + (self._intern_frame(frame), prefix_id), stack, None) + if prefix_stack_id is not None: + self.samples.append(Sample(stack_id=prefix_stack_id, + time_ms=time_ms, + responsiveness=0)) + def _to_json_dict(self) -> Dict: """Converts current Thread to GeckoThread JSON format.""" # Gecko profile format is row-oriented data as List[List], @@ -231,9 +253,36 @@ def process_event(param_dict: Dict) -> None: if not start_time: start_time = time_stamp + # Parse and append the callchain of the current sample into a stack. + stack = [] + if param_dict['callchain']: + for call in param_dict['callchain']: + if 'sym' not in call: + continue + stack.append(f'{call["sym"]["name"]} (in {call["dso"]})') + if len(stack) != 0: + # Reverse the stack, as root come first and the leaf at the end. + stack = stack[::-1] + + # During perf record if -g is not used, the callchain is not available. + # In that case, the symbol and dso are available in the event parameters. + else: + func = param_dict['symbol'] if 'symbol' in param_dict else '[unknown]' + dso = param_dict['dso'] if 'dso' in param_dict else '[unknown]' + stack.append(f'{func} (in {dso})') + + # Add sample to the specific thread. + thread = tid_to_thread.get(tid) + if thread is None: + thread = Thread(comm=comm, pid=pid, tid=tid) + tid_to_thread[tid] = thread + thread._add_sample(comm=comm, stack=stack, time_ms=time_stamp) + # Trace_end runs at the end and will be used to aggregate # the data into the final json object and print it out to stdout. def trace_end() -> None: + threads = [thread._to_json_dict() for thread in tid_to_thread.values()] + # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L305 gecko_profile_with_meta = { "meta": { @@ -252,8 +301,7 @@ def trace_end() -> None: "markerSchema": [], }, "libs": [], - # threads will be implemented in later commits. - # "threads": threads, + "threads": threads, "processes": [], "pausedRanges": [], } -- cgit From f208b2c6f984f9c8086205e826985ab4441ce2da Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Wed, 9 Aug 2023 18:51:22 +0530 Subject: perf scripts python gecko: Launch the profiler UI on the default browser with the appropriate URL All required libraries have been imported and make sure that none of them are external dependencies. To achieve this, created a virt env and verified. Modified usage information and added combined command. Modified the main() function to read the --save-only command-line option and set the output_file variable accordingly. Modified the trace_end() function to check for the output_file variable. If it is set, the profiler data is saved to a local file in Gecko Profile format, or the profiler.firefox.com is opened on the default browser. Included trace_begin() to initialize the Firefox Profiler and launch the default browser to display the profiler.firefox.com. Added a new function launchFirefox() to start a local server and launch the profiler UI on the default browser with the appropriate URL. Created the "CORSRequestHandler" class to enable Cross-Origin Resource Sharing. Summary: This integration now includes a exiting feature to conveniently host the Gecko Profile data on a local server and open it directly in the default web browser. This means that users can now effortlessly visualize and analyze the profiler results with just a single click. The addition of the --save-only command-line option allows users to save the profiler output to a local file in Gecko Profile format, but the real highlight lies in the capability to seamlessly launch a local server, making the data accessible to Firefox Profiler via a web browser. In addition, it's important to highlight that all data are hosted locally, eliminating any concerns about data privacy rules and regulations. Signed-off-by: Anup Sharma Tested-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/ZNOS0vo58DnVLpD8@yoga Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 70 ++++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 7 deletions(-) (limited to 'tools/perf/scripts/python/gecko.py') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index 278c3aed282a..bc5a72f94bfa 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -1,4 +1,4 @@ -# firefox-gecko-converter.py - Convert perf record output to Firefox's gecko profile format +# gecko.py - Convert perf record output to Firefox's gecko profile format # SPDX-License-Identifier: GPL-2.0 # # The script converts perf.data to Gecko Profile Format, @@ -7,14 +7,26 @@ # Usage: # # perf record -a -g -F 99 sleep 60 -# perf script report gecko > output.json +# perf script report gecko +# +# Combined: +# +# perf script gecko -F 99 -a sleep 60 import os import sys +import time import json +import string +import random import argparse +import threading +import webbrowser +import urllib.parse +from os import system from functools import reduce from dataclasses import dataclass, field +from http.server import HTTPServer, SimpleHTTPRequestHandler, test from typing import List, Dict, Optional, NamedTuple, Set, Tuple, Any # Add the Perf-Trace-Util library to the Python path @@ -40,9 +52,15 @@ CATEGORIES = None # The product name is used by the profiler UI to show the Operating system and Processor. PRODUCT = os.popen('uname -op').read().strip() +# store the output file +output_file = None + # Here key = tid, value = Thread tid_to_thread = dict() +# The HTTP server is used to serve the profile to the profiler UI. +http_server_thread = None + # The category index is used by the profiler UI to show the color of the flame graph. USER_CATEGORY_INDEX = 0 KERNEL_CATEGORY_INDEX = 1 @@ -278,9 +296,19 @@ def process_event(param_dict: Dict) -> None: tid_to_thread[tid] = thread thread._add_sample(comm=comm, stack=stack, time_ms=time_stamp) +def trace_begin() -> None: + global output_file + if (output_file is None): + print("Staring Firefox Profiler on your default browser...") + global http_server_thread + http_server_thread = threading.Thread(target=test, args=(CORSRequestHandler, HTTPServer,)) + http_server_thread.daemon = True + http_server_thread.start() + # Trace_end runs at the end and will be used to aggregate # the data into the final json object and print it out to stdout. def trace_end() -> None: + global output_file threads = [thread._to_json_dict() for thread in tid_to_thread.values()] # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L305 @@ -305,22 +333,50 @@ def trace_end() -> None: "processes": [], "pausedRanges": [], } - json.dump(gecko_profile_with_meta, sys.stdout, indent=2) + # launch the profiler on local host if not specified --save-only args, otherwise print to file + if (output_file is None): + output_file = 'gecko_profile.json' + with open(output_file, 'w') as f: + json.dump(gecko_profile_with_meta, f, indent=2) + launchFirefox(output_file) + time.sleep(1) + print(f'[ perf gecko: Captured and wrote into {output_file} ]') + else: + print(f'[ perf gecko: Captured and wrote into {output_file} ]') + with open(output_file, 'w') as f: + json.dump(gecko_profile_with_meta, f, indent=2) + +# Used to enable Cross-Origin Resource Sharing (CORS) for requests coming from 'https://profiler.firefox.com', allowing it to access resources from this server. +class CORSRequestHandler(SimpleHTTPRequestHandler): + def end_headers (self): + self.send_header('Access-Control-Allow-Origin', 'https://profiler.firefox.com') + SimpleHTTPRequestHandler.end_headers(self) + +# start a local server to serve the gecko_profile.json file to the profiler.firefox.com +def launchFirefox(file): + safe_string = urllib.parse.quote_plus(f'http://localhost:8000/{file}') + url = 'https://profiler.firefox.com/from-url/' + safe_string + webbrowser.open(f'{url}') def main() -> None: + global output_file global CATEGORIES - parser = argparse.ArgumentParser(description="Convert perf.data to Firefox\'s Gecko Profile format") + parser = argparse.ArgumentParser(description="Convert perf.data to Firefox\'s Gecko Profile format which can be uploaded to profiler.firefox.com for visualization") # Add the command-line options # Colors must be defined according to this: # https://github.com/firefox-devtools/profiler/blob/50124adbfa488adba6e2674a8f2618cf34b59cd2/res/css/categories.css - parser.add_argument('--user-color', default='yellow', help='Color for the User category') - parser.add_argument('--kernel-color', default='orange', help='Color for the Kernel category') + parser.add_argument('--user-color', default='yellow', help='Color for the User category', choices=['yellow', 'blue', 'purple', 'green', 'orange', 'red', 'grey', 'magenta']) + parser.add_argument('--kernel-color', default='orange', help='Color for the Kernel category', choices=['yellow', 'blue', 'purple', 'green', 'orange', 'red', 'grey', 'magenta']) + # If --save-only is specified, the output will be saved to a file instead of opening Firefox's profiler directly. + parser.add_argument('--save-only', help='Save the output to a file instead of opening Firefox\'s profiler') + # Parse the command-line arguments args = parser.parse_args() # Access the values provided by the user user_color = args.user_color kernel_color = args.kernel_color + output_file = args.save_only CATEGORIES = [ { @@ -336,4 +392,4 @@ def main() -> None: ] if __name__ == '__main__': - main() + main() -- cgit