From 6656ae4df1a6ad6dfb5c4ce4c76136a42abb9bf4 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:28 -0600 Subject: docs: kdoc: consolidate the stripping of private struct/union members There were two locations duplicating the logic of stripping private members and associated comments; coalesce them into one, and add some comments describing what's going on. Output change: we now no longer add extraneous white space around macro definitions. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-2-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index c3fe4bd5eab4..93fcd8807aa8 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -81,6 +81,21 @@ multi_space = KernRe(r'\s\s+') def trim_whitespace(s): return multi_space.sub(' ', s.strip()) +# +# Remove struct/enum members that have been marked "private". +# +def trim_private_members(text): + # + # First look for a "public:" block that ends a private region, then + # handle the "private until the end" case. + # + text = KernRe(r'/\*\s*private:.*?/\*\s*public:.*?\*/', flags=re.S).sub('', text) + text = KernRe(r'/\*\s*private:.*', flags=re.S).sub('', text) + # + # We needed the comments to do the above, but now we can take them out. + # + return KernRe(r'\s*/\*.*?\*/\s*', flags=re.S).sub('', text).strip() + class state: """ State machine enums @@ -568,12 +583,6 @@ class KernelDoc: args_pattern = r'([^,)]+)' sub_prefixes = [ - (KernRe(r'\/\*\s*private:.*?\/\*\s*public:.*?\*\/', re.S | re.I), ''), - (KernRe(r'\/\*\s*private:.*', re.S | re.I), ''), - - # Strip comments - (KernRe(r'\/\*.*?\*\/', re.S), ''), - # Strip attributes (attribute, ' '), (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), @@ -648,6 +657,7 @@ class KernelDoc: (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), ] + members = trim_private_members(members) for search, sub in sub_prefixes: members = search.sub(sub, members) @@ -797,24 +807,18 @@ class KernelDoc: """ Stores an enum inside self.entries array. """ - - # Ignore members marked private - proto = KernRe(r'\/\*\s*private:.*?\/\*\s*public:.*?\*\/', flags=re.S).sub('', proto) - proto = KernRe(r'\/\*\s*private:.*}', flags=re.S).sub('}', proto) - - # Strip comments - proto = KernRe(r'\/\*.*?\*\/', flags=re.S).sub('', proto) - - # Strip #define macros inside enums + # + # Strip preprocessor directives. Note that this depends on the + # trailing semicolon we added in process_proto_type(). + # proto = KernRe(r'#\s*((define|ifdef|if)\s+|endif)[^;]*;', flags=re.S).sub('', proto) - # # Parse out the name and members of the enum. Typedef form first. # r = KernRe(r'typedef\s+enum\s*\{(.*)\}\s*(\w*)\s*;') if r.search(proto): declaration_name = r.group(2) - members = r.group(1).rstrip() + members = trim_private_members(r.group(1)) # # Failing that, look for a straight enum # @@ -822,7 +826,7 @@ class KernelDoc: r = KernRe(r'enum\s+(\w*)\s*\{(.*)\}') if r.match(proto): declaration_name = r.group(1) - members = r.group(2).rstrip() + members = trim_private_members(r.group(2)) # # OK, this isn't going to work. # -- cgit From 259feba4dde78f165b03e231ea9985dfe600c202 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:29 -0600 Subject: docs: kdoc: Move a regex line in dump_struct() The complex struct_members regex was defined far from its use; bring the two together. Remove some extraneous backslashes while making the move. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-3-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 93fcd8807aa8..aa6d11bf29b1 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -551,7 +551,6 @@ class KernelDoc: ] definition_body = r'\{(.*)\}\s*' + "(?:" + '|'.join(qualifiers) + ")?" - struct_members = KernRe(type_pattern + r'([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\}\;]*)(\;)') # Extract struct/union definition members = None @@ -683,6 +682,7 @@ class KernelDoc: # So, we need to have an extra loop on Python to override such # re limitation. + struct_members = KernRe(type_pattern + r'([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\};]*)(;)') while True: tuples = struct_members.findall(members) if not tuples: -- cgit From 5fd513f01169ae93d202b8c30f0837096664e7d7 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:30 -0600 Subject: docs: kdoc: backslashectomy in kdoc_parser A lot of the regular expressions in this file have extraneous backslashes that may have been needed in Perl, but aren't helpful here. Take them out to reduce slightly the visual noise. Escaping of (){}[] has been left in place, even when unnecessary, for visual clarity. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-4-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index aa6d11bf29b1..14ded23f11e0 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -46,7 +46,7 @@ doc_decl = doc_com + KernRe(r'(\w+)', cache=False) known_section_names = 'description|context|returns?|notes?|examples?' known_sections = KernRe(known_section_names, flags = re.I) doc_sect = doc_com + \ - KernRe(r'\s*(\@[.\w]+|\@\.\.\.|' + known_section_names + r')\s*:([^:].*)?$', + KernRe(r'\s*(@[.\w]+|@\.\.\.|' + known_section_names + r')\s*:([^:].*)?$', flags=re.I, cache=False) doc_content = doc_com_body + KernRe(r'(.*)', cache=False) @@ -60,7 +60,7 @@ attribute = KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False) export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) -type_param = KernRe(r"\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) +type_param = KernRe(r"@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) # # Tests for the beginning of a kerneldoc block in its various forms. @@ -405,7 +405,7 @@ class KernelDoc: for arg in args.split(splitter): # Strip comments - arg = KernRe(r'\/\*.*\*\/').sub('', arg) + arg = KernRe(r'/\*.*\*/').sub('', arg) # Ignore argument attributes arg = KernRe(r'\sPOS0?\s').sub(' ', arg) @@ -428,7 +428,7 @@ class KernelDoc: arg = arg.replace('#', ',') - r = KernRe(r'[^\(]+\(\*?\s*([\w\[\]\.]*)\s*\)') + r = KernRe(r'[^\(]+\(\*?\s*([\w\[\].]*)\s*\)') if r.match(arg): param = r.group(1) else: @@ -443,7 +443,7 @@ class KernelDoc: # Array-of-pointers arg = arg.replace('#', ',') - r = KernRe(r'[^\(]+\(\s*\*\s*([\w\[\]\.]*?)\s*(\s*\[\s*[\w]+\s*\]\s*)*\)') + r = KernRe(r'[^\(]+\(\s*\*\s*([\w\[\].]*?)\s*(\s*\[\s*[\w]+\s*\]\s*)*\)') if r.match(arg): param = r.group(1) else: @@ -709,7 +709,7 @@ class KernelDoc: if not arg: continue - r = KernRe(r'^([^\(]+\(\*?\s*)([\w\.]*)(\s*\).*)') + r = KernRe(r'^([^\(]+\(\*?\s*)([\w.]*)(\s*\).*)') if r.match(arg): # Pointer-to-function dtype = r.group(1) @@ -1044,7 +1044,7 @@ class KernelDoc: Stores a typedef inside self.entries array. """ - typedef_type = r'((?:\s+[\w\*]+\b){0,7}\s+(?:\w+\b|\*+))\s*' + typedef_type = r'((?:\s+[\w*]+\b){0,7}\s+(?:\w+\b|\*+))\s*' typedef_ident = r'\*?\s*(\w\S+)\s*' typedef_args = r'\s*\((.*)\);' @@ -1265,7 +1265,7 @@ class KernelDoc: self.dump_section() # Look for doc_com + + doc_end: - r = KernRe(r'\s*\*\s*[a-zA-Z_0-9:\.]+\*/') + r = KernRe(r'\s*\*\s*[a-zA-Z_0-9:.]+\*/') if r.match(line): self.emit_msg(ln, f"suspicious ending line: {line}") @@ -1476,7 +1476,7 @@ class KernelDoc: """Ancillary routine to process a function prototype""" # strip C99-style comments to end of line - line = KernRe(r"\/\/.*$", re.S).sub('', line) + line = KernRe(r"//.*$", re.S).sub('', line) # # Soak up the line's worth of prototype text, stopping at { or ; if present. # -- cgit From 64cf83bcd3217a9583caeb404ff136366a46705c Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:31 -0600 Subject: docs: kdoc: move the prefix transforms out of dump_struct() dump_struct is one of the longest functions in the kdoc_parser class, making it hard to read and reason about. Move the definition of the prefix transformations out of the function, join them with the definition of "attribute" (which was defined at the top of the file but only used here), and reformat the code slightly for shorter line widths. Just code movement in the end. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-5-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 179 +++++++++++++++++++++------------------- 1 file changed, 96 insertions(+), 83 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 14ded23f11e0..3d007d200da6 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -54,8 +54,6 @@ doc_inline_start = KernRe(r'^\s*/\*\*\s*$', cache=False) doc_inline_sect = KernRe(r'\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)', cache=False) doc_inline_end = KernRe(r'^\s*\*/\s*$', cache=False) doc_inline_oneline = KernRe(r'^\s*/\*\*\s*(@[\w\s]+):\s*(.*)\s*\*/\s*$', cache=False) -attribute = KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", - flags=re.I | re.S, cache=False) export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False) export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) @@ -74,6 +72,97 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' r'(?:[-:].*)?$', # description (not captured) cache = False) +# +# Here begins a long set of transformations to turn structure member prefixes +# and macro invocations into something we can parse and generate kdoc for. +# +struct_attribute = KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", + flags=re.I | re.S, cache=False) +struct_args_pattern = r'([^,)]+)' + +struct_prefixes = [ + # Strip attributes + (struct_attribute, ' '), + (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__packed\s*', re.S), ' '), + (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), + (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), + (KernRe(r'\s*____cacheline_aligned', re.S), ' '), + # + # Unwrap struct_group macros based on this definition: + # __struct_group(TAG, NAME, ATTRS, MEMBERS...) + # which has variants like: struct_group(NAME, MEMBERS...) + # Only MEMBERS arguments require documentation. + # + # Parsing them happens on two steps: + # + # 1. drop struct group arguments that aren't at MEMBERS, + # storing them as STRUCT_GROUP(MEMBERS) + # + # 2. remove STRUCT_GROUP() ancillary macro. + # + # The original logic used to remove STRUCT_GROUP() using an + # advanced regex: + # + # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; + # + # with two patterns that are incompatible with + # Python re module, as it has: + # + # - a recursive pattern: (?1) + # - an atomic grouping: (?>...) + # + # I tried a simpler version: but it didn't work either: + # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; + # + # As it doesn't properly match the end parenthesis on some cases. + # + # So, a better solution was crafted: there's now a NestedMatch + # class that ensures that delimiters after a search are properly + # matched. So, the implementation to drop STRUCT_GROUP() will be + # handled in separate. + # + (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), + (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), + (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), + (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), + # + # Replace macros + # + # TODO: use NestedMatch for FOO($1, $2, ...) matches + # + # it is better to also move those to the NestedMatch logic, + # to ensure that parenthesis will be properly matched. + # + (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), + r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), + (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), + r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), + (KernRe(r'DECLARE_BITMAP\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', + re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), + (KernRe(r'DECLARE_HASHTABLE\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', + re.S), r'unsigned long \1[1 << ((\2) - 1)]'), + (KernRe(r'DECLARE_KFIFO\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + + r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), + (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + struct_args_pattern + r',\s*' + + struct_args_pattern + r'\)', re.S), r'\2 *\1'), + (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + struct_args_pattern + r',\s*' + + struct_args_pattern + r'\)', re.S), r'\1 \2[]'), + (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), + (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'), +] +# +# Regexes here are guaranteed to have the end limiter matching +# the start delimiter. Yet, right now, only one replace group +# is allowed. +# +struct_nested_prefixes = [ + (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), +] + + # # A little helper to get rid of excess white space # @@ -578,91 +667,15 @@ class KernelDoc: self.emit_msg(ln, f"expecting prototype for {decl_type} {self.entry.identifier}. Prototype was for {decl_type} {declaration_name} instead\n") return - - args_pattern = r'([^,)]+)' - - sub_prefixes = [ - # Strip attributes - (attribute, ' '), - (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__packed\s*', re.S), ' '), - (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), - (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), - (KernRe(r'\s*____cacheline_aligned', re.S), ' '), - - # Unwrap struct_group macros based on this definition: - # __struct_group(TAG, NAME, ATTRS, MEMBERS...) - # which has variants like: struct_group(NAME, MEMBERS...) - # Only MEMBERS arguments require documentation. - # - # Parsing them happens on two steps: - # - # 1. drop struct group arguments that aren't at MEMBERS, - # storing them as STRUCT_GROUP(MEMBERS) - # - # 2. remove STRUCT_GROUP() ancillary macro. - # - # The original logic used to remove STRUCT_GROUP() using an - # advanced regex: - # - # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; - # - # with two patterns that are incompatible with - # Python re module, as it has: - # - # - a recursive pattern: (?1) - # - an atomic grouping: (?>...) - # - # I tried a simpler version: but it didn't work either: - # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; - # - # As it doesn't properly match the end parenthesis on some cases. - # - # So, a better solution was crafted: there's now a NestedMatch - # class that ensures that delimiters after a search are properly - # matched. So, the implementation to drop STRUCT_GROUP() will be - # handled in separate. - - (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), - (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), - (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), - (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), - - # Replace macros - # - # TODO: use NestedMatch for FOO($1, $2, ...) matches - # - # it is better to also move those to the NestedMatch logic, - # to ensure that parenthesis will be properly matched. - - (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), - (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), - (KernRe(r'DECLARE_BITMAP\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), - (KernRe(r'DECLARE_HASHTABLE\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'unsigned long \1[1 << ((\2) - 1)]'), - (KernRe(r'DECLARE_KFIFO\s*\(' + args_pattern + r',\s*' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'\2 *\1'), - (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'\2 *\1'), - (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'\1 \2[]'), - (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + args_pattern + r'\)', re.S), r'dma_addr_t \1'), - (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + args_pattern + r'\)', re.S), r'__u32 \1'), - ] - - # Regexes here are guaranteed to have the end limiter matching - # the start delimiter. Yet, right now, only one replace group - # is allowed. - - sub_nested_prefixes = [ - (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), - ] - + # + # Go through the list of members applying all of our transformations. + # members = trim_private_members(members) - for search, sub in sub_prefixes: + for search, sub in struct_prefixes: members = search.sub(sub, members) nested = NestedMatch() - - for search, sub in sub_nested_prefixes: + for search, sub in struct_nested_prefixes: members = nested.sub(search, sub, members) # Keeps the original declaration as-is -- cgit From 0f7344129434a6b44d4abb5080a9d67dd734ee07 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:32 -0600 Subject: docs: kdoc: split top-level prototype parsing out of dump_struct() Move the initial split of the prototype into its own function in the ongoing effort to cut dump_struct() down to size. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-6-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 43 +++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 23 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 3d007d200da6..ab896dcd9572 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -624,13 +624,11 @@ class KernelDoc: self.emit_msg(ln, f"No description found for return value of '{declaration_name}'") - def dump_struct(self, ln, proto): - """ - Store an entry for an struct or union - """ - + # + # Split apart a structure prototype; returns (struct|union, name, members) or None + # + def split_struct_proto(self, proto): type_pattern = r'(struct|union)' - qualifiers = [ "__attribute__", "__packed", @@ -638,34 +636,33 @@ class KernelDoc: "____cacheline_aligned_in_smp", "____cacheline_aligned", ] - definition_body = r'\{(.*)\}\s*' + "(?:" + '|'.join(qualifiers) + ")?" - # Extract struct/union definition - members = None - declaration_name = None - decl_type = None - r = KernRe(type_pattern + r'\s+(\w+)\s*' + definition_body) if r.search(proto): - decl_type = r.group(1) - declaration_name = r.group(2) - members = r.group(3) + return (r.group(1), r.group(2), r.group(3)) else: r = KernRe(r'typedef\s+' + type_pattern + r'\s*' + definition_body + r'\s*(\w+)\s*;') - if r.search(proto): - decl_type = r.group(1) - declaration_name = r.group(3) - members = r.group(2) + return (r.group(1), r.group(3), r.group(2)) + return None - if not members: + def dump_struct(self, ln, proto): + """ + Store an entry for an struct or union + """ + # + # Do the basic parse to get the pieces of the declaration. + # + struct_parts = self.split_struct_proto(proto) + if not struct_parts: self.emit_msg(ln, f"{proto} error: Cannot parse struct or union!") return + decl_type, declaration_name, members = struct_parts if self.entry.identifier != declaration_name: - self.emit_msg(ln, - f"expecting prototype for {decl_type} {self.entry.identifier}. Prototype was for {decl_type} {declaration_name} instead\n") + self.emit_msg(ln, f"expecting prototype for {decl_type} {self.entry.identifier}. " + f"Prototype was for {decl_type} {declaration_name} instead\n") return # # Go through the list of members applying all of our transformations. @@ -695,7 +692,7 @@ class KernelDoc: # So, we need to have an extra loop on Python to override such # re limitation. - struct_members = KernRe(type_pattern + r'([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\};]*)(;)') + struct_members = KernRe(r'(struct|union)([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\};]*)(;)') while True: tuples = struct_members.findall(members) if not tuples: -- cgit From 77e3c875f0a83d9192079e88d8569ac36c6b6bea Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:33 -0600 Subject: docs: kdoc: split struct-member rewriting out of dump_struct() The massive loop that massages struct members shares no data with the rest of dump_struct(); split it out into its own function. Code movement only, no other changes. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-7-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 65 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 31 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index ab896dcd9572..fbd7f6ce3360 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -647,37 +647,7 @@ class KernelDoc: return (r.group(1), r.group(3), r.group(2)) return None - def dump_struct(self, ln, proto): - """ - Store an entry for an struct or union - """ - # - # Do the basic parse to get the pieces of the declaration. - # - struct_parts = self.split_struct_proto(proto) - if not struct_parts: - self.emit_msg(ln, f"{proto} error: Cannot parse struct or union!") - return - decl_type, declaration_name, members = struct_parts - - if self.entry.identifier != declaration_name: - self.emit_msg(ln, f"expecting prototype for {decl_type} {self.entry.identifier}. " - f"Prototype was for {decl_type} {declaration_name} instead\n") - return - # - # Go through the list of members applying all of our transformations. - # - members = trim_private_members(members) - for search, sub in struct_prefixes: - members = search.sub(sub, members) - - nested = NestedMatch() - for search, sub in struct_nested_prefixes: - members = nested.sub(search, sub, members) - - # Keeps the original declaration as-is - declaration = members - + def rewrite_struct_members(self, members): # Split nested struct/union elements # # This loop was simpler at the original kernel-doc perl version, as @@ -768,6 +738,39 @@ class KernelDoc: newmember += f"{dtype} {s_id}.{name}; " members = members.replace(oldmember, newmember) + return members + + def dump_struct(self, ln, proto): + """ + Store an entry for an struct or union + """ + # + # Do the basic parse to get the pieces of the declaration. + # + struct_parts = self.split_struct_proto(proto) + if not struct_parts: + self.emit_msg(ln, f"{proto} error: Cannot parse struct or union!") + return + decl_type, declaration_name, members = struct_parts + + if self.entry.identifier != declaration_name: + self.emit_msg(ln, f"expecting prototype for {decl_type} {self.entry.identifier}. " + f"Prototype was for {decl_type} {declaration_name} instead\n") + return + # + # Go through the list of members applying all of our transformations. + # + members = trim_private_members(members) + for search, sub in struct_prefixes: + members = search.sub(sub, members) + + nested = NestedMatch() + for search, sub in struct_nested_prefixes: + members = nested.sub(search, sub, members) + + # Keeps the original declaration as-is + declaration = members + members = self.rewrite_struct_members(members) # Ignore other nested elements, like enums members = re.sub(r'(\{[^\{\}]*\})', '', members) -- cgit From f8208676c1c85c0b91e726954c05f5859b890ccb Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:34 -0600 Subject: docs: kdoc: rework the rewrite_struct_members() main loop Adopt a more Pythonic form for the main loop of this function, getting rid of the "while True:" construction and making the actual loop invariant explicit. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-8-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index fbd7f6ce3360..e11f3d6e9469 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -663,11 +663,8 @@ class KernelDoc: # re limitation. struct_members = KernRe(r'(struct|union)([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\};]*)(;)') - while True: - tuples = struct_members.findall(members) - if not tuples: - break - + tuples = struct_members.findall(members) + while tuples: for t in tuples: newmember = "" maintype = t[0] @@ -738,6 +735,7 @@ class KernelDoc: newmember += f"{dtype} {s_id}.{name}; " members = members.replace(oldmember, newmember) + tuples = struct_members.findall(members) return members def dump_struct(self, ln, proto): -- cgit From fb20e610393b02a832a0bf4964e12c20a7ffa2f8 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:35 -0600 Subject: docs: kdoc: remove an extraneous strip() call ...the variable in question was already strip()ed at the top of the loop. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-9-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 1 - 1 file changed, 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index e11f3d6e9469..0c279aa802a0 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -703,7 +703,6 @@ class KernelDoc: newmember += f"{dtype}{s_id}.{name}{extra}; " else: - arg = arg.strip() # Handle bitmaps arg = KernRe(r':\s*\d+\s*').sub('', arg) -- cgit From a8c4b0a8f1969e5ee0e8abb62cde8f0d7bcc2009 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:36 -0600 Subject: docs: kdoc: Some rewrite_struct_members() commenting Add comments to rewrite_struct_members() describing what it is actually doing, and reformat/comment the main struct_members regex so that it is (more) comprehensible to humans. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-10-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 0c279aa802a0..e3d0270b1a19 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -647,22 +647,28 @@ class KernelDoc: return (r.group(1), r.group(3), r.group(2)) return None + # + # Rewrite the members of a structure or union for easier formatting later on. + # Among other things, this function will turn a member like: + # + # struct { inner_members; } foo; + # + # into: + # + # struct foo; inner_members; + # def rewrite_struct_members(self, members): - # Split nested struct/union elements - # - # This loop was simpler at the original kernel-doc perl version, as - # while ($members =~ m/$struct_members/) { ... } - # reads 'members' string on each interaction. # - # Python behavior is different: it parses 'members' only once, - # creating a list of tuples from the first interaction. + # Process struct/union members from the most deeply nested outward. The + # trick is in the ^{ below - it prevents a match of an outer struct/union + # until the inner one has been munged (removing the "{" in the process). # - # On other words, this won't get nested structs. - # - # So, we need to have an extra loop on Python to override such - # re limitation. - - struct_members = KernRe(r'(struct|union)([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\};]*)(;)') + struct_members = KernRe(r'(struct|union)' # 0: declaration type + r'([^\{\};]+)' # 1: possible name + r'(\{)' + r'([^\{\}]*)' # 3: Contents of declaration + r'(\})' + r'([^\{\};]*)(;)') # 5: Remaining stuff after declaration tuples = struct_members.findall(members) while tuples: for t in tuples: -- cgit From e6dd4e2a5ca1c1e2fb168249532da1d95b5b24af Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:37 -0600 Subject: docs: kdoc: further rewrite_struct_members() cleanup Get rid of some redundant checks, and generally tighten up the code; no logical change. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-11-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 86 ++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 45 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index e3d0270b1a19..b3f937901037 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -673,73 +673,69 @@ class KernelDoc: while tuples: for t in tuples: newmember = "" - maintype = t[0] - s_ids = t[5] - content = t[3] - - oldmember = "".join(t) - - for s_id in s_ids.split(','): + oldmember = "".join(t) # Reconstruct the original formatting + dtype, name, lbr, content, rbr, rest, semi = t + # + # Pass through each field name, normalizing the form and formatting. + # + for s_id in rest.split(','): s_id = s_id.strip() - - newmember += f"{maintype} {s_id}; " + newmember += f"{dtype} {s_id}; " + # + # Remove bitfield/array/pointer info, getting the bare name. + # s_id = KernRe(r'[:\[].*').sub('', s_id) s_id = KernRe(r'^\s*\**(\S+)\s*').sub(r'\1', s_id) - + # + # Pass through the members of this inner structure/union. + # for arg in content.split(';'): arg = arg.strip() - - if not arg: - continue - + # + # Look for (type)(*name)(args) - pointer to function + # r = KernRe(r'^([^\(]+\(\*?\s*)([\w.]*)(\s*\).*)') if r.match(arg): + dtype, name, extra = r.group(1), r.group(2), r.group(3) # Pointer-to-function - dtype = r.group(1) - name = r.group(2) - extra = r.group(3) - - if not name: - continue - if not s_id: # Anonymous struct/union newmember += f"{dtype}{name}{extra}; " else: newmember += f"{dtype}{s_id}.{name}{extra}; " - + # + # Otherwise a non-function member. + # else: - # Handle bitmaps + # + # Remove bitmap and array portions and spaces around commas + # arg = KernRe(r':\s*\d+\s*').sub('', arg) - - # Handle arrays arg = KernRe(r'\[.*\]').sub('', arg) - - # Handle multiple IDs arg = KernRe(r'\s*,\s*').sub(',', arg) - + # + # Look for a normal decl - "type name[,name...]" + # r = KernRe(r'(.*)\s+([\S+,]+)') - if r.search(arg): - dtype = r.group(1) - names = r.group(2) + for name in r.group(2).split(','): + name = KernRe(r'^\s*\**(\S+)\s*').sub(r'\1', name) + if not s_id: + # Anonymous struct/union + newmember += f"{r.group(1)} {name}; " + else: + newmember += f"{r.group(1)} {s_id}.{name}; " else: newmember += f"{arg}; " - continue - - for name in names.split(','): - name = KernRe(r'^\s*\**(\S+)\s*').sub(r'\1', name).strip() - - if not name: - continue - - if not s_id: - # Anonymous struct/union - newmember += f"{dtype} {name}; " - else: - newmember += f"{dtype} {s_id}.{name}; " - + # + # At the end of the s_id loop, replace the original declaration with + # the munged version. + # members = members.replace(oldmember, newmember) + # + # End of the tuple loop - search again and see if there are outer members + # that now turn up. + # tuples = struct_members.findall(members) return members -- cgit From 23c47b09315935df140ca5ce2ddddb85453ed64d Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:38 -0600 Subject: docs: kdoc: extract output formatting from dump_struct() The last thing done in dump_struct() is to format the structure for printing. That, too, is a separate activity; split it out into its own function. dump_struct() now fits in a single, full-hight editor screen. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-12-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 72 +++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 35 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index b3f937901037..878fbfab4ac7 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -739,6 +739,42 @@ class KernelDoc: tuples = struct_members.findall(members) return members + # + # Format the struct declaration into a standard form for inclusion in the + # resulting docs. + # + def format_struct_decl(self, declaration): + # + # Insert newlines, get rid of extra spaces. + # + declaration = KernRe(r'([\{;])').sub(r'\1\n', declaration) + declaration = KernRe(r'\}\s+;').sub('};', declaration) + # + # Format inline enums with each member on its own line. + # + r = KernRe(r'(enum\s+\{[^\}]+),([^\n])') + while r.search(declaration): + declaration = r.sub(r'\1,\n\2', declaration) + # + # Now go through and supply the right number of tabs + # for each line. + # + def_args = declaration.split('\n') + level = 1 + declaration = "" + for clause in def_args: + clause = KernRe(r'\s+').sub(' ', clause.strip(), count=1) + if clause: + if '}' in clause and level > 1: + level -= 1 + if not clause.startswith('#'): + declaration += "\t" * level + declaration += "\t" + clause + "\n" + if "{" in clause and "}" not in clause: + level += 1 + return declaration + + def dump_struct(self, ln, proto): """ Store an entry for an struct or union @@ -777,42 +813,8 @@ class KernelDoc: self.create_parameter_list(ln, decl_type, members, ';', declaration_name) self.check_sections(ln, declaration_name, decl_type) - - # Adjust declaration for better display - declaration = KernRe(r'([\{;])').sub(r'\1\n', declaration) - declaration = KernRe(r'\}\s+;').sub('};', declaration) - - # Better handle inlined enums - while True: - r = KernRe(r'(enum\s+\{[^\}]+),([^\n])') - if not r.search(declaration): - break - - declaration = r.sub(r'\1,\n\2', declaration) - - def_args = declaration.split('\n') - level = 1 - declaration = "" - for clause in def_args: - - clause = clause.strip() - clause = KernRe(r'\s+').sub(' ', clause, count=1) - - if not clause: - continue - - if '}' in clause and level > 1: - level -= 1 - - if not KernRe(r'^\s*#').match(clause): - declaration += "\t" * level - - declaration += "\t" + clause + "\n" - if "{" in clause and "}" not in clause: - level += 1 - self.output_declaration(decl_type, declaration_name, - definition=declaration, + definition=self.format_struct_decl(declaration), purpose=self.entry.declaration_purpose) def dump_enum(self, ln, proto): -- cgit From e282303e718b2007b3db77c6db75ecaf4419a1af Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:39 -0600 Subject: docs: kdoc: a few final dump_struct() touches Add a couple more comments so that each phase of the process is now clearly marked. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-13-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 878fbfab4ac7..9b21fb86709a 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -802,14 +802,15 @@ class KernelDoc: nested = NestedMatch() for search, sub in struct_nested_prefixes: members = nested.sub(search, sub, members) - - # Keeps the original declaration as-is + # + # Deal with embedded struct and union members, and drop enums entirely. + # declaration = members members = self.rewrite_struct_members(members) - - # Ignore other nested elements, like enums members = re.sub(r'(\{[^\{\}]*\})', '', members) - + # + # Output the result and we are done. + # self.create_parameter_list(ln, decl_type, members, ';', declaration_name) self.check_sections(ln, declaration_name, decl_type) -- cgit From fc973dcd73f242480c61eccb1aa7306adafd2907 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 29 Jul 2025 18:43:03 +0200 Subject: docs: kernel-doc: avoid script crash on ancient Python While we do need at least 3.6 for kernel-doc to work, and at least 3.7 for it to output functions and structs with parameters at the right order, let the python binary be compatible with legacy versions. The rationale is that the Kernel build nowadays calls kernel-doc with -none on some places. Better not to bail out when older versions are found. With that, potentially this will run with python 2.7 and 3.2+, according with vermin: $ vermin --no-tips -v ./scripts/kernel-doc Detecting python files.. Analyzing using 24 processes.. 2.7, 3.2 /new_devel/v4l/docs/scripts/kernel-doc Minimum required versions: 2.7, 3.2 3.2 minimal requirement is due to argparse. The minimal version I could check was version 3.4 (using anaconda). Anaconda doesn't support 3.2 or 3.3 anymore, and 3.2 doesn't even compile (I tested compiling Python 3.2 on Fedora 42 and on Fedora 32 - no show). With 3.4, the script didn't crash and emitted the right warning: $ conda create -n py34 python=3.4 $ conda activate py34 python --version Python 3.4.5 $ python ./scripts/kernel-doc --none include/media Error: Python 3.6 or later is required by kernel-doc $ conda deactivate $ python --version Python 3.13.5 $ python ./scripts/kernel-doc --none include/media (no warnings and script ran properly) Supporting 2.7 is out of scope, as it is EOL for 5 years, and changing shebang to point to "python" instead of "python3" would have a wider impact. I did some extra checks about the differences from 3.2 and 3.4, and didn't find anything that would cause troubles: grep -rE "yield from|asyncio|pathlib|async|await|enum" scripts/kernel-doc Also, it doesn't use "@" operator. So, I'm confident that it should run (producing the exit warning) since Python 3.2. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/87d55e76b0b1391cb7a83e3e965dbddb83fa9786.1753806485.git.mchehab+huawei@kernel.org --- scripts/kernel-doc.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index fc3d46ef519f..d9fe2bcbd39c 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -2,8 +2,17 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright(c) 2025: Mauro Carvalho Chehab . # -# pylint: disable=C0103,R0915 -# +# pylint: disable=C0103,R0912,R0914,R0915 + +# NOTE: While kernel-doc requires at least version 3.6 to run, the +# command line should work with Python 3.2+ (tested with 3.4). +# The rationale is that it shall fail gracefully during Kernel +# compilation with older Kernel versions. Due to that: +# - encoding line is needed here; +# - no f-strings can be used on this file. +# - the libraries that require newer versions can only be included +# after Python version is checked. + # Converted from the kernel-doc script originally written in Perl # under GPLv2, copyrighted since 1998 by the following authors: # @@ -107,9 +116,6 @@ SRC_DIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) -from kdoc_files import KernelFiles # pylint: disable=C0413 -from kdoc_output import RestFormat, ManFormat # pylint: disable=C0413 - DESC = """ Read C language source or header FILEs, extract embedded documentation comments, and print formatted documentation to standard output. @@ -273,14 +279,22 @@ def main(): python_ver = sys.version_info[:2] if python_ver < (3,6): - logger.warning("Python 3.6 or later is required by kernel-doc") + # Depending on Kernel configuration, kernel-doc --none is called at + # build time. As we don't want to break compilation due to the + # usage of an old Python version, return 0 here. + if args.none: + logger.error("Python 3.6 or later is required by kernel-doc. skipping checks") + sys.exit(0) - # Return 0 here to avoid breaking compilation - sys.exit(0) + sys.exit("Python 3.6 or later is required by kernel-doc. Aborting.") if python_ver < (3,7): logger.warning("Python 3.7 or later is required for correct results") + # Import kernel-doc libraries only after checking Python version + from kdoc_files import KernelFiles # pylint: disable=C0415 + from kdoc_output import RestFormat, ManFormat # pylint: disable=C0415 + if args.man: out_style = ManFormat(modulename=args.modulename) elif args.none: @@ -308,11 +322,11 @@ def main(): sys.exit(0) if args.werror: - print(f"{error_count} warnings as errors") + print("%s warnings as errors" % error_count) # pylint: disable=C0209 sys.exit(error_count) if args.verbose: - print(f"{error_count} errors") + print("%s errors" % error_count) # pylint: disable=C0209 if args.none: sys.exit(0) -- cgit From 7bb184222e9fa8b839247c3a1cb1bcf628e519a5 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:18 +0200 Subject: scripts: sphinx-pre-install: fix version check for Fedora The script is now picking the wrong version. Fix it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/9d1e5c9906534e2bae586f891770066346463146.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 3f8d6925e896..07234d482fa8 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -413,7 +413,7 @@ sub give_redhat_hints() my $old = 0; my $rel; my $noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts"; - $rel = $1 if ($system_release =~ /(release|Linux)\s+(\d+)/); + $rel = $2 if ($system_release =~ /(release|Linux)\s+(\d+)/); if (!($system_release =~ /Fedora/)) { $map{"virtualenv"} = "python-virtualenv"; -- cgit From a9c50a51e0129057dbaf58fbaf7ba1a8a69e9b21 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:19 +0200 Subject: scripts: sphinx-pre-install: rename it to scripts/sphinx-pre-install.pl That helps us to later replace the scripts. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/08d9a32a5aaf3784fef016594efe505d7c5a2697.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 1056 ----------------------------------------- scripts/sphinx-pre-install.pl | 1056 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1056 insertions(+), 1056 deletions(-) delete mode 100755 scripts/sphinx-pre-install create mode 100755 scripts/sphinx-pre-install.pl (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install deleted file mode 100755 index 07234d482fa8..000000000000 --- a/scripts/sphinx-pre-install +++ /dev/null @@ -1,1056 +0,0 @@ -#!/usr/bin/env perl -# SPDX-License-Identifier: GPL-2.0-or-later -use strict; - -# Copyright (c) 2017-2020 Mauro Carvalho Chehab -# - -my $prefix = "./"; -$prefix = "$ENV{'srctree'}/" if ($ENV{'srctree'}); - -my $conf = $prefix . "Documentation/conf.py"; -my $requirement_file = $prefix . "Documentation/sphinx/requirements.txt"; -my $virtenv_prefix = "sphinx_"; - -# -# Static vars -# - -my %missing; -my $system_release; -my $need = 0; -my $optional = 0; -my $need_symlink = 0; -my $need_sphinx = 0; -my $need_pip = 0; -my $need_virtualenv = 0; -my $rec_sphinx_upgrade = 0; -my $verbose_warn_install = 1; -my $install = ""; -my $virtenv_dir = ""; -my $python_cmd = ""; -my $activate_cmd; -my $min_version; -my $cur_version; -my $rec_version = "3.4.3"; -my $latest_avail_ver; - -# -# Command line arguments -# - -my $pdf = 1; -my $virtualenv = 1; -my $version_check = 0; - -# -# List of required texlive packages on Fedora and OpenSuse -# - -my %texlive = ( - 'amsfonts.sty' => 'texlive-amsfonts', - 'amsmath.sty' => 'texlive-amsmath', - 'amssymb.sty' => 'texlive-amsfonts', - 'amsthm.sty' => 'texlive-amscls', - 'anyfontsize.sty' => 'texlive-anyfontsize', - 'atbegshi.sty' => 'texlive-oberdiek', - 'bm.sty' => 'texlive-tools', - 'capt-of.sty' => 'texlive-capt-of', - 'cmap.sty' => 'texlive-cmap', - 'ecrm1000.tfm' => 'texlive-ec', - 'eqparbox.sty' => 'texlive-eqparbox', - 'eu1enc.def' => 'texlive-euenc', - 'fancybox.sty' => 'texlive-fancybox', - 'fancyvrb.sty' => 'texlive-fancyvrb', - 'float.sty' => 'texlive-float', - 'fncychap.sty' => 'texlive-fncychap', - 'footnote.sty' => 'texlive-mdwtools', - 'framed.sty' => 'texlive-framed', - 'luatex85.sty' => 'texlive-luatex85', - 'multirow.sty' => 'texlive-multirow', - 'needspace.sty' => 'texlive-needspace', - 'palatino.sty' => 'texlive-psnfss', - 'parskip.sty' => 'texlive-parskip', - 'polyglossia.sty' => 'texlive-polyglossia', - 'tabulary.sty' => 'texlive-tabulary', - 'threeparttable.sty' => 'texlive-threeparttable', - 'titlesec.sty' => 'texlive-titlesec', - 'ucs.sty' => 'texlive-ucs', - 'upquote.sty' => 'texlive-upquote', - 'wrapfig.sty' => 'texlive-wrapfig', - 'ctexhook.sty' => 'texlive-ctex', -); - -# -# Subroutines that checks if a feature exists -# - -sub check_missing(%) -{ - my %map = %{$_[0]}; - - foreach my $prog (sort keys %missing) { - my $is_optional = $missing{$prog}; - - # At least on some LTS distros like CentOS 7, texlive doesn't - # provide all packages we need. When such distros are - # detected, we have to disable PDF output. - # - # So, we need to ignore the packages that distros would - # need for LaTeX to work - if ($is_optional == 2 && !$pdf) { - $optional--; - next; - } - - if ($verbose_warn_install) { - if ($is_optional) { - print "Warning: better to also install \"$prog\".\n"; - } else { - print "ERROR: please install \"$prog\", otherwise, build won't work.\n"; - } - } - if (defined($map{$prog})) { - $install .= " " . $map{$prog}; - } else { - $install .= " " . $prog; - } - } - - $install =~ s/^\s//; -} - -sub add_package($$) -{ - my $package = shift; - my $is_optional = shift; - - $missing{$package} = $is_optional; - if ($is_optional) { - $optional++; - } else { - $need++; - } -} - -sub check_missing_file($$$) -{ - my $files = shift; - my $package = shift; - my $is_optional = shift; - - for (@$files) { - return if(-e $_); - } - - add_package($package, $is_optional); -} - -sub findprog($) -{ - foreach(split(/:/, $ENV{PATH})) { - return "$_/$_[0]" if(-x "$_/$_[0]"); - } -} - -sub find_python_no_venv() -{ - my $prog = shift; - - my $cur_dir = qx(pwd); - $cur_dir =~ s/\s+$//; - - foreach my $dir (split(/:/, $ENV{PATH})) { - next if ($dir =~ m,($cur_dir)/sphinx,); - return "$dir/python3" if(-x "$dir/python3"); - } - foreach my $dir (split(/:/, $ENV{PATH})) { - next if ($dir =~ m,($cur_dir)/sphinx,); - return "$dir/python" if(-x "$dir/python"); - } - return "python"; -} - -sub check_program($$) -{ - my $prog = shift; - my $is_optional = shift; - - return $prog if findprog($prog); - - add_package($prog, $is_optional); -} - -sub check_perl_module($$) -{ - my $prog = shift; - my $is_optional = shift; - - my $err = system("perl -M$prog -e 1 2>/dev/null /dev/null"); - return if ($err == 0); - - add_package($prog, $is_optional); -} - -sub check_python_module($$) -{ - my $prog = shift; - my $is_optional = shift; - - return if (!$python_cmd); - - my $err = system("$python_cmd -c 'import $prog' 2>/dev/null /dev/null"); - return if ($err == 0); - - add_package($prog, $is_optional); -} - -sub check_rpm_missing($$) -{ - my @pkgs = @{$_[0]}; - my $is_optional = $_[1]; - - foreach my $prog(@pkgs) { - my $err = system("rpm -q '$prog' 2>/dev/null >/dev/null"); - add_package($prog, $is_optional) if ($err); - } -} - -sub check_pacman_missing($$) -{ - my @pkgs = @{$_[0]}; - my $is_optional = $_[1]; - - foreach my $prog(@pkgs) { - my $err = system("pacman -Q '$prog' 2>/dev/null >/dev/null"); - add_package($prog, $is_optional) if ($err); - } -} - -sub check_missing_tex($) -{ - my $is_optional = shift; - my $kpsewhich = findprog("kpsewhich"); - - foreach my $prog(keys %texlive) { - my $package = $texlive{$prog}; - if (!$kpsewhich) { - add_package($package, $is_optional); - next; - } - my $file = qx($kpsewhich $prog); - add_package($package, $is_optional) if ($file =~ /^\s*$/); - } -} - -sub get_sphinx_fname() -{ - if ($ENV{'SPHINXBUILD'}) { - return $ENV{'SPHINXBUILD'}; - } - - my $fname = "sphinx-build"; - return $fname if findprog($fname); - - $fname = "sphinx-build-3"; - if (findprog($fname)) { - $need_symlink = 1; - return $fname; - } - - return ""; -} - -sub get_sphinx_version($) -{ - my $cmd = shift; - my $ver; - - open IN, "$cmd --version 2>&1 |"; - while () { - if (m/^\s*sphinx-build\s+([\d\.]+)((\+\/[\da-f]+)|(b\d+))?$/) { - $ver=$1; - last; - } - # Sphinx 1.2.x uses a different format - if (m/^\s*Sphinx.*\s+([\d\.]+)$/) { - $ver=$1; - last; - } - } - close IN; - return $ver; -} - -sub check_sphinx() -{ - open IN, $conf or die "Can't open $conf"; - while () { - if (m/^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]/) { - $min_version=$1; - last; - } - } - close IN; - - die "Can't get needs_sphinx version from $conf" if (!$min_version); - - $virtenv_dir = $virtenv_prefix . "latest"; - - my $sphinx = get_sphinx_fname(); - if ($sphinx eq "") { - $need_sphinx = 1; - return; - } - - $cur_version = get_sphinx_version($sphinx); - die "$sphinx didn't return its version" if (!$cur_version); - - if ($cur_version lt $min_version) { - printf "ERROR: Sphinx version is %s. It should be >= %s\n", - $cur_version, $min_version; - $need_sphinx = 1; - return; - } - - return if ($cur_version lt $rec_version); - - # On version check mode, just assume Sphinx has all mandatory deps - exit (0) if ($version_check); -} - -# -# Ancillary subroutines -# - -sub catcheck($) -{ - my $res = ""; - $res = qx(cat $_[0]) if (-r $_[0]); - return $res; -} - -sub which($) -{ - my $file = shift; - my @path = split ":", $ENV{PATH}; - - foreach my $dir(@path) { - my $name = $dir.'/'.$file; - return $name if (-x $name ); - } - return undef; -} - -# -# Subroutines that check distro-specific hints -# - -sub give_debian_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-yaml", - "ensurepip" => "python3-venv", - "virtualenv" => "virtualenv", - "dot" => "graphviz", - "convert" => "imagemagick", - "Pod::Usage" => "perl-modules", - "xelatex" => "texlive-xetex", - "rsvg-convert" => "librsvg2-bin", - ); - - if ($pdf) { - check_missing_file(["/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty"], - "texlive-lang-chinese", 2); - - check_missing_file(["/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"], - "fonts-dejavu", 2); - - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc"], - "fonts-noto-cjk", 2); - } - - check_program("dvipng", 2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo apt-get install $install\n"); -} - -sub give_redhat_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-pyyaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive-xetex-bin", - "rsvg-convert" => "librsvg2-tools", - ); - - my @fedora26_opt_pkgs = ( - "graphviz-gd", # Fedora 26: needed for PDF support - ); - - my @fedora_tex_pkgs = ( - "texlive-collection-fontsrecommended", - "texlive-collection-latex", - "texlive-xecjk", - "dejavu-sans-fonts", - "dejavu-serif-fonts", - "dejavu-sans-mono-fonts", - ); - - # - # Checks valid for RHEL/CentOS version 7.x. - # - my $old = 0; - my $rel; - my $noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts"; - $rel = $2 if ($system_release =~ /(release|Linux)\s+(\d+)/); - - if (!($system_release =~ /Fedora/)) { - $map{"virtualenv"} = "python-virtualenv"; - - if ($rel && $rel < 8) { - $old = 1; - $pdf = 0; - - printf("Note: texlive packages on RHEL/CENTOS <= 7 are incomplete. Can't support PDF output\n"); - printf("If you want to build PDF, please read:\n"); - printf("\thttps://www.systutorials.com/241660/how-to-install-tex-live-on-centos-7-linux/\n"); - } - } else { - if ($rel && $rel < 26) { - $old = 1; - } - if ($rel && $rel >= 38) { - $noto_sans_redhat = "google-noto-sans-cjk-fonts"; - } - } - if (!$rel) { - printf("Couldn't identify release number\n"); - $old = 1; - $pdf = 0; - } - - if ($pdf) { - check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc"], - $noto_sans_redhat, 2); - } - - check_rpm_missing(\@fedora26_opt_pkgs, 2) if ($pdf && !$old); - check_rpm_missing(\@fedora_tex_pkgs, 2) if ($pdf); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - - if (!$old) { - # dnf, for Fedora 18+ - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo dnf install -y $install\n"); - } else { - # yum, for RHEL (and clones) or Fedora version < 18 - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo yum install -y $install\n"); - } -} - -sub give_opensuse_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-pyyaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive-xetex-bin", - ); - - # On Tumbleweed, this package is also named rsvg-convert - $map{"rsvg-convert"} = "rsvg-view" if (!($system_release =~ /Tumbleweed/)); - - my @suse_tex_pkgs = ( - "texlive-babel-english", - "texlive-caption", - "texlive-colortbl", - "texlive-courier", - "texlive-dvips", - "texlive-helvetic", - "texlive-makeindex", - "texlive-metafont", - "texlive-metapost", - "texlive-palatino", - "texlive-preview", - "texlive-times", - "texlive-zapfchan", - "texlive-zapfding", - ); - - $map{"latexmk"} = "texlive-latexmk-bin"; - - # FIXME: add support for installing CJK fonts - # - # I tried hard, but was unable to find a way to install - # "Noto Sans CJK SC" on openSUSE - - check_rpm_missing(\@suse_tex_pkgs, 2) if ($pdf); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo zypper install --no-recommends $install\n"); -} - -sub give_mageia_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-yaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive", - "rsvg-convert" => "librsvg2", - ); - - my @tex_pkgs = ( - "texlive-fontsextra", - ); - - $map{"latexmk"} = "texlive-collection-basic"; - - my $packager_cmd; - my $noto_sans; - if ($system_release =~ /OpenMandriva/) { - $packager_cmd = "dnf install"; - $noto_sans = "noto-sans-cjk-fonts"; - @tex_pkgs = ( "texlive-collection-fontsextra" ); - } else { - $packager_cmd = "urpmi"; - $noto_sans = "google-noto-sans-cjk-ttc-fonts"; - } - - - if ($pdf) { - check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/TTF/NotoSans-Regular.ttf"], - $noto_sans, 2); - } - - check_rpm_missing(\@tex_pkgs, 2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo $packager_cmd $install\n"); -} - -sub give_arch_linux_hints() -{ - my %map = ( - "yaml" => "python-yaml", - "virtualenv" => "python-virtualenv", - "dot" => "graphviz", - "convert" => "imagemagick", - "xelatex" => "texlive-xetex", - "latexmk" => "texlive-core", - "rsvg-convert" => "extra/librsvg", - ); - - my @archlinux_tex_pkgs = ( - "texlive-core", - "texlive-latexextra", - "ttf-dejavu", - ); - check_pacman_missing(\@archlinux_tex_pkgs, 2) if ($pdf); - - if ($pdf) { - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], - "noto-fonts-cjk", 2); - } - - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo pacman -S $install\n"); -} - -sub give_gentoo_hints() -{ - my %map = ( - "yaml" => "dev-python/pyyaml", - "virtualenv" => "dev-python/virtualenv", - "dot" => "media-gfx/graphviz", - "convert" => "media-gfx/imagemagick", - "xelatex" => "dev-texlive/texlive-xetex media-fonts/dejavu", - "rsvg-convert" => "gnome-base/librsvg", - ); - - check_missing_file(["/usr/share/fonts/dejavu/DejaVuSans.ttf"], - "media-fonts/dejavu", 2) if ($pdf); - - if ($pdf) { - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", - "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc"], - "media-fonts/noto-cjk", 2); - } - - check_missing(\%map); - - return if (!$need && !$optional); - - printf("You should run:\n") if ($verbose_warn_install); - printf("\n"); - - my $imagemagick = "media-gfx/imagemagick svg png"; - my $cairo = "media-gfx/graphviz cairo pdf"; - my $portage_imagemagick = "/etc/portage/package.use/imagemagick"; - my $portage_cairo = "/etc/portage/package.use/graphviz"; - - if (qx(grep imagemagick $portage_imagemagick 2>/dev/null) eq "") { - printf("\tsudo su -c 'echo \"$imagemagick\" > $portage_imagemagick'\n") - } - if (qx(grep graphviz $portage_cairo 2>/dev/null) eq "") { - printf("\tsudo su -c 'echo \"$cairo\" > $portage_cairo'\n"); - } - - printf("\tsudo emerge --ask $install\n"); - -} - -sub check_distros() -{ - # Distro-specific hints - if ($system_release =~ /Red Hat Enterprise Linux/) { - give_redhat_hints; - return; - } - if ($system_release =~ /CentOS/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Scientific Linux/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Oracle Linux Server/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Fedora/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Ubuntu/) { - give_debian_hints; - return; - } - if ($system_release =~ /Debian/) { - give_debian_hints; - return; - } - if ($system_release =~ /openSUSE/) { - give_opensuse_hints; - return; - } - if ($system_release =~ /Mageia/) { - give_mageia_hints; - return; - } - if ($system_release =~ /OpenMandriva/) { - give_mageia_hints; - return; - } - if ($system_release =~ /Arch Linux/) { - give_arch_linux_hints; - return; - } - if ($system_release =~ /Gentoo/) { - give_gentoo_hints; - return; - } - - # - # Fall-back to generic hint code for other distros - # That's far from ideal, specially for LaTeX dependencies. - # - my %map = ( - "sphinx-build" => "sphinx" - ); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - print "I don't know distro $system_release.\n"; - print "So, I can't provide you a hint with the install procedure.\n"; - print "There are likely missing dependencies.\n"; -} - -# -# Common dependencies -# - -sub deactivate_help() -{ - printf "\n If you want to exit the virtualenv, you can use:\n"; - printf "\tdeactivate\n"; -} - -sub get_virtenv() -{ - my $ver; - my $min_activate = "$ENV{'PWD'}/${virtenv_prefix}${min_version}/bin/activate"; - my @activates = glob "$ENV{'PWD'}/${virtenv_prefix}*/bin/activate"; - - @activates = sort {$b cmp $a} @activates; - - foreach my $f (@activates) { - next if ($f lt $min_activate); - - my $sphinx_cmd = $f; - $sphinx_cmd =~ s/activate/sphinx-build/; - next if (! -f $sphinx_cmd); - - my $ver = get_sphinx_version($sphinx_cmd); - - if (!$ver) { - $f =~ s#/bin/activate##; - print("Warning: virtual environment $f is not working.\nPython version upgrade? Remove it with:\n\n\trm -rf $f\n\n"); - } - - if ($need_sphinx && ($ver ge $min_version)) { - return ($f, $ver); - } elsif ($ver gt $cur_version) { - return ($f, $ver); - } - } - return ("", ""); -} - -sub recommend_sphinx_upgrade() -{ - my $venv_ver; - - # Avoid running sphinx-builds from venv if $cur_version is good - if ($cur_version && ($cur_version ge $rec_version)) { - $latest_avail_ver = $cur_version; - return; - } - - # Get the highest version from sphinx_*/bin/sphinx-build and the - # corresponding command to activate the venv/virtenv - ($activate_cmd, $venv_ver) = get_virtenv(); - - # Store the highest version from Sphinx existing virtualenvs - if (($activate_cmd ne "") && ($venv_ver gt $cur_version)) { - $latest_avail_ver = $venv_ver; - } else { - $latest_avail_ver = $cur_version if ($cur_version); - } - - # As we don't know package version of Sphinx, and there's no - # virtual environments, don't check if upgrades are needed - if (!$virtualenv) { - return if (!$latest_avail_ver); - } - - # Either there are already a virtual env or a new one should be created - $need_pip = 1; - - return if (!$latest_avail_ver); - - # Return if the reason is due to an upgrade or not - if ($latest_avail_ver lt $rec_version) { - $rec_sphinx_upgrade = 1; - } - - return $latest_avail_ver; -} - -# -# The logic here is complex, as it have to deal with different versions: -# - minimal supported version; -# - minimal PDF version; -# - recommended version. -# It also needs to work fine with both distro's package and venv/virtualenv -sub recommend_sphinx_version($) -{ - my $virtualenv_cmd = shift; - - # Version is OK. Nothing to do. - if ($cur_version && ($cur_version ge $rec_version)) { - return; - }; - - if (!$need_sphinx) { - # sphinx-build is present and its version is >= $min_version - - #only recommend enabling a newer virtenv version if makes sense. - if ($latest_avail_ver gt $cur_version) { - printf "\nYou may also use the newer Sphinx version $latest_avail_ver with:\n"; - printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - return if ($latest_avail_ver ge $rec_version); - } - - if (!$virtualenv) { - # No sphinx either via package or via virtenv. As we can't - # Compare the versions here, just return, recommending the - # user to install it from the package distro. - return if (!$latest_avail_ver); - - # User doesn't want a virtenv recommendation, but he already - # installed one via virtenv with a newer version. - # So, print commands to enable it - if ($latest_avail_ver gt $cur_version) { - printf "\nYou may also use the Sphinx virtualenv version $latest_avail_ver with:\n"; - printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - print "\n"; - } else { - $need++ if ($need_sphinx); - } - - # Suggest newer versions if current ones are too old - if ($latest_avail_ver && $latest_avail_ver ge $min_version) { - # If there's a good enough version, ask the user to enable it - if ($latest_avail_ver ge $rec_version) { - printf "\nNeed to activate Sphinx (version $latest_avail_ver) on virtualenv with:\n"; - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - - # Version is above the minimal required one, but may be - # below the recommended one. So, print warnings/notes - - if ($latest_avail_ver lt $rec_version) { - print "Warning: It is recommended at least Sphinx version $rec_version.\n"; - } - } - - # At this point, either it needs Sphinx or upgrade is recommended, - # both via pip - - if ($rec_sphinx_upgrade) { - if (!$virtualenv) { - print "Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n"; - } else { - print "To upgrade Sphinx, use:\n\n"; - } - } else { - print "\nSphinx needs to be installed either:\n1) via pip/pypi with:\n\n"; - } - - $python_cmd = find_python_no_venv(); - - printf "\t$virtualenv_cmd $virtenv_dir\n"; - - printf "\t. $virtenv_dir/bin/activate\n"; - printf "\tpip install -r $requirement_file\n"; - deactivate_help(); - - printf "\n2) As a package with:\n"; - - my $old_need = $need; - my $old_optional = $optional; - %missing = (); - $pdf = 0; - $optional = 0; - $install = ""; - $verbose_warn_install = 0; - - add_package("python-sphinx", 0); - - check_distros(); - - $need = $old_need; - $optional = $old_optional; - - printf "\n Please note that Sphinx >= 3.0 will currently produce false-positive\n"; - printf " warning when the same name is used for more than one type (functions,\n"; - printf " structs, enums,...). This is known Sphinx bug. For more details, see:\n"; - printf "\thttps://github.com/sphinx-doc/sphinx/pull/8313\n"; -} - -sub check_needs() -{ - # Check if Sphinx is already accessible from current environment - check_sphinx(); - - if ($system_release) { - print "Detected OS: $system_release.\n"; - } else { - print "Unknown OS\n"; - } - printf "Sphinx version: %s\n\n", $cur_version if ($cur_version); - - # Check python command line, trying first python3 - $python_cmd = findprog("python3"); - $python_cmd = check_program("python", 0) if (!$python_cmd); - - # Check the type of virtual env, depending on Python version - if ($python_cmd) { - if ($virtualenv) { - my $tmp = qx($python_cmd --version 2>&1); - if ($tmp =~ m/(\d+\.)(\d+\.)/) { - if ($1 < 3) { - # Fail if it finds python2 (or worse) - die "Python 3 is required to build the kernel docs\n"; - } - if ($1 == 3 && $2 < 3) { - # Need Python 3.3 or upper for venv - $need_virtualenv = 1; - } - } else { - die "Warning: couldn't identify $python_cmd version!"; - } - } else { - add_package("python-sphinx", 0); - } - } - - my $venv_ver = recommend_sphinx_upgrade(); - - my $virtualenv_cmd; - - if ($need_pip) { - # Set virtualenv command line, if python < 3.3 - if ($need_virtualenv) { - $virtualenv_cmd = findprog("virtualenv-3"); - $virtualenv_cmd = findprog("virtualenv-3.5") if (!$virtualenv_cmd); - if (!$virtualenv_cmd) { - check_program("virtualenv", 0); - $virtualenv_cmd = "virtualenv"; - } - } else { - $virtualenv_cmd = "$python_cmd -m venv"; - check_python_module("ensurepip", 0); - } - } - - # Check for needed programs/tools - check_perl_module("Pod::Usage", 0); - check_python_module("yaml", 0); - check_program("make", 0); - check_program("gcc", 0); - check_program("dot", 1); - check_program("convert", 1); - - # Extra PDF files - should use 2 for is_optional - check_program("xelatex", 2) if ($pdf); - check_program("rsvg-convert", 2) if ($pdf); - check_program("latexmk", 2) if ($pdf); - - # Do distro-specific checks and output distro-install commands - check_distros(); - - if (!$python_cmd) { - if ($need == 1) { - die "Can't build as $need mandatory dependency is missing"; - } elsif ($need) { - die "Can't build as $need mandatory dependencies are missing"; - } - } - - # Check if sphinx-build is called sphinx-build-3 - if ($need_symlink) { - printf "\tsudo ln -sf %s /usr/bin/sphinx-build\n\n", - which("sphinx-build-3"); - } - - recommend_sphinx_version($virtualenv_cmd); - printf "\n"; - - print "All optional dependencies are met.\n" if (!$optional); - - if ($need == 1) { - die "Can't build as $need mandatory dependency is missing"; - } elsif ($need) { - die "Can't build as $need mandatory dependencies are missing"; - } - - print "Needed package dependencies are met.\n"; -} - -# -# Main -# - -while (@ARGV) { - my $arg = shift(@ARGV); - - if ($arg eq "--no-virtualenv") { - $virtualenv = 0; - } elsif ($arg eq "--no-pdf"){ - $pdf = 0; - } elsif ($arg eq "--version-check"){ - $version_check = 1; - } else { - print "Usage:\n\t$0 <--no-virtualenv> <--no-pdf> <--version-check>\n\n"; - print "Where:\n"; - print "\t--no-virtualenv\t- Recommend installing Sphinx instead of using a virtualenv\n"; - print "\t--version-check\t- if version is compatible, don't check for missing dependencies\n"; - print "\t--no-pdf\t- don't check for dependencies required to build PDF docs\n\n"; - exit -1; - } -} - -# -# Determine the system type. There's no standard unique way that would -# work with all distros with a minimal package install. So, several -# methods are used here. -# -# By default, it will use lsb_release function. If not available, it will -# fail back to reading the known different places where the distro name -# is stored -# - -$system_release = qx(lsb_release -d) if which("lsb_release"); -$system_release =~ s/Description:\s*// if ($system_release); -$system_release = catcheck("/etc/system-release") if !$system_release; -$system_release = catcheck("/etc/redhat-release") if !$system_release; -$system_release = catcheck("/etc/lsb-release") if !$system_release; -$system_release = catcheck("/etc/gentoo-release") if !$system_release; - -# This seems more common than LSB these days -if (!$system_release) { - my %os_var; - if (open IN, "cat /etc/os-release|") { - while () { - if (m/^([\w\d\_]+)=\"?([^\"]*)\"?\n/) { - $os_var{$1}=$2; - } - } - $system_release = $os_var{"NAME"}; - if (defined($os_var{"VERSION_ID"})) { - $system_release .= " " . $os_var{"VERSION_ID"} if (defined($os_var{"VERSION_ID"})); - } else { - $system_release .= " " . $os_var{"VERSION"}; - } - } -} -$system_release = catcheck("/etc/issue") if !$system_release; -$system_release =~ s/\s+$//; - -check_needs; diff --git a/scripts/sphinx-pre-install.pl b/scripts/sphinx-pre-install.pl new file mode 100755 index 000000000000..07234d482fa8 --- /dev/null +++ b/scripts/sphinx-pre-install.pl @@ -0,0 +1,1056 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0-or-later +use strict; + +# Copyright (c) 2017-2020 Mauro Carvalho Chehab +# + +my $prefix = "./"; +$prefix = "$ENV{'srctree'}/" if ($ENV{'srctree'}); + +my $conf = $prefix . "Documentation/conf.py"; +my $requirement_file = $prefix . "Documentation/sphinx/requirements.txt"; +my $virtenv_prefix = "sphinx_"; + +# +# Static vars +# + +my %missing; +my $system_release; +my $need = 0; +my $optional = 0; +my $need_symlink = 0; +my $need_sphinx = 0; +my $need_pip = 0; +my $need_virtualenv = 0; +my $rec_sphinx_upgrade = 0; +my $verbose_warn_install = 1; +my $install = ""; +my $virtenv_dir = ""; +my $python_cmd = ""; +my $activate_cmd; +my $min_version; +my $cur_version; +my $rec_version = "3.4.3"; +my $latest_avail_ver; + +# +# Command line arguments +# + +my $pdf = 1; +my $virtualenv = 1; +my $version_check = 0; + +# +# List of required texlive packages on Fedora and OpenSuse +# + +my %texlive = ( + 'amsfonts.sty' => 'texlive-amsfonts', + 'amsmath.sty' => 'texlive-amsmath', + 'amssymb.sty' => 'texlive-amsfonts', + 'amsthm.sty' => 'texlive-amscls', + 'anyfontsize.sty' => 'texlive-anyfontsize', + 'atbegshi.sty' => 'texlive-oberdiek', + 'bm.sty' => 'texlive-tools', + 'capt-of.sty' => 'texlive-capt-of', + 'cmap.sty' => 'texlive-cmap', + 'ecrm1000.tfm' => 'texlive-ec', + 'eqparbox.sty' => 'texlive-eqparbox', + 'eu1enc.def' => 'texlive-euenc', + 'fancybox.sty' => 'texlive-fancybox', + 'fancyvrb.sty' => 'texlive-fancyvrb', + 'float.sty' => 'texlive-float', + 'fncychap.sty' => 'texlive-fncychap', + 'footnote.sty' => 'texlive-mdwtools', + 'framed.sty' => 'texlive-framed', + 'luatex85.sty' => 'texlive-luatex85', + 'multirow.sty' => 'texlive-multirow', + 'needspace.sty' => 'texlive-needspace', + 'palatino.sty' => 'texlive-psnfss', + 'parskip.sty' => 'texlive-parskip', + 'polyglossia.sty' => 'texlive-polyglossia', + 'tabulary.sty' => 'texlive-tabulary', + 'threeparttable.sty' => 'texlive-threeparttable', + 'titlesec.sty' => 'texlive-titlesec', + 'ucs.sty' => 'texlive-ucs', + 'upquote.sty' => 'texlive-upquote', + 'wrapfig.sty' => 'texlive-wrapfig', + 'ctexhook.sty' => 'texlive-ctex', +); + +# +# Subroutines that checks if a feature exists +# + +sub check_missing(%) +{ + my %map = %{$_[0]}; + + foreach my $prog (sort keys %missing) { + my $is_optional = $missing{$prog}; + + # At least on some LTS distros like CentOS 7, texlive doesn't + # provide all packages we need. When such distros are + # detected, we have to disable PDF output. + # + # So, we need to ignore the packages that distros would + # need for LaTeX to work + if ($is_optional == 2 && !$pdf) { + $optional--; + next; + } + + if ($verbose_warn_install) { + if ($is_optional) { + print "Warning: better to also install \"$prog\".\n"; + } else { + print "ERROR: please install \"$prog\", otherwise, build won't work.\n"; + } + } + if (defined($map{$prog})) { + $install .= " " . $map{$prog}; + } else { + $install .= " " . $prog; + } + } + + $install =~ s/^\s//; +} + +sub add_package($$) +{ + my $package = shift; + my $is_optional = shift; + + $missing{$package} = $is_optional; + if ($is_optional) { + $optional++; + } else { + $need++; + } +} + +sub check_missing_file($$$) +{ + my $files = shift; + my $package = shift; + my $is_optional = shift; + + for (@$files) { + return if(-e $_); + } + + add_package($package, $is_optional); +} + +sub findprog($) +{ + foreach(split(/:/, $ENV{PATH})) { + return "$_/$_[0]" if(-x "$_/$_[0]"); + } +} + +sub find_python_no_venv() +{ + my $prog = shift; + + my $cur_dir = qx(pwd); + $cur_dir =~ s/\s+$//; + + foreach my $dir (split(/:/, $ENV{PATH})) { + next if ($dir =~ m,($cur_dir)/sphinx,); + return "$dir/python3" if(-x "$dir/python3"); + } + foreach my $dir (split(/:/, $ENV{PATH})) { + next if ($dir =~ m,($cur_dir)/sphinx,); + return "$dir/python" if(-x "$dir/python"); + } + return "python"; +} + +sub check_program($$) +{ + my $prog = shift; + my $is_optional = shift; + + return $prog if findprog($prog); + + add_package($prog, $is_optional); +} + +sub check_perl_module($$) +{ + my $prog = shift; + my $is_optional = shift; + + my $err = system("perl -M$prog -e 1 2>/dev/null /dev/null"); + return if ($err == 0); + + add_package($prog, $is_optional); +} + +sub check_python_module($$) +{ + my $prog = shift; + my $is_optional = shift; + + return if (!$python_cmd); + + my $err = system("$python_cmd -c 'import $prog' 2>/dev/null /dev/null"); + return if ($err == 0); + + add_package($prog, $is_optional); +} + +sub check_rpm_missing($$) +{ + my @pkgs = @{$_[0]}; + my $is_optional = $_[1]; + + foreach my $prog(@pkgs) { + my $err = system("rpm -q '$prog' 2>/dev/null >/dev/null"); + add_package($prog, $is_optional) if ($err); + } +} + +sub check_pacman_missing($$) +{ + my @pkgs = @{$_[0]}; + my $is_optional = $_[1]; + + foreach my $prog(@pkgs) { + my $err = system("pacman -Q '$prog' 2>/dev/null >/dev/null"); + add_package($prog, $is_optional) if ($err); + } +} + +sub check_missing_tex($) +{ + my $is_optional = shift; + my $kpsewhich = findprog("kpsewhich"); + + foreach my $prog(keys %texlive) { + my $package = $texlive{$prog}; + if (!$kpsewhich) { + add_package($package, $is_optional); + next; + } + my $file = qx($kpsewhich $prog); + add_package($package, $is_optional) if ($file =~ /^\s*$/); + } +} + +sub get_sphinx_fname() +{ + if ($ENV{'SPHINXBUILD'}) { + return $ENV{'SPHINXBUILD'}; + } + + my $fname = "sphinx-build"; + return $fname if findprog($fname); + + $fname = "sphinx-build-3"; + if (findprog($fname)) { + $need_symlink = 1; + return $fname; + } + + return ""; +} + +sub get_sphinx_version($) +{ + my $cmd = shift; + my $ver; + + open IN, "$cmd --version 2>&1 |"; + while () { + if (m/^\s*sphinx-build\s+([\d\.]+)((\+\/[\da-f]+)|(b\d+))?$/) { + $ver=$1; + last; + } + # Sphinx 1.2.x uses a different format + if (m/^\s*Sphinx.*\s+([\d\.]+)$/) { + $ver=$1; + last; + } + } + close IN; + return $ver; +} + +sub check_sphinx() +{ + open IN, $conf or die "Can't open $conf"; + while () { + if (m/^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]/) { + $min_version=$1; + last; + } + } + close IN; + + die "Can't get needs_sphinx version from $conf" if (!$min_version); + + $virtenv_dir = $virtenv_prefix . "latest"; + + my $sphinx = get_sphinx_fname(); + if ($sphinx eq "") { + $need_sphinx = 1; + return; + } + + $cur_version = get_sphinx_version($sphinx); + die "$sphinx didn't return its version" if (!$cur_version); + + if ($cur_version lt $min_version) { + printf "ERROR: Sphinx version is %s. It should be >= %s\n", + $cur_version, $min_version; + $need_sphinx = 1; + return; + } + + return if ($cur_version lt $rec_version); + + # On version check mode, just assume Sphinx has all mandatory deps + exit (0) if ($version_check); +} + +# +# Ancillary subroutines +# + +sub catcheck($) +{ + my $res = ""; + $res = qx(cat $_[0]) if (-r $_[0]); + return $res; +} + +sub which($) +{ + my $file = shift; + my @path = split ":", $ENV{PATH}; + + foreach my $dir(@path) { + my $name = $dir.'/'.$file; + return $name if (-x $name ); + } + return undef; +} + +# +# Subroutines that check distro-specific hints +# + +sub give_debian_hints() +{ + my %map = ( + "python-sphinx" => "python3-sphinx", + "yaml" => "python3-yaml", + "ensurepip" => "python3-venv", + "virtualenv" => "virtualenv", + "dot" => "graphviz", + "convert" => "imagemagick", + "Pod::Usage" => "perl-modules", + "xelatex" => "texlive-xetex", + "rsvg-convert" => "librsvg2-bin", + ); + + if ($pdf) { + check_missing_file(["/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty"], + "texlive-lang-chinese", 2); + + check_missing_file(["/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"], + "fonts-dejavu", 2); + + check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc"], + "fonts-noto-cjk", 2); + } + + check_program("dvipng", 2) if ($pdf); + check_missing(\%map); + + return if (!$need && !$optional); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo apt-get install $install\n"); +} + +sub give_redhat_hints() +{ + my %map = ( + "python-sphinx" => "python3-sphinx", + "yaml" => "python3-pyyaml", + "virtualenv" => "python3-virtualenv", + "dot" => "graphviz", + "convert" => "ImageMagick", + "Pod::Usage" => "perl-Pod-Usage", + "xelatex" => "texlive-xetex-bin", + "rsvg-convert" => "librsvg2-tools", + ); + + my @fedora26_opt_pkgs = ( + "graphviz-gd", # Fedora 26: needed for PDF support + ); + + my @fedora_tex_pkgs = ( + "texlive-collection-fontsrecommended", + "texlive-collection-latex", + "texlive-xecjk", + "dejavu-sans-fonts", + "dejavu-serif-fonts", + "dejavu-sans-mono-fonts", + ); + + # + # Checks valid for RHEL/CentOS version 7.x. + # + my $old = 0; + my $rel; + my $noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts"; + $rel = $2 if ($system_release =~ /(release|Linux)\s+(\d+)/); + + if (!($system_release =~ /Fedora/)) { + $map{"virtualenv"} = "python-virtualenv"; + + if ($rel && $rel < 8) { + $old = 1; + $pdf = 0; + + printf("Note: texlive packages on RHEL/CENTOS <= 7 are incomplete. Can't support PDF output\n"); + printf("If you want to build PDF, please read:\n"); + printf("\thttps://www.systutorials.com/241660/how-to-install-tex-live-on-centos-7-linux/\n"); + } + } else { + if ($rel && $rel < 26) { + $old = 1; + } + if ($rel && $rel >= 38) { + $noto_sans_redhat = "google-noto-sans-cjk-fonts"; + } + } + if (!$rel) { + printf("Couldn't identify release number\n"); + $old = 1; + $pdf = 0; + } + + if ($pdf) { + check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc"], + $noto_sans_redhat, 2); + } + + check_rpm_missing(\@fedora26_opt_pkgs, 2) if ($pdf && !$old); + check_rpm_missing(\@fedora_tex_pkgs, 2) if ($pdf); + check_missing_tex(2) if ($pdf); + check_missing(\%map); + + return if (!$need && !$optional); + + if (!$old) { + # dnf, for Fedora 18+ + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo dnf install -y $install\n"); + } else { + # yum, for RHEL (and clones) or Fedora version < 18 + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo yum install -y $install\n"); + } +} + +sub give_opensuse_hints() +{ + my %map = ( + "python-sphinx" => "python3-sphinx", + "yaml" => "python3-pyyaml", + "virtualenv" => "python3-virtualenv", + "dot" => "graphviz", + "convert" => "ImageMagick", + "Pod::Usage" => "perl-Pod-Usage", + "xelatex" => "texlive-xetex-bin", + ); + + # On Tumbleweed, this package is also named rsvg-convert + $map{"rsvg-convert"} = "rsvg-view" if (!($system_release =~ /Tumbleweed/)); + + my @suse_tex_pkgs = ( + "texlive-babel-english", + "texlive-caption", + "texlive-colortbl", + "texlive-courier", + "texlive-dvips", + "texlive-helvetic", + "texlive-makeindex", + "texlive-metafont", + "texlive-metapost", + "texlive-palatino", + "texlive-preview", + "texlive-times", + "texlive-zapfchan", + "texlive-zapfding", + ); + + $map{"latexmk"} = "texlive-latexmk-bin"; + + # FIXME: add support for installing CJK fonts + # + # I tried hard, but was unable to find a way to install + # "Noto Sans CJK SC" on openSUSE + + check_rpm_missing(\@suse_tex_pkgs, 2) if ($pdf); + check_missing_tex(2) if ($pdf); + check_missing(\%map); + + return if (!$need && !$optional); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo zypper install --no-recommends $install\n"); +} + +sub give_mageia_hints() +{ + my %map = ( + "python-sphinx" => "python3-sphinx", + "yaml" => "python3-yaml", + "virtualenv" => "python3-virtualenv", + "dot" => "graphviz", + "convert" => "ImageMagick", + "Pod::Usage" => "perl-Pod-Usage", + "xelatex" => "texlive", + "rsvg-convert" => "librsvg2", + ); + + my @tex_pkgs = ( + "texlive-fontsextra", + ); + + $map{"latexmk"} = "texlive-collection-basic"; + + my $packager_cmd; + my $noto_sans; + if ($system_release =~ /OpenMandriva/) { + $packager_cmd = "dnf install"; + $noto_sans = "noto-sans-cjk-fonts"; + @tex_pkgs = ( "texlive-collection-fontsextra" ); + } else { + $packager_cmd = "urpmi"; + $noto_sans = "google-noto-sans-cjk-ttc-fonts"; + } + + + if ($pdf) { + check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/TTF/NotoSans-Regular.ttf"], + $noto_sans, 2); + } + + check_rpm_missing(\@tex_pkgs, 2) if ($pdf); + check_missing(\%map); + + return if (!$need && !$optional); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo $packager_cmd $install\n"); +} + +sub give_arch_linux_hints() +{ + my %map = ( + "yaml" => "python-yaml", + "virtualenv" => "python-virtualenv", + "dot" => "graphviz", + "convert" => "imagemagick", + "xelatex" => "texlive-xetex", + "latexmk" => "texlive-core", + "rsvg-convert" => "extra/librsvg", + ); + + my @archlinux_tex_pkgs = ( + "texlive-core", + "texlive-latexextra", + "ttf-dejavu", + ); + check_pacman_missing(\@archlinux_tex_pkgs, 2) if ($pdf); + + if ($pdf) { + check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], + "noto-fonts-cjk", 2); + } + + check_missing(\%map); + + return if (!$need && !$optional); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo pacman -S $install\n"); +} + +sub give_gentoo_hints() +{ + my %map = ( + "yaml" => "dev-python/pyyaml", + "virtualenv" => "dev-python/virtualenv", + "dot" => "media-gfx/graphviz", + "convert" => "media-gfx/imagemagick", + "xelatex" => "dev-texlive/texlive-xetex media-fonts/dejavu", + "rsvg-convert" => "gnome-base/librsvg", + ); + + check_missing_file(["/usr/share/fonts/dejavu/DejaVuSans.ttf"], + "media-fonts/dejavu", 2) if ($pdf); + + if ($pdf) { + check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", + "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc"], + "media-fonts/noto-cjk", 2); + } + + check_missing(\%map); + + return if (!$need && !$optional); + + printf("You should run:\n") if ($verbose_warn_install); + printf("\n"); + + my $imagemagick = "media-gfx/imagemagick svg png"; + my $cairo = "media-gfx/graphviz cairo pdf"; + my $portage_imagemagick = "/etc/portage/package.use/imagemagick"; + my $portage_cairo = "/etc/portage/package.use/graphviz"; + + if (qx(grep imagemagick $portage_imagemagick 2>/dev/null) eq "") { + printf("\tsudo su -c 'echo \"$imagemagick\" > $portage_imagemagick'\n") + } + if (qx(grep graphviz $portage_cairo 2>/dev/null) eq "") { + printf("\tsudo su -c 'echo \"$cairo\" > $portage_cairo'\n"); + } + + printf("\tsudo emerge --ask $install\n"); + +} + +sub check_distros() +{ + # Distro-specific hints + if ($system_release =~ /Red Hat Enterprise Linux/) { + give_redhat_hints; + return; + } + if ($system_release =~ /CentOS/) { + give_redhat_hints; + return; + } + if ($system_release =~ /Scientific Linux/) { + give_redhat_hints; + return; + } + if ($system_release =~ /Oracle Linux Server/) { + give_redhat_hints; + return; + } + if ($system_release =~ /Fedora/) { + give_redhat_hints; + return; + } + if ($system_release =~ /Ubuntu/) { + give_debian_hints; + return; + } + if ($system_release =~ /Debian/) { + give_debian_hints; + return; + } + if ($system_release =~ /openSUSE/) { + give_opensuse_hints; + return; + } + if ($system_release =~ /Mageia/) { + give_mageia_hints; + return; + } + if ($system_release =~ /OpenMandriva/) { + give_mageia_hints; + return; + } + if ($system_release =~ /Arch Linux/) { + give_arch_linux_hints; + return; + } + if ($system_release =~ /Gentoo/) { + give_gentoo_hints; + return; + } + + # + # Fall-back to generic hint code for other distros + # That's far from ideal, specially for LaTeX dependencies. + # + my %map = ( + "sphinx-build" => "sphinx" + ); + check_missing_tex(2) if ($pdf); + check_missing(\%map); + print "I don't know distro $system_release.\n"; + print "So, I can't provide you a hint with the install procedure.\n"; + print "There are likely missing dependencies.\n"; +} + +# +# Common dependencies +# + +sub deactivate_help() +{ + printf "\n If you want to exit the virtualenv, you can use:\n"; + printf "\tdeactivate\n"; +} + +sub get_virtenv() +{ + my $ver; + my $min_activate = "$ENV{'PWD'}/${virtenv_prefix}${min_version}/bin/activate"; + my @activates = glob "$ENV{'PWD'}/${virtenv_prefix}*/bin/activate"; + + @activates = sort {$b cmp $a} @activates; + + foreach my $f (@activates) { + next if ($f lt $min_activate); + + my $sphinx_cmd = $f; + $sphinx_cmd =~ s/activate/sphinx-build/; + next if (! -f $sphinx_cmd); + + my $ver = get_sphinx_version($sphinx_cmd); + + if (!$ver) { + $f =~ s#/bin/activate##; + print("Warning: virtual environment $f is not working.\nPython version upgrade? Remove it with:\n\n\trm -rf $f\n\n"); + } + + if ($need_sphinx && ($ver ge $min_version)) { + return ($f, $ver); + } elsif ($ver gt $cur_version) { + return ($f, $ver); + } + } + return ("", ""); +} + +sub recommend_sphinx_upgrade() +{ + my $venv_ver; + + # Avoid running sphinx-builds from venv if $cur_version is good + if ($cur_version && ($cur_version ge $rec_version)) { + $latest_avail_ver = $cur_version; + return; + } + + # Get the highest version from sphinx_*/bin/sphinx-build and the + # corresponding command to activate the venv/virtenv + ($activate_cmd, $venv_ver) = get_virtenv(); + + # Store the highest version from Sphinx existing virtualenvs + if (($activate_cmd ne "") && ($venv_ver gt $cur_version)) { + $latest_avail_ver = $venv_ver; + } else { + $latest_avail_ver = $cur_version if ($cur_version); + } + + # As we don't know package version of Sphinx, and there's no + # virtual environments, don't check if upgrades are needed + if (!$virtualenv) { + return if (!$latest_avail_ver); + } + + # Either there are already a virtual env or a new one should be created + $need_pip = 1; + + return if (!$latest_avail_ver); + + # Return if the reason is due to an upgrade or not + if ($latest_avail_ver lt $rec_version) { + $rec_sphinx_upgrade = 1; + } + + return $latest_avail_ver; +} + +# +# The logic here is complex, as it have to deal with different versions: +# - minimal supported version; +# - minimal PDF version; +# - recommended version. +# It also needs to work fine with both distro's package and venv/virtualenv +sub recommend_sphinx_version($) +{ + my $virtualenv_cmd = shift; + + # Version is OK. Nothing to do. + if ($cur_version && ($cur_version ge $rec_version)) { + return; + }; + + if (!$need_sphinx) { + # sphinx-build is present and its version is >= $min_version + + #only recommend enabling a newer virtenv version if makes sense. + if ($latest_avail_ver gt $cur_version) { + printf "\nYou may also use the newer Sphinx version $latest_avail_ver with:\n"; + printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); + printf "\t. $activate_cmd\n"; + deactivate_help(); + + return; + } + return if ($latest_avail_ver ge $rec_version); + } + + if (!$virtualenv) { + # No sphinx either via package or via virtenv. As we can't + # Compare the versions here, just return, recommending the + # user to install it from the package distro. + return if (!$latest_avail_ver); + + # User doesn't want a virtenv recommendation, but he already + # installed one via virtenv with a newer version. + # So, print commands to enable it + if ($latest_avail_ver gt $cur_version) { + printf "\nYou may also use the Sphinx virtualenv version $latest_avail_ver with:\n"; + printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); + printf "\t. $activate_cmd\n"; + deactivate_help(); + + return; + } + print "\n"; + } else { + $need++ if ($need_sphinx); + } + + # Suggest newer versions if current ones are too old + if ($latest_avail_ver && $latest_avail_ver ge $min_version) { + # If there's a good enough version, ask the user to enable it + if ($latest_avail_ver ge $rec_version) { + printf "\nNeed to activate Sphinx (version $latest_avail_ver) on virtualenv with:\n"; + printf "\t. $activate_cmd\n"; + deactivate_help(); + + return; + } + + # Version is above the minimal required one, but may be + # below the recommended one. So, print warnings/notes + + if ($latest_avail_ver lt $rec_version) { + print "Warning: It is recommended at least Sphinx version $rec_version.\n"; + } + } + + # At this point, either it needs Sphinx or upgrade is recommended, + # both via pip + + if ($rec_sphinx_upgrade) { + if (!$virtualenv) { + print "Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n"; + } else { + print "To upgrade Sphinx, use:\n\n"; + } + } else { + print "\nSphinx needs to be installed either:\n1) via pip/pypi with:\n\n"; + } + + $python_cmd = find_python_no_venv(); + + printf "\t$virtualenv_cmd $virtenv_dir\n"; + + printf "\t. $virtenv_dir/bin/activate\n"; + printf "\tpip install -r $requirement_file\n"; + deactivate_help(); + + printf "\n2) As a package with:\n"; + + my $old_need = $need; + my $old_optional = $optional; + %missing = (); + $pdf = 0; + $optional = 0; + $install = ""; + $verbose_warn_install = 0; + + add_package("python-sphinx", 0); + + check_distros(); + + $need = $old_need; + $optional = $old_optional; + + printf "\n Please note that Sphinx >= 3.0 will currently produce false-positive\n"; + printf " warning when the same name is used for more than one type (functions,\n"; + printf " structs, enums,...). This is known Sphinx bug. For more details, see:\n"; + printf "\thttps://github.com/sphinx-doc/sphinx/pull/8313\n"; +} + +sub check_needs() +{ + # Check if Sphinx is already accessible from current environment + check_sphinx(); + + if ($system_release) { + print "Detected OS: $system_release.\n"; + } else { + print "Unknown OS\n"; + } + printf "Sphinx version: %s\n\n", $cur_version if ($cur_version); + + # Check python command line, trying first python3 + $python_cmd = findprog("python3"); + $python_cmd = check_program("python", 0) if (!$python_cmd); + + # Check the type of virtual env, depending on Python version + if ($python_cmd) { + if ($virtualenv) { + my $tmp = qx($python_cmd --version 2>&1); + if ($tmp =~ m/(\d+\.)(\d+\.)/) { + if ($1 < 3) { + # Fail if it finds python2 (or worse) + die "Python 3 is required to build the kernel docs\n"; + } + if ($1 == 3 && $2 < 3) { + # Need Python 3.3 or upper for venv + $need_virtualenv = 1; + } + } else { + die "Warning: couldn't identify $python_cmd version!"; + } + } else { + add_package("python-sphinx", 0); + } + } + + my $venv_ver = recommend_sphinx_upgrade(); + + my $virtualenv_cmd; + + if ($need_pip) { + # Set virtualenv command line, if python < 3.3 + if ($need_virtualenv) { + $virtualenv_cmd = findprog("virtualenv-3"); + $virtualenv_cmd = findprog("virtualenv-3.5") if (!$virtualenv_cmd); + if (!$virtualenv_cmd) { + check_program("virtualenv", 0); + $virtualenv_cmd = "virtualenv"; + } + } else { + $virtualenv_cmd = "$python_cmd -m venv"; + check_python_module("ensurepip", 0); + } + } + + # Check for needed programs/tools + check_perl_module("Pod::Usage", 0); + check_python_module("yaml", 0); + check_program("make", 0); + check_program("gcc", 0); + check_program("dot", 1); + check_program("convert", 1); + + # Extra PDF files - should use 2 for is_optional + check_program("xelatex", 2) if ($pdf); + check_program("rsvg-convert", 2) if ($pdf); + check_program("latexmk", 2) if ($pdf); + + # Do distro-specific checks and output distro-install commands + check_distros(); + + if (!$python_cmd) { + if ($need == 1) { + die "Can't build as $need mandatory dependency is missing"; + } elsif ($need) { + die "Can't build as $need mandatory dependencies are missing"; + } + } + + # Check if sphinx-build is called sphinx-build-3 + if ($need_symlink) { + printf "\tsudo ln -sf %s /usr/bin/sphinx-build\n\n", + which("sphinx-build-3"); + } + + recommend_sphinx_version($virtualenv_cmd); + printf "\n"; + + print "All optional dependencies are met.\n" if (!$optional); + + if ($need == 1) { + die "Can't build as $need mandatory dependency is missing"; + } elsif ($need) { + die "Can't build as $need mandatory dependencies are missing"; + } + + print "Needed package dependencies are met.\n"; +} + +# +# Main +# + +while (@ARGV) { + my $arg = shift(@ARGV); + + if ($arg eq "--no-virtualenv") { + $virtualenv = 0; + } elsif ($arg eq "--no-pdf"){ + $pdf = 0; + } elsif ($arg eq "--version-check"){ + $version_check = 1; + } else { + print "Usage:\n\t$0 <--no-virtualenv> <--no-pdf> <--version-check>\n\n"; + print "Where:\n"; + print "\t--no-virtualenv\t- Recommend installing Sphinx instead of using a virtualenv\n"; + print "\t--version-check\t- if version is compatible, don't check for missing dependencies\n"; + print "\t--no-pdf\t- don't check for dependencies required to build PDF docs\n\n"; + exit -1; + } +} + +# +# Determine the system type. There's no standard unique way that would +# work with all distros with a minimal package install. So, several +# methods are used here. +# +# By default, it will use lsb_release function. If not available, it will +# fail back to reading the known different places where the distro name +# is stored +# + +$system_release = qx(lsb_release -d) if which("lsb_release"); +$system_release =~ s/Description:\s*// if ($system_release); +$system_release = catcheck("/etc/system-release") if !$system_release; +$system_release = catcheck("/etc/redhat-release") if !$system_release; +$system_release = catcheck("/etc/lsb-release") if !$system_release; +$system_release = catcheck("/etc/gentoo-release") if !$system_release; + +# This seems more common than LSB these days +if (!$system_release) { + my %os_var; + if (open IN, "cat /etc/os-release|") { + while () { + if (m/^([\w\d\_]+)=\"?([^\"]*)\"?\n/) { + $os_var{$1}=$2; + } + } + $system_release = $os_var{"NAME"}; + if (defined($os_var{"VERSION_ID"})) { + $system_release .= " " . $os_var{"VERSION_ID"} if (defined($os_var{"VERSION_ID"})); + } else { + $system_release .= " " . $os_var{"VERSION"}; + } + } +} +$system_release = catcheck("/etc/issue") if !$system_release; +$system_release =~ s/\s+$//; + +check_needs; -- cgit From ca9087f50772aa5794dd655ff5a0719dd4b83b1f Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:20 +0200 Subject: scripts: sphinx-pre-install: Convert script to Python Port scripts/sphinx-pre-install to Python. That allows a better maintainance of the file. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/8d236079545aff796900bde4cdaaf79411d04074.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 1102 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1102 insertions(+) create mode 100755 scripts/sphinx-pre-install.py (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py new file mode 100755 index 000000000000..dcee2181b72f --- /dev/null +++ b/scripts/sphinx-pre-install.py @@ -0,0 +1,1102 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (c) 2017-2025 Mauro Carvalho Chehab +# +# pylint: disable=C0103,C0114,C0115,C0116,C0301 +# pylint: disable=R0902,R0904,R0912,R0915,R1705,R1710,E1121 + + +import argparse +import os +import re +import subprocess +import sys +from glob import glob + + +def parse_version(version): + """Convert a major.minor.patch version into a tuple""" +# + return tuple(int(x) for x in version.split(".")) + + +def ver_str(version): + """Returns a version tuple as major.minor.patch""" + + return ".".join([str(x) for x in version]) + + +RECOMMENDED_VERSION = parse_version("3.4.3") + + +class SphinxDependencyChecker: + # List of required texlive packages on Fedora and OpenSuse + texlive = { + "amsfonts.sty": "texlive-amsfonts", + "amsmath.sty": "texlive-amsmath", + "amssymb.sty": "texlive-amsfonts", + "amsthm.sty": "texlive-amscls", + "anyfontsize.sty": "texlive-anyfontsize", + "atbegshi.sty": "texlive-oberdiek", + "bm.sty": "texlive-tools", + "capt-of.sty": "texlive-capt-of", + "cmap.sty": "texlive-cmap", + "ctexhook.sty": "texlive-ctex", + "ecrm1000.tfm": "texlive-ec", + "eqparbox.sty": "texlive-eqparbox", + "eu1enc.def": "texlive-euenc", + "fancybox.sty": "texlive-fancybox", + "fancyvrb.sty": "texlive-fancyvrb", + "float.sty": "texlive-float", + "fncychap.sty": "texlive-fncychap", + "footnote.sty": "texlive-mdwtools", + "framed.sty": "texlive-framed", + "luatex85.sty": "texlive-luatex85", + "multirow.sty": "texlive-multirow", + "needspace.sty": "texlive-needspace", + "palatino.sty": "texlive-psnfss", + "parskip.sty": "texlive-parskip", + "polyglossia.sty": "texlive-polyglossia", + "tabulary.sty": "texlive-tabulary", + "threeparttable.sty": "texlive-threeparttable", + "titlesec.sty": "texlive-titlesec", + "ucs.sty": "texlive-ucs", + "upquote.sty": "texlive-upquote", + "wrapfig.sty": "texlive-wrapfig", + } + + def __init__(self, args): + self.pdf = args.pdf + self.virtualenv = args.virtualenv + self.version_check = args.version_check + + self.missing = {} + + self.need = 0 + self.optional = 0 + self.need_symlink = 0 + self.need_sphinx = 0 + self.need_pip = 0 + self.need_virtualenv = 0 + self.rec_sphinx_upgrade = 0 + self.verbose_warn_install = 1 + + self.system_release = "" + self.install = "" + self.virtenv_dir = "" + self.python_cmd = "" + self.activate_cmd = "" + + self.min_version = (0, 0, 0) + self.cur_version = (0, 0, 0) + self.latest_avail_ver = (0, 0, 0) + self.venv_ver = (0, 0, 0) + + prefix = os.environ.get("srctree", ".") + "/" + + self.conf = prefix + "Documentation/conf.py" + self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" + self.virtenv_prefix = ["sphinx_", "Sphinx_" ] + + # + # Ancillary methods that don't depend on self + # + + @staticmethod + def which(prog): + for path in os.environ.get("PATH", "").split(":"): + full_path = os.path.join(path, prog) + if os.access(full_path, os.X_OK): + return full_path + + return None + + @staticmethod + def find_python_no_venv(): + # FIXME: does it makes sense now that this script is in Python? + + result = subprocess.run(["pwd"], capture_output=True, text=True) + cur_dir = result.stdout.strip() + + python_names = ["python3", "python"] + + for d in os.environ.get("PATH", "").split(":"): + if f"{cur_dir}/sphinx" in d: + continue + + for p in python_names: + if os.access(os.path.join(d, p), os.X_OK): + return os.path.join(d, p) + + # Python not found at the PATH + return python_names[-1] + + @staticmethod + def run(*args, **kwargs): + """Excecute a command, hiding its output by default""" + + if not kwargs.get('capture_output', False): + if 'stdout' not in kwargs: + kwargs['stdout'] = subprocess.DEVNULL + if 'stderr' not in kwargs: + kwargs['stderr'] = subprocess.DEVNULL + + return subprocess.run(*args, **kwargs) + + # + # Methods to check if a feature exists + # + + # Note: is_optional has 3 states: + # - 0: mandatory + # - 1: optional, but nice to have + # - 2: LaTeX optional - pdf builds without it, but may have visual impact + + def check_missing(self, progs): + for prog, is_optional in sorted(self.missing.items()): + # At least on some LTS distros like CentOS 7, texlive doesn't + # provide all packages we need. When such distros are + # detected, we have to disable PDF output. + # + # So, we need to ignore the packages that distros would + # need for LaTeX to work + if is_optional == 2 and not self.pdf: + self.optional -= 1 + continue + + if self.verbose_warn_install: + if is_optional: + print(f'Warning: better to also install "{prog}".') + else: + print(f'ERROR: please install "{prog}", otherwise, build won\'t work.') + + self.install += " " + progs.get(prog, prog) + + self.install = self.install.lstrip() + + def add_package(self, package, is_optional): + self.missing[package] = is_optional + if is_optional: + self.optional += 1 + else: + self.need += 1 + + def check_missing_file(self, files, package, is_optional): + for f in files: + if os.path.exists(f): + return + self.add_package(package, is_optional) + + def check_program(self, prog, is_optional): + found = self.which(prog) + if found: + return found + + self.add_package(prog, is_optional) + + return None + + def check_perl_module(self, prog, is_optional): + # While testing with lxc download template, one of the + # distros (Oracle) didn't have perl - nor even an option to install + # before installing oraclelinux-release-el9 package. + # + # Check it before running an error. If perl is not there, + # add it as a mandatory package, as some parts of the doc builder + # needs it. + if not self.which("perl"): + self.add_package("perl", 0) + self.add_package(prog, is_optional) + return + + try: + self.run(["perl", f"-M{prog}", "-e", "1"], check=True) + except subprocess.CalledProcessError: + self.add_package(prog, is_optional) + + def check_python_module(self, module, is_optional): + # FIXME: is it needed at the Python version? Maybe due to venv? + if not self.python_cmd: + return + + try: + self.run([self.python_cmd, "-c", f"import {module}"], check=True) + except subprocess.CalledProcessError: + self.add_package(module, is_optional) + + def check_rpm_missing(self, pkgs, is_optional): + for prog in pkgs: + try: + self.run(["rpm", "-q", prog], check=True) + except subprocess.CalledProcessError: + self.add_package(prog, is_optional) + + def check_pacman_missing(self, pkgs, is_optional): + for prog in pkgs: + try: + self.run(["pacman", "-Q", prog], check=True) + except subprocess.CalledProcessError: + self.add_package(prog, is_optional) + + def check_missing_tex(self, is_optional): + kpsewhich = self.which("kpsewhich") + for prog, package in self.texlive.items(): + + # If kpsewhich is not there, just add it to deps + if not kpsewhich: + self.add_package(package, is_optional) + continue + + # Check if the package is needed + try: + result = self.run( + [kpsewhich, prog], stdout=subprocess.PIPE, text=True, check=True + ) + + # Didn't find. Add it + if not result.stdout.strip(): + self.add_package(package, is_optional) + + except subprocess.CalledProcessError: + # kpsewhich returned an error. Add it, just in case + self.add_package(package, is_optional) + + def get_sphinx_fname(self): + if "SPHINXBUILD" in os.environ: + return os.environ["SPHINXBUILD"] + + fname = "sphinx-build" + if self.which(fname): + return fname + + fname = "sphinx-build-3" + if self.which(fname): + self.need_symlink = 1 + return fname + + return "" + + def get_sphinx_version(self, cmd): + try: + result = self.run([cmd, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, check=True) + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + for line in result.stdout.split("\n"): + match = re.match(r"^sphinx-build\s+([\d\.]+)(?:\+(?:/[\da-f]+)|b\d+)?\s*$", line) + if match: + return parse_version(match.group(1)) + + match = re.match(r"^Sphinx.*\s+([\d\.]+)\s*$", line) + if match: + return parse_version(match.group(1)) + + def check_sphinx(self): + try: + with open(self.conf, "r", encoding="utf-8") as f: + for line in f: + match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line) + if match: + self.min_version = parse_version(match.group(1)) + break + except IOError: + sys.exit(f"Can't open {self.conf}") + + if not self.min_version: + sys.exit(f"Can't get needs_sphinx version from {self.conf}") + + self.virtenv_dir = self.virtenv_prefix[0] + "latest" + + sphinx = self.get_sphinx_fname() + if not sphinx: + self.need_sphinx = 1 + return + + self.cur_version = self.get_sphinx_version(sphinx) + if not self.cur_version: + sys.exit(f"{sphinx} didn't return its version") + + if self.cur_version < self.min_version: + curver = ver_str(self.cur_version) + minver = ver_str(self.min_version) + + print(f"ERROR: Sphinx version is {curver}. It should be >= {minver}") + self.need_sphinx = 1 + return + + # On version check mode, just assume Sphinx has all mandatory deps + if self.version_check and self.cur_version >= RECOMMENDED_VERSION: + sys.exit(0) + + def catcheck(self, filename): + if os.path.exists(filename): + with open(filename, "r", encoding="utf-8") as f: + return f.read().strip() + return "" + + # + # Distro-specific hints methods + # + + def give_debian_hints(self): + progs = { + "Pod::Usage": "perl-modules", + "convert": "imagemagick", + "dot": "graphviz", + "ensurepip": "python3-venv", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2-bin", + "virtualenv": "virtualenv", + "xelatex": "texlive-xetex", + "yaml": "python3-yaml", + } + + if self.pdf: + pdf_pkgs = { + "texlive-lang-chinese": [ + "/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty", + ], + "fonts-dejavu": [ + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + ], + "fonts-noto-cjk": [ + "/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc", + ], + } + + for package, files in pdf_pkgs.items(): + self.check_missing_file(files, package, 2) + + self.check_program("dvipng", 2) + + self.check_missing(progs) + + if not self.need and not self.optional: + return + + if self.verbose_warn_install: + print("You should run:") + print(f"\n\tsudo apt-get install {self.install}") + + def give_redhat_hints(self): + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2-tools", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive-xetex-bin", + "yaml": "python3-pyyaml", + } + + fedora26_opt_pkgs = [ + "graphviz-gd", # Fedora 26: needed for PDF support + ] + + fedora_tex_pkgs = [ + "dejavu-sans-fonts", + "dejavu-sans-mono-fonts", + "dejavu-serif-fonts", + "texlive-collection-fontsrecommended", + "texlive-collection-latex", + "texlive-xecjk", + ] + + old = 0 + rel = None + pkg_manager = "dnf" + + match = re.search(r"(release|Linux)\s+(\d+)", self.system_release) + if match: + rel = int(match.group(2)) + + if not rel: + print("Couldn't identify release number") + old = 1 + self.pdf = False + elif re.search("Fedora", self.system_release): + # Fedora 38 and upper use this CJK font + + noto_sans_redhat = "google-noto-sans-cjk-fonts" + else: + # Almalinux, CentOS, RHEL, ... + + # at least up to version 9 (and Fedora < 38), that's the CJK font + noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts" + + progs["virtualenv"] = "python-virtualenv" + + if rel and rel < 8: + old = 1 + self.pdf = False + + # RHEL 7 is in ELS, currently up to Jun, 2026 + + print("Note: texlive packages on RHEL/CENTOS <= 7 are incomplete. Can't support PDF output") + print("If you want to build PDF, please read:") + print("\thttps://www.systutorials.com/241660/how-to-install-tex-live-on-centos-7-linux/") + + if self.pdf: + pdf_pkgs = [ + "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc", + ] + + self.check_missing_file(pdf_pkgs, noto_sans_redhat, 2) + + if not old: + self.check_rpm_missing(fedora26_opt_pkgs, 2) + self.check_rpm_missing(fedora_tex_pkgs, 2) + + self.check_missing_tex(2) + + self.check_missing(progs) + + if not self.need and not self.optional: + return + + if self.verbose_warn_install: + print("You should run:") + + if old: + # dnf is there since Fedora 18+ and RHEL 8 + pkg_manager = "yum" + + print(f"\n\tsudo {pkg_manager} install -y {self.install}") + + def give_opensuse_hints(self): + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive-xetex-bin", + "yaml": "python3-pyyaml", + } + + # On Tumbleweed, this package is also named rsvg-convert + if not re.search(r"Tumbleweed", self.system_release): + progs["rsvg-convert"] = "rsvg-view" + + suse_tex_pkgs = [ + "texlive-babel-english", + "texlive-caption", + "texlive-colortbl", + "texlive-courier", + "texlive-dvips", + "texlive-helvetic", + "texlive-makeindex", + "texlive-metafont", + "texlive-metapost", + "texlive-palatino", + "texlive-preview", + "texlive-times", + "texlive-zapfchan", + "texlive-zapfding", + ] + + progs["latexmk"] = "texlive-latexmk-bin" + + # FIXME: add support for installing CJK fonts + # + # I tried hard, but was unable to find a way to install + # "Noto Sans CJK SC" on openSUSE + + if self.pdf: + self.check_rpm_missing(suse_tex_pkgs, 2) + if self.pdf: + self.check_missing_tex(2) + self.check_missing(progs) + + if not self.need and not self.optional: + return + + if self.verbose_warn_install: + print("You should run:") + print(f"\n\tsudo zypper install --no-recommends {self.install}") + + def give_mageia_hints(self): + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive", + "yaml": "python3-yaml", + } + + tex_pkgs = [ + "texlive-fontsextra", + ] + + if re.search(r"OpenMandriva", self.system_release): + packager_cmd = "dnf install" + noto_sans = "noto-sans-cjk-fonts" + tex_pkgs = ["texlive-collection-fontsextra"] + else: + packager_cmd = "urpmi" + noto_sans = "google-noto-sans-cjk-ttc-fonts" + + progs["latexmk"] = "texlive-collection-basic" + + if self.pdf: + pdf_pkgs = [ + "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/TTF/NotoSans-Regular.ttf", + ] + + self.check_missing_file(pdf_pkgs, noto_sans, 2) + self.check_rpm_missing(tex_pkgs, 2) + + self.check_missing(progs) + + if not self.need and not self.optional: + return + if self.verbose_warn_install: + print("You should run:") + print(f"\n\tsudo {packager_cmd} {self.install}") + + def give_arch_linux_hints(self): + progs = { + "convert": "imagemagick", + "dot": "graphviz", + "latexmk": "texlive-core", + "rsvg-convert": "extra/librsvg", + "virtualenv": "python-virtualenv", + "xelatex": "texlive-xetex", + "yaml": "python-yaml", + } + + archlinux_tex_pkgs = [ + "texlive-core", + "texlive-latexextra", + "ttf-dejavu", + ] + + if self.pdf: + self.check_pacman_missing(archlinux_tex_pkgs, 2) + + self.check_missing_file( + ["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], + "noto-fonts-cjk", + 2, + ) + + self.check_missing(progs) + + if not self.need and not self.optional: + return + if self.verbose_warn_install: + print("You should run:") + print(f"\n\tsudo pacman -S {self.install}") + + def give_gentoo_hints(self): + progs = { + "convert": "media-gfx/imagemagick", + "dot": "media-gfx/graphviz", + "rsvg-convert": "gnome-base/librsvg", + "virtualenv": "dev-python/virtualenv", + "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", + "yaml": "dev-python/pyyaml", + } + + if self.pdf: + pdf_pkgs = { + "media-fonts/dejavu": [ + "/usr/share/fonts/dejavu/DejaVuSans.ttf", + ], + "media-fonts/noto-cjk": [ + "/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", + "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc", + ], + } + for package, files in pdf_pkgs.items(): + self.check_missing_file(files, package, 2) + + self.check_missing(progs) + + if not self.need and not self.optional: + return + + if self.verbose_warn_install: + print("You should run:") + print("\n") + + imagemagick = "media-gfx/imagemagick svg png" + cairo = "media-gfx/graphviz cairo pdf" + portage_imagemagick = "/etc/portage/package.use/imagemagick" + portage_cairo = "/etc/portage/package.use/graphviz" + + result = self.run(["grep", "imagemagick", portage_imagemagick], + stdout=subprocess.PIPE, text=True) + if not result.stdout.strip(): + print(f"\tsudo su -c 'echo \"{imagemagick}\" > {portage_imagemagick}'") + + result = self.run(["grep", "graphviz", portage_cairo], + stdout=subprocess.PIPE, text=True) + + if not result.stdout.strip(): + print(f"\tsudo su -c 'echo \"{cairo}\" > {portage_cairo}'") + + print(f"\tsudo emerge --ask {self.install}") + + # + # Dispatch the check to an os_specific hinter + # + + def check_distros(self): + # OS-specific hints logic + os_hints = { + re.compile("Red Hat Enterprise Linux"): self.give_redhat_hints, + re.compile("Fedora"): self.give_redhat_hints, + re.compile("AlmaLinux"): self.give_redhat_hints, + re.compile("Amazon Linux"): self.give_redhat_hints, + re.compile("CentOS"): self.give_redhat_hints, + re.compile("openEuler"): self.give_redhat_hints, + re.compile("Oracle Linux Server"): self.give_redhat_hints, + re.compile("Rocky Linux"): self.give_redhat_hints, + re.compile("Scientific Linux"): self.give_redhat_hints, + re.compile("Springdale Open Enterprise"): self.give_redhat_hints, + + re.compile("Ubuntu"): self.give_debian_hints, + re.compile("Debian"): self.give_debian_hints, + re.compile("Devuan"): self.give_debian_hints, + re.compile("Kali"): self.give_debian_hints, + re.compile("Mint"): self.give_debian_hints, + + re.compile("openSUSE"): self.give_opensuse_hints, + + re.compile("Mageia"): self.give_mageia_hints, + re.compile("OpenMandriva"): self.give_mageia_hints, + + re.compile("Arch Linux"): self.give_arch_linux_hints, + re.compile("Gentoo"): self.give_gentoo_hints, + } + + # If the OS is detected, use per-OS hint logic + for regex, os_hint in os_hints.items(): + if regex.search(self.system_release): + os_hint() + + return + + # + # Fall-back to generic hint code for other distros + # That's far from ideal, specially for LaTeX dependencies. + # + progs = {"sphinx-build": "sphinx"} + if self.pdf: + self.check_missing_tex(2) + + self.check_missing(progs) + + print(f"I don't know distro {self.system_release}.") + print("So, I can't provide you a hint with the install procedure.") + print("There are likely missing dependencies.") + + # + # Common dependencies + # + def deactivate_help(self): + print("\n If you want to exit the virtualenv, you can use:") + print("\tdeactivate") + + def get_virtenv(self): + cwd = os.getcwd() + + activates = [] + + # Add all sphinx prefixes with possible version numbers + for p in self.virtenv_prefix: + activates += glob(f"{cwd}/{p}[0-9]*/bin/activate") + + activates.sort(reverse=True, key=str.lower) + + # Place sphinx_latest first, if it exists + for p in self.virtenv_prefix: + activates = glob(f"{cwd}/{p}*latest/bin/activate") + activates + + ver = (0, 0, 0) + for f in activates: + # Discard too old Sphinx virtual environments + match = re.search(r"(\d+)\.(\d+)\.(\d+)", f) + if match: + ver = (int(match.group(1)), int(match.group(2)), int(match.group(3))) + + if ver < self.min_version: + continue + + sphinx_cmd = f.replace("activate", "sphinx-build") + if not os.path.isfile(sphinx_cmd): + continue + + ver = self.get_sphinx_version(sphinx_cmd) + + if not ver: + venv_dir = f.replace("/bin/activate", "") + print(f"Warning: virtual environment {venv_dir} is not working.\n" \ + "Python version upgrade? Remove it with:\n\n" \ + "\trm -rf {venv_dir}\n\n") + else: + if self.need_sphinx and ver >= self.min_version: + return (f, ver) + elif parse_version(ver) > self.cur_version: + return (f, ver) + + return ("", ver) + + def recommend_sphinx_upgrade(self): + # Avoid running sphinx-builds from venv if cur_version is good + if self.cur_version and self.cur_version >= RECOMMENDED_VERSION: + self.latest_avail_ver = self.cur_version + return None + + # Get the highest version from sphinx_*/bin/sphinx-build and the + # corresponding command to activate the venv/virtenv + self.activate_cmd, self.venv_ver = self.get_virtenv() + + # Store the highest version from Sphinx existing virtualenvs + if self.activate_cmd and self.venv_ver > self.cur_version: + self.latest_avail_ver = self.venv_ver + else: + if self.cur_version: + self.latest_avail_ver = self.cur_version + else: + self.latest_avail_ver = (0, 0, 0) + + # As we don't know package version of Sphinx, and there's no + # virtual environments, don't check if upgrades are needed + if not self.virtualenv: + if not self.latest_avail_ver: + return None + + return self.latest_avail_ver + + # Either there are already a virtual env or a new one should be created + self.need_pip = 1 + + if not self.latest_avail_ver: + return None + + # Return if the reason is due to an upgrade or not + if self.latest_avail_ver != (0, 0, 0): + if self.latest_avail_ver < RECOMMENDED_VERSION: + self.rec_sphinx_upgrade = 1 + + return self.latest_avail_ver + + def recommend_sphinx_version(self, virtualenv_cmd): + # The logic here is complex, as it have to deal with different versions: + # - minimal supported version; + # - minimal PDF version; + # - recommended version. + # It also needs to work fine with both distro's package and venv/virtualenv + + # Version is OK. Nothing to do. + if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: + return + + if not self.need_sphinx: + # sphinx-build is present and its version is >= $min_version + + # only recommend enabling a newer virtenv version if makes sense. + if self.latest_avail_ver and self.latest_avail_ver > self.cur_version: + print("\nYou may also use the newer Sphinx version {self.latest_avail_ver} with:") + if f"{self.virtenv_prefix}" in os.getcwd(): + print("\tdeactivate") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + + if self.latest_avail_ver and self.latest_avail_ver >= RECOMMENDED_VERSION: + return + + if not self.virtualenv: + # No sphinx either via package or via virtenv. As we can't + # Compare the versions here, just return, recommending the + # user to install it from the package distro. + if not self.latest_avail_ver or self.latest_avail_ver == (0, 0, 0): + return + + # User doesn't want a virtenv recommendation, but he already + # installed one via virtenv with a newer version. + # So, print commands to enable it + if self.latest_avail_ver > self.cur_version: + print("\nYou may also use the Sphinx virtualenv version {self.latest_avail_ver} with:") + if f"{self.virtenv_prefix}" in os.getcwd(): + print("\tdeactivate") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + print("\n") + else: + if self.need_sphinx: + self.need += 1 + + # Suggest newer versions if current ones are too old + if self.latest_avail_ver and self.latest_avail_ver >= self.min_version: + if self.latest_avail_ver >= RECOMMENDED_VERSION: + print("\nNeed to activate Sphinx (version {self.latest_avail_ver}) on virtualenv with:") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + + # Version is above the minimal required one, but may be + # below the recommended one. So, print warnings/notes + if self.latest_avail_ver < RECOMMENDED_VERSION: + print(f"Warning: It is recommended at least Sphinx version {RECOMMENDED_VERSION}.") + + # At this point, either it needs Sphinx or upgrade is recommended, + # both via pip + + if self.rec_sphinx_upgrade: + if not self.virtualenv: + print("Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n") + else: + print("To upgrade Sphinx, use:\n\n") + else: + print("\nSphinx needs to be installed either:\n1) via pip/pypi with:\n") + + self.python_cmd = self.find_python_no_venv() + + print(f"\t{virtualenv_cmd} {self.virtenv_dir}") + print(f"\t. {self.virtenv_dir}/bin/activate") + print(f"\tpip install -r {self.requirement_file}") + self.deactivate_help() + + print("\n2) As a package with:") + + old_need = self.need + old_optional = self.optional + self.missing = {} + self.pdf = False + self.optional = 0 + self.install = "" + old_verbose = self.verbose_warn_install + self.verbose_warn_install = 0 + + self.add_package("python-sphinx", 0) + + self.check_distros() + + self.need = old_need + self.optional = old_optional + self.verbose_warn_install = old_verbose + + print("\n" \ + " Please note that Sphinx >= 3.0 will currently produce false-positive\n" \ + " warning when the same name is used for more than one type (functions,\n" \ + " structs, enums,...). This is known Sphinx bug. For more details, see:\n" \ + "\thttps://github.com/sphinx-doc/sphinx/pull/8313") + + def check_needs(self): + self.get_system_release() + + # Check if Sphinx is already accessible from current environment + self.check_sphinx() + + if self.system_release: + print(f"Detected OS: {self.system_release}.") + else: + print("Unknown OS") + if self.cur_version != (0, 0, 0): + ver = ver_str(self.cur_version) + print(f"Sphinx version: {ver}\n") + + # FIXME: Check python command line, trying first python3 + self.python_cmd = self.which("python3") + if not self.python_cmd: + self.python_cmd = self.check_program("python", 0) + + # Check the type of virtual env, depending on Python version + if self.python_cmd: + if self.virtualenv: + try: + result = self.run( + [self.python_cmd, "--version"], + capture_output=True, + text=True, + check=True, + ) + + output = result.stdout + result.stderr + + match = re.search(r"(\d+)\.(\d+)\.", output) + if match: + major = int(match.group(1)) + minor = int(match.group(2)) + + if major < 3: + sys.exit("Python 3 is required to build the kernel docs") + if major == 3 and minor < 3: + self.need_virtualenv = True + else: + sys.exit(f"Warning: couldn't identify {self.python_cmd} version!") + + except subprocess.CalledProcessError as e: + sys.exit(f"Error checking Python version: {e}") + else: + self.add_package("python-sphinx", 0) + + self.venv_ver = self.recommend_sphinx_upgrade() + + virtualenv_cmd = "" + + if self.need_pip: + # Set virtualenv command line, if python < 3.3 + # FIXME: can be removed as we're now with an upper min requirement + # but then we need to check python version + if self.need_virtualenv: + virtualenv_cmd = self.which("virtualenv-3") + if not virtualenv_cmd: + virtualenv_cmd = self.which("virtualenv-3.5") + if not virtualenv_cmd: + self.check_program("virtualenv", 0) + virtualenv_cmd = "virtualenv" + else: + virtualenv_cmd = f"{self.python_cmd} -m venv" + self.check_python_module("ensurepip", 0) + + # Check for needed programs/tools + self.check_perl_module("Pod::Usage", 0) + self.check_python_module("yaml", 0) + self.check_program("make", 0) + self.check_program("gcc", 0) + self.check_program("dot", 1) + self.check_program("convert", 1) + + if self.pdf: + # Extra PDF files - should use 2 for LaTeX is_optional + self.check_program("xelatex", 2) + self.check_program("rsvg-convert", 2) + self.check_program("latexmk", 2) + + # Do distro-specific checks and output distro-install commands + self.check_distros() + + if not self.python_cmd: + if self.need == 1: + sys.exit("Can't build as 1 mandatory dependency is missing") + elif self.need: + sys.exit(f"Can't build as {self.need} mandatory dependencies are missing") + + # Check if sphinx-build is called sphinx-build-3 + if self.need_symlink: + sphinx_path = self.which("sphinx-build-3") + if sphinx_path: + print(f"\tsudo ln -sf {sphinx_path} /usr/bin/sphinx-build\n") + + self.recommend_sphinx_version(virtualenv_cmd) + print("") + + if not self.optional: + print("All optional dependencies are met.") + + if self.need == 1: + sys.exit("Can't build as 1 mandatory dependency is missing") + elif self.need: + sys.exit(f"Can't build as {self.need} mandatory dependencies are missing") + + print("Needed package dependencies are met.") + + def get_system_release(self): + """ + Determine the system type. There's no unique way that would work + with all distros with a minimal package install. So, several + methods are used here. + + By default, it will use lsb_release function. If not available, it will + fail back to reading the known different places where the distro name + is stored. + + Several modern distros now have /etc/os-release, which usually have + a decent coverage. + """ + + if self.which("lsb_release"): + result = self.run(["lsb_release", "-d"], capture_output=True, text=True) + self.system_release = result.stdout.replace("Description:", "").strip() + + release_files = [ + "/etc/system-release", + "/etc/redhat-release", + "/etc/lsb-release", + "/etc/gentoo-release", + ] + + if not self.system_release: + for f in release_files: + self.system_release = self.catcheck(f) + if self.system_release: + break + + # This seems more common than LSB these days + if not self.system_release: + os_var = {} + try: + with open("/etc/os-release", "r", encoding="utf-8") as f: + for line in f: + match = re.match(r"^([\w\d\_]+)=\"?([^\"]*)\"?\n", line) + if match: + os_var[match.group(1)] = match.group(2) + + self.system_release = os_var.get("NAME", "") + if "VERSION_ID" in os_var: + self.system_release += " " + os_var["VERSION_ID"] + elif "VERSION" in os_var: + self.system_release += " " + os_var["VERSION"] + except IOError: + pass + + if not self.system_release: + self.system_release = self.catcheck("/etc/issue") + + self.system_release = self.system_release.strip() + +DESCRIPTION = """ +Process some flags related to Sphinx installation and documentation build. +""" + + +def main(): + parser = argparse.ArgumentParser(description=DESCRIPTION) + + parser.add_argument( + "--no-virtualenv", + action="store_false", + dest="virtualenv", + help="Recommend installing Sphinx instead of using a virtualenv", + ) + + parser.add_argument( + "--no-pdf", + action="store_false", + dest="pdf", + help="Don't check for dependencies required to build PDF docs", + ) + + parser.add_argument( + "--version-check", + action="store_true", + dest="version_check", + help="If version is compatible, don't check for missing dependencies", + ) + + args = parser.parse_args() + + checker = SphinxDependencyChecker(args) + + checker.check_needs() + + +if __name__ == "__main__": + main() -- cgit From 56a8767751c4bd2b93ba8840a78eccae7018d57c Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:21 +0200 Subject: scripts: sphinx-pre-install: Make it compatible with Python 3.6 The minimal version requirements we have is 3.9. Yet, the script which detects it is this one. So, let's try supporting an old version here, as we may want to suggest to upgrade Python version to build the docs. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/39d6e27a047bc3cc8208ac5e11fe6ba44faff9c4.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index dcee2181b72f..71d86b230b22 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -115,7 +115,8 @@ class SphinxDependencyChecker: def find_python_no_venv(): # FIXME: does it makes sense now that this script is in Python? - result = subprocess.run(["pwd"], capture_output=True, text=True) + result = SphinxDependencyChecker.run(["pwd"], capture_output=True, + text=True) cur_dir = result.stdout.strip() python_names = ["python3", "python"] @@ -135,12 +136,23 @@ class SphinxDependencyChecker: def run(*args, **kwargs): """Excecute a command, hiding its output by default""" - if not kwargs.get('capture_output', False): + capture_output = kwargs.pop('capture_output', False) + + if capture_output: + if 'stdout' not in kwargs: + kwargs['stdout'] = subprocess.PIPE + if 'stderr' not in kwargs: + kwargs['stderr'] = subprocess.PIPE + else: if 'stdout' not in kwargs: kwargs['stdout'] = subprocess.DEVNULL if 'stderr' not in kwargs: kwargs['stderr'] = subprocess.DEVNULL + # Don't break with older Python versions + if 'text' in kwargs and sys.version_info < (3, 7): + kwargs['universal_newlines'] = kwargs.pop('text') + return subprocess.run(*args, **kwargs) # -- cgit From 728648b6a5faa1d6be1e90c04a71481ba33fd08d Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:22 +0200 Subject: scripts: sphinx-pre-install: run on a supported version The scripts/sphinx-pre-install is used to detect problems at the system environment and adjust it to build the Kernel documentation. If the version is too old, it won't run, though. Check if the version which started the script is valid. If not, seek for a new one that is compatible with documentation build. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/76627055a7f82f6a79296ddbd873fa5ac8f82a1d.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 65 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 71d86b230b22..3912359d2bae 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -5,6 +5,9 @@ # pylint: disable=C0103,C0114,C0115,C0116,C0301 # pylint: disable=R0902,R0904,R0912,R0915,R1705,R1710,E1121 +# Note: this script requires at least Python 3.6 to run. +# Don't add changes not compatible with it, it is meant to report +# incompatible python versions. import argparse import os @@ -16,7 +19,6 @@ from glob import glob def parse_version(version): """Convert a major.minor.patch version into a tuple""" -# return tuple(int(x) for x in version.split(".")) @@ -27,6 +29,7 @@ def ver_str(version): RECOMMENDED_VERSION = parse_version("3.4.3") +MIN_PYTHON_VERSION = parse_version("3.7") class SphinxDependencyChecker: @@ -132,6 +135,65 @@ class SphinxDependencyChecker: # Python not found at the PATH return python_names[-1] + @staticmethod + def get_python_version(cmd): + + result = SphinxDependencyChecker.run([cmd, "--version"], + capture_output=True, text=True) + version = result.stdout.strip() + + match = re.search(r"(\d+\.\d+\.\d+)", version) + if match: + return parse_version(match.group(1)) + + print(f"Can't parse version {version}") + return (0, 0, 0) + + @staticmethod + def find_python(): + + patterns = [ + "python3.[0-9]", + "python3.[0-9][0-9]", + ] + + new_python_cmd = None + + # Seek for a python binary newer than MIN_PYTHON_VERSION + for path in os.getenv("PATH", "").split(":"): + for pattern in patterns: + for cmd in glob(os.path.join(path, pattern)): + if os.path.isfile(cmd) and os.access(cmd, os.X_OK): + version = SphinxDependencyChecker.get_python_version(cmd) + if version >= MIN_PYTHON_VERSION: + return(cmd) + + @staticmethod + def check_python(): + + cur_ver = sys.version_info[:3] + if cur_ver >= MIN_PYTHON_VERSION: + return + + python_ver = ver_str(cur_ver) + + new_python_cmd = SphinxDependencyChecker.find_python() + if not new_python_cmd: + print(f"ERROR: Python version {python_ver} is not spported anymore") + print(f" Can't find a new version. This script may fail") + return + + # Restart script using the newer version + script_path = os.path.abspath(sys.argv[0]) + args = [new_python_cmd, script_path] + sys.argv[1:] + + print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") + + try: + os.execv(new_python_cmd, args) + except OSError as e: + sys.exit(f"Failed to restart with {new_python_cmd}: {e}") + @staticmethod def run(*args, **kwargs): """Excecute a command, hiding its output by default""" @@ -1107,6 +1169,7 @@ def main(): checker = SphinxDependencyChecker(args) + checker.check_python() checker.check_needs() -- cgit From ea5dd67722c1ac8ae65ae5a9390452931890f52e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:23 +0200 Subject: scripts: sphinx-pre-install: drop obsolete routines Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/afb97854f05b0852f4c6c1fcee102e9697c14cd8.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 3912359d2bae..b639acd455cc 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -114,27 +114,6 @@ class SphinxDependencyChecker: return None - @staticmethod - def find_python_no_venv(): - # FIXME: does it makes sense now that this script is in Python? - - result = SphinxDependencyChecker.run(["pwd"], capture_output=True, - text=True) - cur_dir = result.stdout.strip() - - python_names = ["python3", "python"] - - for d in os.environ.get("PATH", "").split(":"): - if f"{cur_dir}/sphinx" in d: - continue - - for p in python_names: - if os.access(os.path.join(d, p), os.X_OK): - return os.path.join(d, p) - - # Python not found at the PATH - return python_names[-1] - @staticmethod def get_python_version(cmd): @@ -940,7 +919,7 @@ class SphinxDependencyChecker: else: print("\nSphinx needs to be installed either:\n1) via pip/pypi with:\n") - self.python_cmd = self.find_python_no_venv() + self.python_cmd = os.path.abspath(sys.argv[0]) print(f"\t{virtualenv_cmd} {self.virtenv_dir}") print(f"\t. {self.virtenv_dir}/bin/activate") -- cgit From f25bf12afc03ce88ca82fd2ed67fbee5c1ed0cd8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:24 +0200 Subject: scripts: sphinx-pre-install: drop support for old virtualenv Up to Python 3.2, the virtual environment were created via virtualenv binary. As we dropped support for such old version, clean up the code. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/31afe394bcfd8f7e450263c1922d2c73b91d36d8.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 71 ++++++++++++------------------------------- 1 file changed, 19 insertions(+), 52 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index b639acd455cc..0a73b1b33842 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -80,7 +80,6 @@ class SphinxDependencyChecker: self.need_symlink = 0 self.need_sphinx = 0 self.need_pip = 0 - self.need_virtualenv = 0 self.rec_sphinx_upgrade = 0 self.verbose_warn_install = 1 @@ -919,12 +918,14 @@ class SphinxDependencyChecker: else: print("\nSphinx needs to be installed either:\n1) via pip/pypi with:\n") - self.python_cmd = os.path.abspath(sys.argv[0]) - - print(f"\t{virtualenv_cmd} {self.virtenv_dir}") - print(f"\t. {self.virtenv_dir}/bin/activate") - print(f"\tpip install -r {self.requirement_file}") - self.deactivate_help() + if not virtualenv_cmd: + print(" Currently not possible.\n") + print(" Please upgrade Python to a newer version and run this script again") + else: + print(f"\t{virtualenv_cmd} {self.virtenv_dir}") + print(f"\t. {self.virtenv_dir}/bin/activate") + print(f"\tpip install -r {self.requirement_file}") + self.deactivate_help() print("\n2) As a package with:") @@ -953,6 +954,7 @@ class SphinxDependencyChecker: def check_needs(self): self.get_system_release() + self.python_cmd = sys.executable # Check if Sphinx is already accessible from current environment self.check_sphinx() @@ -965,56 +967,21 @@ class SphinxDependencyChecker: ver = ver_str(self.cur_version) print(f"Sphinx version: {ver}\n") - # FIXME: Check python command line, trying first python3 - self.python_cmd = self.which("python3") - if not self.python_cmd: - self.python_cmd = self.check_program("python", 0) - # Check the type of virtual env, depending on Python version - if self.python_cmd: - if self.virtualenv: - try: - result = self.run( - [self.python_cmd, "--version"], - capture_output=True, - text=True, - check=True, - ) - - output = result.stdout + result.stderr - - match = re.search(r"(\d+)\.(\d+)\.", output) - if match: - major = int(match.group(1)) - minor = int(match.group(2)) - - if major < 3: - sys.exit("Python 3 is required to build the kernel docs") - if major == 3 and minor < 3: - self.need_virtualenv = True - else: - sys.exit(f"Warning: couldn't identify {self.python_cmd} version!") - - except subprocess.CalledProcessError as e: - sys.exit(f"Error checking Python version: {e}") - else: - self.add_package("python-sphinx", 0) + virtualenv_cmd = None - self.venv_ver = self.recommend_sphinx_upgrade() + if sys.version_info < MIN_PYTHON_VERSION: + min_ver = ver_str(MIN_PYTHON_VERSION) + print(f"ERROR: at least python {min_ver} is required to build the kernel docs") + self.need_sphinx = 1 - virtualenv_cmd = "" + self.venv_ver = self.recommend_sphinx_upgrade() if self.need_pip: - # Set virtualenv command line, if python < 3.3 - # FIXME: can be removed as we're now with an upper min requirement - # but then we need to check python version - if self.need_virtualenv: - virtualenv_cmd = self.which("virtualenv-3") - if not virtualenv_cmd: - virtualenv_cmd = self.which("virtualenv-3.5") - if not virtualenv_cmd: - self.check_program("virtualenv", 0) - virtualenv_cmd = "virtualenv" + if sys.version_info < MIN_PYTHON_VERSION: + self.need_pip = False + print("Warning: python version is not supported.") + else: virtualenv_cmd = f"{self.python_cmd} -m venv" self.check_python_module("ensurepip", 0) -- cgit From 2cb4877b74349e33be9e209fbcce6114c9a38b9f Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:25 +0200 Subject: scripts: sphinx-pre-install: Address issues with OpenSUSE Leap 15.x On openSUSE Leap 15.6, which is the current LTS version, has two Sphinx packages. The normal one requires Python 3.6, which we don't support anymore. However, it also has Python 3.11 with a newer Sphinx version (7.2.6). Suggest the newer version: Detected OS: openSUSE Leap 15.6. ERROR: at least python 3.7 is required to build the kernel docs Warning: python version is not supported. Warning: better to also install "convert". Warning: better to also install "dot". ERROR: please install "yaml", otherwise, build won't work. You should run: sudo zypper install --no-recommends ImageMagick graphviz python311-pyyaml Sphinx needs to be installed either: 1) via pip/pypi with: Currently not possible. Please upgrade Python to a newer version and run this script again 2) As a package with: sudo zypper install --no-recommends python311-Sphinx Please note that Sphinx >= 3.0 will currently produce false-positive warning when the same name is used for more than one type (functions, structs, enums,...). This is known Sphinx bug. For more details, see: https://github.com/sphinx-doc/sphinx/pull/8313 Can't build as 2 mandatory dependencies are missing Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/a1600e292b63f96f40163e350238812158ebd6c2.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 0a73b1b33842..eca42d90ed01 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -556,6 +556,22 @@ class SphinxDependencyChecker: progs["latexmk"] = "texlive-latexmk-bin" + match = re.search(r"(Leap)\s+(\d+).(\d)", self.system_release) + if match: + rel = int(match.group(2)) + + # Leap 15.x uses Python 3.6, which is not compatible with + # the build system anymore. Suggest Python 3.11 + if rel == 15: + if not self.which(self.python_cmd): + self.add_package(self.python_cmd, 0) + + progs.update({ + "python-sphinx": "python311-Sphinx", + "virtualenv": "python311-virtualenv", + "yaml": "python311-pyyaml", + }) + # FIXME: add support for installing CJK fonts # # I tried hard, but was unable to find a way to install -- cgit From c5ffae0fa965884d09458002e3d990c039602809 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:26 +0200 Subject: scripts: sphinx-pre-install: fix opensuse Leap hint for PyYAML On Leap, the name of the package is python311-PyYAML. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0266c5e28e136c62de33d2d7f7d5e69b273e0d01.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index eca42d90ed01..65438c198674 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -569,7 +569,7 @@ class SphinxDependencyChecker: progs.update({ "python-sphinx": "python311-Sphinx", "virtualenv": "python311-virtualenv", - "yaml": "python311-pyyaml", + "yaml": "python311-PyYAML", }) # FIXME: add support for installing CJK fonts -- cgit From 582b0f95c92e0a0ec79d23ed434786251bd4378b Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:27 +0200 Subject: scripts: sphinx-pre-install: fix support for gentoo Some gentoo packages have changes. Fix it. While here, improve emerge portage use logic. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/283987633aae7d99e0eff41e24eb6285c1b6b335.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 65438c198674..b793796329c8 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -669,12 +669,13 @@ class SphinxDependencyChecker: def give_gentoo_hints(self): progs = { - "convert": "media-gfx/imagemagick", - "dot": "media-gfx/graphviz", - "rsvg-convert": "gnome-base/librsvg", - "virtualenv": "dev-python/virtualenv", - "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", - "yaml": "dev-python/pyyaml", + "convert": "media-gfx/imagemagick", + "dot": "media-gfx/graphviz", + "rsvg-convert": "gnome-base/librsvg", + "virtualenv": "dev-python/virtualenv", + "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", + "yaml": "dev-python/pyyaml", + "python-sphinx": "dev-python/sphinx", } if self.pdf: @@ -699,21 +700,17 @@ class SphinxDependencyChecker: print("You should run:") print("\n") - imagemagick = "media-gfx/imagemagick svg png" - cairo = "media-gfx/graphviz cairo pdf" - portage_imagemagick = "/etc/portage/package.use/imagemagick" - portage_cairo = "/etc/portage/package.use/graphviz" - result = self.run(["grep", "imagemagick", portage_imagemagick], - stdout=subprocess.PIPE, text=True) - if not result.stdout.strip(): - print(f"\tsudo su -c 'echo \"{imagemagick}\" > {portage_imagemagick}'") - - result = self.run(["grep", "graphviz", portage_cairo], - stdout=subprocess.PIPE, text=True) + portages = [ + "media-gfx/imagemagick", + "media-gfx/graphviz", + ] - if not result.stdout.strip(): - print(f"\tsudo su -c 'echo \"{cairo}\" > {portage_cairo}'") + for p in portages: + result = self.run(["grep", p, "/etc/portage/package.use/*"], + stdout=subprocess.PIPE, text=True) + if not result.stdout.strip(): + print(f"\tsudo emerge -av1 {p}") print(f"\tsudo emerge --ask {self.install}") -- cgit From 94a161d998a103346088d97da6d1cfec4df97ddc Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:28 +0200 Subject: scripts: sphinx-pre-install: Address issues with OpenSUSE Tumbleweed On Tumbleweed, package names are named after python-313*, as it also has older python versions on it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/fe0b5f7c18d7b32e0229c0890371e1441ffea294.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index b793796329c8..0e165ad05fdb 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -571,6 +571,14 @@ class SphinxDependencyChecker: "virtualenv": "python311-virtualenv", "yaml": "python311-PyYAML", }) + else: + # Tumbleweed defaults to Python 3.11 + + progs.update({ + "python-sphinx": "python313-Sphinx", + "virtualenv": "python313-virtualenv", + "yaml": "python313-PyYAML", + }) # FIXME: add support for installing CJK fonts # -- cgit From 12bdcf898977172a93fc30668f8918da83c51f86 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:29 +0200 Subject: scripts: sphinx-pre-install: only show portage hints once On gentoo, doesn't repeat instructions about how to enable portage. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/d4e7ef1d5e1f0412eb9f9ae4913dc64c0821e002.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 0e165ad05fdb..808d31bfa790 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -89,6 +89,9 @@ class SphinxDependencyChecker: self.python_cmd = "" self.activate_cmd = "" + # Certain hints are meant to be shown only once + self.first_hint = True + self.min_version = (0, 0, 0) self.cur_version = (0, 0, 0) self.latest_avail_ver = (0, 0, 0) @@ -714,11 +717,14 @@ class SphinxDependencyChecker: "media-gfx/graphviz", ] - for p in portages: - result = self.run(["grep", p, "/etc/portage/package.use/*"], - stdout=subprocess.PIPE, text=True) - if not result.stdout.strip(): - print(f"\tsudo emerge -av1 {p}") + if self.first_hint: + for p in portages: + result = self.run(["grep", p, "/etc/portage/package.use/*"], + stdout=subprocess.PIPE, text=True) + if not result.stdout.strip(): + print(f"\tsudo emerge -av1 {p}") + + self.first_hint = False print(f"\tsudo emerge --ask {self.install}") -- cgit From 8f6f54c464d7fccfb8da46ca0afcb0860e6d73bb Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:30 +0200 Subject: scripts: sphinx-pre-install: cleanup rhel support Rhel < 8.0 is not supported anymore. Drop support for it. Rhel 8 is problematic: at least on the tests I did with a docker repo, it didn't work, but it could be due to the issue that it is actually different than a real One. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/62fe8ab243ad39f4964f1f74b965e43dc8f10e23.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 808d31bfa790..42f55e67256d 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -487,15 +487,31 @@ class SphinxDependencyChecker: progs["virtualenv"] = "python-virtualenv" - if rel and rel < 8: + if not rel or rel < 8: old = 1 self.pdf = False - # RHEL 7 is in ELS, currently up to Jun, 2026 + print("ERROR: Distro not supported. Too old?") + return + + # TODO: check if RHEL8 still works. + # On my tests with docker "redhat/ubi8" image, there's no + # python3-sphinx (or similar) package. It comes with Python 3.6, + # but there are other python packages over there, so it may be + # possible to work with venv. + + if self.first_hint: + print("Note: RHEL-based distros typically require extra repositories.\n" \ + "For most, enabling epel and crb are enough:\n" \ + "\tsudo dnf install -y epel-release", \ + "\tsudo dnf config-manager --set-enabled crb\n" \ + "Yet, some may have other required repositories. Those commands could be useful:" \ + "\tsudo dnf repolist all" \ + "\tsudo repoquery --available --info ", + "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want") + + self.first_hint = False - print("Note: texlive packages on RHEL/CENTOS <= 7 are incomplete. Can't support PDF output") - print("If you want to build PDF, please read:") - print("\thttps://www.systutorials.com/241660/how-to-install-tex-live-on-centos-7-linux/") if self.pdf: pdf_pkgs = [ -- cgit From 637fa6b38113fddca7da2f0507f709f46a44047a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:31 +0200 Subject: scripts: sphinx-pre-install: output Python and docutils version Specially when debugging issues, knowing the versions is important. Add it to the script output. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/96142ec916b9064c859b200639c3595115080322.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 42f55e67256d..de5bcfd052b5 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -154,6 +154,20 @@ class SphinxDependencyChecker: cur_ver = sys.version_info[:3] if cur_ver >= MIN_PYTHON_VERSION: + ver = ver_str(cur_ver) + print(f"Python version: {ver}") + + # This could be useful for debugging purposes + if SphinxDependencyChecker.which("docutils"): + result = SphinxDependencyChecker.run(["docutils", "--version"], + capture_output=True, text=True) + ver = result.stdout.strip() + match = re.search(r"(\d+\.\d+\.\d+)", ver) + if match: + ver = match.group(1) + + print(f"Docutils version: {ver}") + return python_ver = ver_str(cur_ver) -- cgit From cccc5389811a34eccbd0f531c89d284f1cebdd70 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:32 +0200 Subject: scripts: sphinx-pre-install: add a missing f-string marker I forgot one f-string marker, with turned to be affecting 3 lines, because of cut-and-paste ;-) Use the proper f-string marker to print Sphinx version at the hint lines. Yet, we don't want to print as a tuple, so call ver_str() for it. Ideally, we would be placing it directly at the f-string, but Python 3.6 f-string support was pretty much limited. Only 3.12 (PEP 701) makes it similar to Perl, allowing expressions inside it. It sounds that function call itself was introduced on 3.7. As we explicitly want this one to run on 3.6, as latest Leap comes with it, we can't use function calls on f-string. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/b0ad1795446b17a00ba2dd83f366e784253668e6.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index de5bcfd052b5..6a244105f7ef 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -914,12 +914,15 @@ class SphinxDependencyChecker: if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: return + if self.latest_avail_ver: + latest_avail_ver = ver_str(self.latest_avail_ver) + if not self.need_sphinx: # sphinx-build is present and its version is >= $min_version # only recommend enabling a newer virtenv version if makes sense. if self.latest_avail_ver and self.latest_avail_ver > self.cur_version: - print("\nYou may also use the newer Sphinx version {self.latest_avail_ver} with:") + print(f"\nYou may also use the newer Sphinx version {latest_avail_ver} with:") if f"{self.virtenv_prefix}" in os.getcwd(): print("\tdeactivate") print(f"\t. {self.activate_cmd}") @@ -940,7 +943,7 @@ class SphinxDependencyChecker: # installed one via virtenv with a newer version. # So, print commands to enable it if self.latest_avail_ver > self.cur_version: - print("\nYou may also use the Sphinx virtualenv version {self.latest_avail_ver} with:") + print(f"\nYou may also use the Sphinx virtualenv version {latest_avail_ver} with:") if f"{self.virtenv_prefix}" in os.getcwd(): print("\tdeactivate") print(f"\t. {self.activate_cmd}") @@ -954,7 +957,7 @@ class SphinxDependencyChecker: # Suggest newer versions if current ones are too old if self.latest_avail_ver and self.latest_avail_ver >= self.min_version: if self.latest_avail_ver >= RECOMMENDED_VERSION: - print("\nNeed to activate Sphinx (version {self.latest_avail_ver}) on virtualenv with:") + print(f"\nNeed to activate Sphinx (version {latest_avail_ver}) on virtualenv with:") print(f"\t. {self.activate_cmd}") self.deactivate_help() return -- cgit From 8b18e86f6c0063e33096bf84ffcd9fb9ced4844d Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:33 +0200 Subject: scripts: sphinx-pre-install: fix Leap support for rsvg-convert There is a test logic meant to be for Leap, renaming rsvg-convert package. Well, at least on latest Leap releases, this is wrong, causing install to fail. Drop it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/6fcc94533d860e2f6f4f2c7d6ceb6cca70f48bb6.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 6a244105f7ef..6b5f9eda91bb 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -566,10 +566,6 @@ class SphinxDependencyChecker: "yaml": "python3-pyyaml", } - # On Tumbleweed, this package is also named rsvg-convert - if not re.search(r"Tumbleweed", self.system_release): - progs["rsvg-convert"] = "rsvg-view" - suse_tex_pkgs = [ "texlive-babel-english", "texlive-caption", -- cgit From e53e6d395fe036bb2c24dc79dc728e9a82ee0b82 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:34 +0200 Subject: scripts: sphinx-pre-install: fix rhel recomendations Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/393a299a850ba9d94c6a8965e78db4da2dbf7e37.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 6b5f9eda91bb..1f64909953b9 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -519,9 +519,9 @@ class SphinxDependencyChecker: "For most, enabling epel and crb are enough:\n" \ "\tsudo dnf install -y epel-release", \ "\tsudo dnf config-manager --set-enabled crb\n" \ - "Yet, some may have other required repositories. Those commands could be useful:" \ - "\tsudo dnf repolist all" \ - "\tsudo repoquery --available --info ", + "Yet, some may have other required repositories. Those commands could be useful:\n" \ + "\tsudo dnf repolist all\n" \ + "\tsudo dnf repoquery --available --info \n", "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want") self.first_hint = False -- cgit From 42e3f9f360e70bea4ee7d4e895f042d86ee11500 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:35 +0200 Subject: scripts: sphinx-pre-install: remove Scientific Linux According with its website: https://scientificlinux.org/ Scientific Linux reached end of life in June 30, 2024. Also, it was based on RHEL 7, which is not compatible with our build system anymore. So, drop support for it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/dde5e0c95017022840f8a522ce44759e51f52aa1.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 1 - 1 file changed, 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 1f64909953b9..cd9c83b17971 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -769,7 +769,6 @@ class SphinxDependencyChecker: re.compile("openEuler"): self.give_redhat_hints, re.compile("Oracle Linux Server"): self.give_redhat_hints, re.compile("Rocky Linux"): self.give_redhat_hints, - re.compile("Scientific Linux"): self.give_redhat_hints, re.compile("Springdale Open Enterprise"): self.give_redhat_hints, re.compile("Ubuntu"): self.give_debian_hints, -- cgit From 1a7da749f18335d7996df3004225ca7418a1052d Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:36 +0200 Subject: scripts: sphinx-pre-install: improve Gentoo package deps logic It took me a lot of time, but I guess understand now what it takes to install a package on Gentoo. Handling dependencies is a nightmare, as Gentoo refuses to emerge some packages if there's no package.use file describing them. To make it worse, compilation flags shall also be present there for some packages. If USE is not perfect, error/warning messages like those are shown: gnome-base/librsvg dev-texlive/texlive-xetex media-fonts/dejavu dev-python/pyyaml ... !!! The following binary packages have been ignored due to non matching USE: =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_13 qt6 svg =media-gfx/graphviz-12.2.1-r1 X pdf python_single_target_python3_12 -python_single_target_python3_13 qt6 svg =media-gfx/graphviz-12.2.1-r1 X pdf qt6 svg =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 qt6 svg =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 python_single_target_python3_12 -python_single_target_python3_13 qt6 svg =media-fonts/noto-cjk-20190416 X =app-text/texlive-core-2024-r1 X cjk -xetex =app-text/texlive-core-2024-r1 X -xetex =app-text/texlive-core-2024-r1 -xetex =dev-libs/zziplib-0.13.79-r1 sdl If emerge is allowed, it will simply ignore the above packages, creating an incomplete installation, which will later fail when one tries to build docs with images or build PDFs. After the fix, command line commands to produce the needed USE chain will be emitted, if they don't exist yet. sudo su -c 'echo "media-gfx/graphviz" > /etc/portage/package.use/graphviz' sudo su -c 'echo "media-gfx/imagemagick" > /etc/portage/package.use/imagemagick' sudo su -c 'echo "media-libs/harfbuzz icu" > /etc/portage/package.use/media-libs' sudo su -c 'echo "media-fonts/noto-cjk" > /etc/portage/package.use/media-fonts' sudo su -c 'echo "app-text/texlive-core xetex" > /etc/portage/package.use/texlive' sudo su -c 'echo "dev-libs/zziplib sdl" > /etc/portage/package.use/zziblib' The new logic tries to be smart enough to detect for missing files and missing arguments. Yet, as Gentoo seems to require users to manage those package.use files by hand, the logic isn't perfect: users may still need to verify for conflicts on different use files. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/365fe5e7d568da932dcffde65f48f2c1256cb773.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 88 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 77 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index cd9c83b17971..94f3d2e32fd6 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -737,22 +737,88 @@ class SphinxDependencyChecker: print("You should run:") print("\n") - - portages = [ - "media-gfx/imagemagick", - "media-gfx/graphviz", - ] + # Handling dependencies is a nightmare, as Gentoo refuses to emerge + # some packages if there's no package.use file describing them. + # To make it worse, compilation flags shall also be present there + # for some packages. If USE is not perfect, error/warning messages + # like those are shown: + # + # !!! The following binary packages have been ignored due to non matching USE: + # + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_13 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf python_single_target_python3_12 -python_single_target_python3_13 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 python_single_target_python3_12 -python_single_target_python3_13 qt6 svg + # =media-fonts/noto-cjk-20190416 X + # =app-text/texlive-core-2024-r1 X cjk -xetex + # =app-text/texlive-core-2024-r1 X -xetex + # =app-text/texlive-core-2024-r1 -xetex + # =dev-libs/zziplib-0.13.79-r1 sdl + # + # And will ignore such packages, installing the remaining ones. That + # affects mostly the image extension and PDF generation. + + # Package dependencies and the minimal needed args: + portages = { + "graphviz": "media-gfx/graphviz", + "imagemagick": "media-gfx/imagemagick", + "media-libs": "media-libs/harfbuzz icu", + "media-fonts": "media-fonts/noto-cjk", + "texlive": "app-text/texlive-core xetex", + "zziblib": "dev-libs/zziplib sdl", + } if self.first_hint: - for p in portages: - result = self.run(["grep", p, "/etc/portage/package.use/*"], - stdout=subprocess.PIPE, text=True) - if not result.stdout.strip(): - print(f"\tsudo emerge -av1 {p}") + use_base = "/etc/portage/package.use" + files = glob(f"{use_base}/*") + + for fname, portage in portages.items(): + install = False + + while install == False: + if not files: + # No files under package.usage. Install all + install = True + break + + args = portage.split(" ") + + name = args.pop(0) + + cmd = ["grep", "-l", "-E", rf"^{name}\b" ] + files + result = self.run(cmd, stdout=subprocess.PIPE, text=True) + if result.returncode or not result.stdout.strip(): + # File containing portage name not found + install = True + break + + # Ensure that needed USE flags are present + if args: + match_fname = result.stdout.strip() + with open(match_fname, 'r', encoding='utf8', + errors='backslashreplace') as fp: + for line in fp: + for arg in args: + if arg.startswith("-"): + continue + + if not re.search(rf"\s*{arg}\b", line): + # Needed file argument not found + install = True + break + + # Everything looks ok, don't install + break + + # emit a code to setup missing USE + if install: + print(f"\tsudo su -c 'echo \"{portage}\" > {use_base}/{fname}'") self.first_hint = False - print(f"\tsudo emerge --ask {self.install}") + # Now, we can use emerge and let it respect USE + print(f"\tsudo emerge --ask --changed-use --binpkg-respect-use=y {self.install}") # # Dispatch the check to an os_specific hinter -- cgit From fb08659be07e646f8103449e02d5e58b32c284d9 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:37 +0200 Subject: scripts: sphinx-pre-install: fix OpenMandriva support OpenMandriva Lx 4.3 has different package names for ImageMagick and yaml. Fix them to ensure that system setup will pass. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0b4e7aa88c96e6a5b8f2e6f381b3e21124680d33.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 94f3d2e32fd6..2f6036eadc94 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -647,6 +647,11 @@ class SphinxDependencyChecker: packager_cmd = "dnf install" noto_sans = "noto-sans-cjk-fonts" tex_pkgs = ["texlive-collection-fontsextra"] + + # Tested on OpenMandriva Lx 4.3 + progs["convert"] = "imagemagick" + progs["yaml"] = "python-pyyaml" + else: packager_cmd = "urpmi" noto_sans = "google-noto-sans-cjk-ttc-fonts" -- cgit From 1ad72e9dfa5ef24ca489b7cbaa5f6f85b1fe87d8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:38 +0200 Subject: scripts: sphinx-pre-install: move package instructions to a new func Currently, if Python < 3.7, package install will fail. That happens with OpenSuse Leap and RHEL-based ver 8 distros. OpenSuse allows installing Sphinx with Python 3.11, but RHEL-based distros don't. Prepare to recomend only venv on such cases. For now, just split the recomendation on a new function that will check for a paramtere to be called. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/4fb2181c960e89774309a833f80209a1a3ab10d2.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 44 +++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 2f6036eadc94..2360ca2ed21c 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -89,6 +89,9 @@ class SphinxDependencyChecker: self.python_cmd = "" self.activate_cmd = "" + # Some distros may not have a Sphinx shipped package compatible with + # our minimal requirements + self.package_supported = True # Certain hints are meant to be shown only once self.first_hint = True @@ -969,6 +972,27 @@ class SphinxDependencyChecker: return self.latest_avail_ver + def recommend_package(self): + + print("\n2) As a package with:") + + old_need = self.need + old_optional = self.optional + self.missing = {} + self.pdf = False + self.optional = 0 + self.install = "" + old_verbose = self.verbose_warn_install + self.verbose_warn_install = 0 + + self.add_package("python-sphinx", 0) + + self.check_distros() + + self.need = old_need + self.optional = old_optional + self.verbose_warn_install = old_verbose + def recommend_sphinx_version(self, virtualenv_cmd): # The logic here is complex, as it have to deal with different versions: # - minimal supported version; @@ -1053,24 +1077,8 @@ class SphinxDependencyChecker: print(f"\tpip install -r {self.requirement_file}") self.deactivate_help() - print("\n2) As a package with:") - - old_need = self.need - old_optional = self.optional - self.missing = {} - self.pdf = False - self.optional = 0 - self.install = "" - old_verbose = self.verbose_warn_install - self.verbose_warn_install = 0 - - self.add_package("python-sphinx", 0) - - self.check_distros() - - self.need = old_need - self.optional = old_optional - self.verbose_warn_install = old_verbose + if self.package_supported: + self.recommend_package() print("\n" \ " Please note that Sphinx >= 3.0 will currently produce false-positive\n" \ -- cgit From 6db1d3977baf7ac895033624897d38fb60cbe3f7 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:39 +0200 Subject: scripts: sphinx-pre-install: adjust a warning message There is one extra space at the first line. Also, as now we only support Python 3.4+, update the text. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/74a17edd70364ca623a54b62bd97a344bb474988.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 2360ca2ed21c..365590f81551 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -1081,8 +1081,8 @@ class SphinxDependencyChecker: self.recommend_package() print("\n" \ - " Please note that Sphinx >= 3.0 will currently produce false-positive\n" \ - " warning when the same name is used for more than one type (functions,\n" \ + " Please note that Sphinx currentlys produce false-positive\n" \ + " warnings when the same name is used for more than one type (functions,\n" \ " structs, enums,...). This is known Sphinx bug. For more details, see:\n" \ "\thttps://github.com/sphinx-doc/sphinx/pull/8313") -- cgit From 8cd256524a920782268a0dd7e3e0404c34bfc65f Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:40 +0200 Subject: scripts: sphinx-pre-install: better handle Python min version Don't do any recommendations about Sphinx install with too old python versions. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/013aeb848ecc3f6b69b4518cf3d335bd2353b6e1.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 365590f81551..a5c777e529ec 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -92,6 +92,10 @@ class SphinxDependencyChecker: # Some distros may not have a Sphinx shipped package compatible with # our minimal requirements self.package_supported = True + + # Recommend a new python version + self.recommend_python = None + # Certain hints are meant to be shown only once self.first_hint = True @@ -511,11 +515,11 @@ class SphinxDependencyChecker: print("ERROR: Distro not supported. Too old?") return - # TODO: check if RHEL8 still works. - # On my tests with docker "redhat/ubi8" image, there's no - # python3-sphinx (or similar) package. It comes with Python 3.6, - # but there are other python packages over there, so it may be - # possible to work with venv. + # RHEL 8 uses Python 3.6, which is not compatible with + # the build system anymore. Suggest Python 3.11 + if rel == 8: + self.add_package("python39", 0) + self.recommend_python = True if self.first_hint: print("Note: RHEL-based distros typically require extra repositories.\n" \ @@ -596,6 +600,7 @@ class SphinxDependencyChecker: # the build system anymore. Suggest Python 3.11 if rel == 15: if not self.which(self.python_cmd): + self.recommend_python = True self.add_package(self.python_cmd, 0) progs.update({ @@ -1000,6 +1005,11 @@ class SphinxDependencyChecker: # - recommended version. # It also needs to work fine with both distro's package and venv/virtualenv + if self.recommend_python: + print("\nPython version is incompatible with doc build.\n" \ + "Please upgrade it and re-run.\n") + return + # Version is OK. Nothing to do. if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: return -- cgit From 9ecda2e10101537e97bcb2b05049e58306ac4e1b Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:41 +0200 Subject: scripts: sphinx-pre-install: convert is_optional to a class When is_optional was added in Perl, it was a boolean. With time, it ended becoming a sort of enum, which makes the module harder to maintain. Convert it to a enum-like class and add more options to it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/42290a24f3b1dbea9ebe19747cf5622bb2f2cf5c.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 192 ++++++++++++++++++++++++++++-------------- 1 file changed, 130 insertions(+), 62 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index a5c777e529ec..0963da21c27b 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -32,6 +32,53 @@ RECOMMENDED_VERSION = parse_version("3.4.3") MIN_PYTHON_VERSION = parse_version("3.7") +class DepType: + + # Internal types of dependencies. + _SYS_TYPE = 0 + _PHY_TYPE = 1 + _PDF_TYPE = 2 + + # Let's define keys as a tuple with the type and mandatory/optional. + # This way, checking for optional or type is easy. + + SYSTEM_MANDATORY = (_SYS_TYPE, True) + PYTHON_MANDATORY = (_PHY_TYPE, True) + PDF_MANDATORY = (_PDF_TYPE, True) + + # Currently we're not using all optional types, but let's keep all + # combinations here, as we may end needing them in the future. Also, + # it allows a name() function that handles all possibilities. + SYSTEM_OPTIONAL = (_SYS_TYPE, False) + PYTHON_OPTIONAL = (_PHY_TYPE, False) + PDF_OPTIONAL = (_PDF_TYPE, True) + + @staticmethod + def name(dtype): + if dtype[0] == DepType._SYS_TYPE: + msg = "build" + elif dtype[0] == DepType._PHY_TYPE: + msg = "Python" + else: + msg = "PDF" + + if dtype[1]: + return f"ERROR: {msg} mandatory deps missing" + else: + out = f"Warning: {msg} optional deps missing" + + @staticmethod + def is_optional(dtype): + return not dtype[1] + + @staticmethod + def is_pdf(dtype): + if (dtype[0] == DepType._PDF_TYPE): + return True + + return False + + class SphinxDependencyChecker: # List of required texlive packages on Fedora and OpenSuse texlive = { @@ -223,56 +270,68 @@ class SphinxDependencyChecker: # Methods to check if a feature exists # - # Note: is_optional has 3 states: - # - 0: mandatory - # - 1: optional, but nice to have - # - 2: LaTeX optional - pdf builds without it, but may have visual impact - def check_missing(self, progs): - for prog, is_optional in sorted(self.missing.items()): + run = {} + + for prog, dtype in sorted(self.missing.items()): # At least on some LTS distros like CentOS 7, texlive doesn't # provide all packages we need. When such distros are # detected, we have to disable PDF output. # # So, we need to ignore the packages that distros would # need for LaTeX to work - if is_optional == 2 and not self.pdf: + if DepType.is_pdf(dtype) and not self.pdf: self.optional -= 1 continue + if not dtype in run: + run[dtype] = [] + + run[dtype].append(prog) + + output_msg = "" + + for dtype in sorted(run.keys()): + progs = " ".join(run[dtype]) + if self.verbose_warn_install: - if is_optional: - print(f'Warning: better to also install "{prog}".') - else: - print(f'ERROR: please install "{prog}", otherwise, build won\'t work.') + try: + name = DepType.name(dtype) + output_msg += f'{name}:\t{progs}\n' + except KeyError: + raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") - self.install += " " + progs.get(prog, prog) + self.install += " " + progs + + if output_msg: + print(f"\n{output_msg}\n") self.install = self.install.lstrip() - def add_package(self, package, is_optional): - self.missing[package] = is_optional + def add_package(self, package, dtype): + is_optional = DepType.is_optional(dtype) + self.missing[package] = dtype if is_optional: self.optional += 1 else: self.need += 1 - def check_missing_file(self, files, package, is_optional): + def check_missing_file(self, files, package, dtype): for f in files: if os.path.exists(f): return - self.add_package(package, is_optional) + self.add_package(package, dtype) - def check_program(self, prog, is_optional): + def check_program(self, prog, dtype): found = self.which(prog) if found: return found - self.add_package(prog, is_optional) + self.add_package(prog, dtype) return None - def check_perl_module(self, prog, is_optional): + def check_perl_module(self, prog, dtype): # While testing with lxc download template, one of the # distros (Oracle) didn't have perl - nor even an option to install # before installing oraclelinux-release-el9 package. @@ -281,46 +340,52 @@ class SphinxDependencyChecker: # add it as a mandatory package, as some parts of the doc builder # needs it. if not self.which("perl"): - self.add_package("perl", 0) - self.add_package(prog, is_optional) + self.add_package("perl", DepType.SYSTEM_MANDATORY) + self.add_package(prog, dtype) return try: self.run(["perl", f"-M{prog}", "-e", "1"], check=True) except subprocess.CalledProcessError: - self.add_package(prog, is_optional) + self.add_package(prog, dtype) - def check_python_module(self, module, is_optional): - # FIXME: is it needed at the Python version? Maybe due to venv? - if not self.python_cmd: - return + def check_python_module(self, module, is_optional=False): + if is_optional: + dtype = DepType.PYTHON_OPTIONAL + else: + dtype = DepType.PYTHON_MANDATORY try: self.run([self.python_cmd, "-c", f"import {module}"], check=True) except subprocess.CalledProcessError: - self.add_package(module, is_optional) + self.add_package(module, dtype) - def check_rpm_missing(self, pkgs, is_optional): + def check_rpm_missing(self, pkgs, dtype): for prog in pkgs: try: self.run(["rpm", "-q", prog], check=True) except subprocess.CalledProcessError: - self.add_package(prog, is_optional) + self.add_package(prog, dtype) - def check_pacman_missing(self, pkgs, is_optional): + def check_pacman_missing(self, pkgs, dtype): for prog in pkgs: try: self.run(["pacman", "-Q", prog], check=True) except subprocess.CalledProcessError: - self.add_package(prog, is_optional) + self.add_package(prog, dtype) + + def check_missing_tex(self, is_optional=False): + if is_optional: + dtype = DepType.PDF_OPTIONAL + else: + dtype = DepType.PDF_MANDATORY - def check_missing_tex(self, is_optional): kpsewhich = self.which("kpsewhich") for prog, package in self.texlive.items(): # If kpsewhich is not there, just add it to deps if not kpsewhich: - self.add_package(package, is_optional) + self.add_package(package, dtype) continue # Check if the package is needed @@ -331,11 +396,11 @@ class SphinxDependencyChecker: # Didn't find. Add it if not result.stdout.strip(): - self.add_package(package, is_optional) + self.add_package(package, dtype) except subprocess.CalledProcessError: # kpsewhich returned an error. Add it, just in case - self.add_package(package, is_optional) + self.add_package(package, dtype) def get_sphinx_fname(self): if "SPHINXBUILD" in os.environ: @@ -446,9 +511,9 @@ class SphinxDependencyChecker: } for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, 2) + self.check_missing_file(files, package, DepType.PDF_MANDATORY) - self.check_program("dvipng", 2) + self.check_program("dvipng", DepType.PDF_MANDATORY) self.check_missing(progs) @@ -518,7 +583,7 @@ class SphinxDependencyChecker: # RHEL 8 uses Python 3.6, which is not compatible with # the build system anymore. Suggest Python 3.11 if rel == 8: - self.add_package("python39", 0) + self.add_package("python39", DepType.SYSTEM_MANDATORY) self.recommend_python = True if self.first_hint: @@ -540,13 +605,13 @@ class SphinxDependencyChecker: "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc", ] - self.check_missing_file(pdf_pkgs, noto_sans_redhat, 2) + self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepType.PDF_MANDATORY) if not old: - self.check_rpm_missing(fedora26_opt_pkgs, 2) - self.check_rpm_missing(fedora_tex_pkgs, 2) + self.check_rpm_missing(fedora26_opt_pkgs, DepType.PDF_MANDATORY) + self.check_rpm_missing(fedora_tex_pkgs, DepType.PDF_MANDATORY) - self.check_missing_tex(2) + self.check_missing_tex() self.check_missing(progs) @@ -601,7 +666,7 @@ class SphinxDependencyChecker: if rel == 15: if not self.which(self.python_cmd): self.recommend_python = True - self.add_package(self.python_cmd, 0) + self.add_package(self.python_cmd, DepType.SYSTEM_MANDATORY) progs.update({ "python-sphinx": "python311-Sphinx", @@ -623,9 +688,9 @@ class SphinxDependencyChecker: # "Noto Sans CJK SC" on openSUSE if self.pdf: - self.check_rpm_missing(suse_tex_pkgs, 2) + self.check_rpm_missing(suse_tex_pkgs, DepType.PDF_MANDATORY) if self.pdf: - self.check_missing_tex(2) + self.check_missing_tex() self.check_missing(progs) if not self.need and not self.optional: @@ -672,8 +737,8 @@ class SphinxDependencyChecker: "/usr/share/fonts/TTF/NotoSans-Regular.ttf", ] - self.check_missing_file(pdf_pkgs, noto_sans, 2) - self.check_rpm_missing(tex_pkgs, 2) + self.check_missing_file(pdf_pkgs, noto_sans, DepType.PDF_MANDATORY) + self.check_rpm_missing(tex_pkgs, DepType.PDF_MANDATORY) self.check_missing(progs) @@ -701,7 +766,7 @@ class SphinxDependencyChecker: ] if self.pdf: - self.check_pacman_missing(archlinux_tex_pkgs, 2) + self.check_pacman_missing(archlinux_tex_pkgs, DepType.PDF_MANDATORY) self.check_missing_file( ["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], @@ -739,7 +804,7 @@ class SphinxDependencyChecker: ], } for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, 2) + self.check_missing_file(files, package, DepType.PDF_MANDATORY) self.check_missing(progs) @@ -878,7 +943,7 @@ class SphinxDependencyChecker: # progs = {"sphinx-build": "sphinx"} if self.pdf: - self.check_missing_tex(2) + self.check_missing_tex() self.check_missing(progs) @@ -990,7 +1055,7 @@ class SphinxDependencyChecker: old_verbose = self.verbose_warn_install self.verbose_warn_install = 0 - self.add_package("python-sphinx", 0) + self.add_package("python-sphinx", DepType.PYTHON_MANDATORY) self.check_distros() @@ -1010,6 +1075,7 @@ class SphinxDependencyChecker: "Please upgrade it and re-run.\n") return + # Version is OK. Nothing to do. if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: return @@ -1128,21 +1194,23 @@ class SphinxDependencyChecker: else: virtualenv_cmd = f"{self.python_cmd} -m venv" - self.check_python_module("ensurepip", 0) + self.check_python_module("ensurepip") # Check for needed programs/tools - self.check_perl_module("Pod::Usage", 0) - self.check_python_module("yaml", 0) - self.check_program("make", 0) - self.check_program("gcc", 0) - self.check_program("dot", 1) - self.check_program("convert", 1) + self.check_perl_module("Pod::Usage", DepType.SYSTEM_MANDATORY) + + self.check_program("make", DepType.SYSTEM_MANDATORY) + self.check_program("gcc", DepType.SYSTEM_MANDATORY) + + self.check_program("dot", DepType.SYSTEM_OPTIONAL) + self.check_program("convert", DepType.SYSTEM_OPTIONAL) + + self.check_python_module("yaml") if self.pdf: - # Extra PDF files - should use 2 for LaTeX is_optional - self.check_program("xelatex", 2) - self.check_program("rsvg-convert", 2) - self.check_program("latexmk", 2) + self.check_program("xelatex", DepType.PDF_MANDATORY) + self.check_program("rsvg-convert", DepType.PDF_MANDATORY) + self.check_program("latexmk", DepType.PDF_MANDATORY) # Do distro-specific checks and output distro-install commands self.check_distros() -- cgit From 272f5e0390dd9b5f62b480eae482a67d2493630e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:42 +0200 Subject: scripts: sphinx-pre-install: better handle RHEL-based distros Better implement support for RHEL-based distros. While here, get rid of a Fedora 28 support which cause troubles with server distros. Also, get rid of yum, as RHEL8 already suppords dnf, and this is not the minimal version we may still support. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/4d1b27d3a381f011e150bb50176babba83af9e1a.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 0963da21c27b..592223fa686f 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -536,10 +536,6 @@ class SphinxDependencyChecker: "yaml": "python3-pyyaml", } - fedora26_opt_pkgs = [ - "graphviz-gd", # Fedora 26: needed for PDF support - ] - fedora_tex_pkgs = [ "dejavu-sans-fonts", "dejavu-sans-mono-fonts", @@ -549,9 +545,8 @@ class SphinxDependencyChecker: "texlive-xecjk", ] - old = 0 + fedora = False rel = None - pkg_manager = "dnf" match = re.search(r"(release|Linux)\s+(\d+)", self.system_release) if match: @@ -559,12 +554,12 @@ class SphinxDependencyChecker: if not rel: print("Couldn't identify release number") - old = 1 self.pdf = False elif re.search("Fedora", self.system_release): # Fedora 38 and upper use this CJK font noto_sans_redhat = "google-noto-sans-cjk-fonts" + fedora = True else: # Almalinux, CentOS, RHEL, ... @@ -574,9 +569,6 @@ class SphinxDependencyChecker: progs["virtualenv"] = "python-virtualenv" if not rel or rel < 8: - old = 1 - self.pdf = False - print("ERROR: Distro not supported. Too old?") return @@ -607,11 +599,16 @@ class SphinxDependencyChecker: self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepType.PDF_MANDATORY) - if not old: - self.check_rpm_missing(fedora26_opt_pkgs, DepType.PDF_MANDATORY) - self.check_rpm_missing(fedora_tex_pkgs, DepType.PDF_MANDATORY) + self.check_rpm_missing(fedora_tex_pkgs, DepType.PDF_MANDATORY) + + self.check_missing_tex(DepType.PDF_MANDATORY) + + # There's no texlive-ctex on RHEL 8 repositories. This will + # likely affect CJK pdf build only. + if not fedora and rel == 8: + if "texlive-ctex" in self.missing: + del self.missing["texlive-ctex"] - self.check_missing_tex() self.check_missing(progs) @@ -621,11 +618,7 @@ class SphinxDependencyChecker: if self.verbose_warn_install: print("You should run:") - if old: - # dnf is there since Fedora 18+ and RHEL 8 - pkg_manager = "yum" - - print(f"\n\tsudo {pkg_manager} install -y {self.install}") + print(f"\n\tsudo dnf install -y {self.install}") def give_opensuse_hints(self): progs = { -- cgit From 2cab00fb178a31bca45adc1ff5f0994679d116a4 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:43 +0200 Subject: scripts: sphinx-pre-install: move missing logic to a separate class Better manage dependencies by placing them on a distro-independent class. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/f4f5bf276e07dc494f5dc83c4c2d087be7f790e6.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 251 +++++++++++++++++++++++------------------- 1 file changed, 138 insertions(+), 113 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 592223fa686f..47dce1fcddfb 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -32,7 +32,7 @@ RECOMMENDED_VERSION = parse_version("3.4.3") MIN_PYTHON_VERSION = parse_version("3.7") -class DepType: +class DepManager: # Internal types of dependencies. _SYS_TYPE = 0 @@ -53,11 +53,18 @@ class DepType: PYTHON_OPTIONAL = (_PHY_TYPE, False) PDF_OPTIONAL = (_PDF_TYPE, True) + def __init__(self, pdf): + self.missing = {} + self.run = {} + self.need = 0 + self.optional = 0 + self.pdf = pdf + @staticmethod def name(dtype): - if dtype[0] == DepType._SYS_TYPE: + if dtype[0] == DepManager._SYS_TYPE: msg = "build" - elif dtype[0] == DepType._PHY_TYPE: + elif dtype[0] == DepManager._PHY_TYPE: msg = "Python" else: msg = "PDF" @@ -73,11 +80,75 @@ class DepType: @staticmethod def is_pdf(dtype): - if (dtype[0] == DepType._PDF_TYPE): + if (dtype[0] == DepManager._PDF_TYPE): return True return False + def add_package(self, package, dtype): + is_optional = DepManager.is_optional(dtype) + self.missing[package] = dtype + if is_optional: + self.optional += 1 + else: + self.need += 1 + + def del_package(self, package): + if package in self.missing: + del self.missing[package] + + def clear_deps(self): + """ + Clear dependencies without changing needed/optional. + + This is an ackward way to have a separate section to recommend + a package after system main dependencies. + + TODO: rework the logic to prevent needing it + """ + + self.missing = {} + + def check_missing(self, progs): + self.run = {} + + for prog, dtype in sorted(self.missing.items()): + # At least on some LTS distros like CentOS 7, texlive doesn't + # provide all packages we need. When such distros are + # detected, we have to disable PDF output. + # + # So, we need to ignore the packages that distros would + # need for LaTeX to work + if DepManager.is_pdf(dtype) and not self.pdf: + self.optional -= 1 + continue + + if not dtype in self.run: + self.run[dtype] = [] + + self.run[dtype].append(progs.get(prog, prog)) + + install = [] + for dtype in self.run.keys(): + install += self.run[dtype] + + return " ".join(sorted(set(install))) + + def warn_install(self): + + output_msg = "" + + for dtype in sorted(self.run.keys()): + progs = " ".join(sorted(set(self.run[dtype]))) + + try: + name = DepManager.name(dtype) + output_msg += f'{name}:\t{progs}\n' + except KeyError: + raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") + + if output_msg: + print(f"\n{output_msg}\n") class SphinxDependencyChecker: # List of required texlive packages on Fedora and OpenSuse @@ -120,10 +191,8 @@ class SphinxDependencyChecker: self.virtualenv = args.virtualenv self.version_check = args.version_check - self.missing = {} + self.deps = DepManager(self.pdf) - self.need = 0 - self.optional = 0 self.need_symlink = 0 self.need_sphinx = 0 self.need_pip = 0 @@ -270,64 +339,18 @@ class SphinxDependencyChecker: # Methods to check if a feature exists # - def check_missing(self, progs): - run = {} - - for prog, dtype in sorted(self.missing.items()): - # At least on some LTS distros like CentOS 7, texlive doesn't - # provide all packages we need. When such distros are - # detected, we have to disable PDF output. - # - # So, we need to ignore the packages that distros would - # need for LaTeX to work - if DepType.is_pdf(dtype) and not self.pdf: - self.optional -= 1 - continue - - if not dtype in run: - run[dtype] = [] - - run[dtype].append(prog) - - output_msg = "" - - for dtype in sorted(run.keys()): - progs = " ".join(run[dtype]) - - if self.verbose_warn_install: - try: - name = DepType.name(dtype) - output_msg += f'{name}:\t{progs}\n' - except KeyError: - raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") - - self.install += " " + progs - - if output_msg: - print(f"\n{output_msg}\n") - - self.install = self.install.lstrip() - - def add_package(self, package, dtype): - is_optional = DepType.is_optional(dtype) - self.missing[package] = dtype - if is_optional: - self.optional += 1 - else: - self.need += 1 - def check_missing_file(self, files, package, dtype): for f in files: if os.path.exists(f): return - self.add_package(package, dtype) + self.deps.add_package(package, dtype) def check_program(self, prog, dtype): found = self.which(prog) if found: return found - self.add_package(prog, dtype) + self.deps.add_package(prog, dtype) return None @@ -340,52 +363,52 @@ class SphinxDependencyChecker: # add it as a mandatory package, as some parts of the doc builder # needs it. if not self.which("perl"): - self.add_package("perl", DepType.SYSTEM_MANDATORY) - self.add_package(prog, dtype) + self.deps.add_package("perl", DepManager.SYSTEM_MANDATORY) + self.deps.add_package(prog, dtype) return try: self.run(["perl", f"-M{prog}", "-e", "1"], check=True) except subprocess.CalledProcessError: - self.add_package(prog, dtype) + self.deps.add_package(prog, dtype) def check_python_module(self, module, is_optional=False): if is_optional: - dtype = DepType.PYTHON_OPTIONAL + dtype = DepManager.PYTHON_OPTIONAL else: - dtype = DepType.PYTHON_MANDATORY + dtype = DepManager.PYTHON_MANDATORY try: self.run([self.python_cmd, "-c", f"import {module}"], check=True) except subprocess.CalledProcessError: - self.add_package(module, dtype) + self.deps.add_package(module, dtype) def check_rpm_missing(self, pkgs, dtype): for prog in pkgs: try: self.run(["rpm", "-q", prog], check=True) except subprocess.CalledProcessError: - self.add_package(prog, dtype) + self.deps.add_package(prog, dtype) def check_pacman_missing(self, pkgs, dtype): for prog in pkgs: try: self.run(["pacman", "-Q", prog], check=True) except subprocess.CalledProcessError: - self.add_package(prog, dtype) + self.deps.add_package(prog, dtype) def check_missing_tex(self, is_optional=False): if is_optional: - dtype = DepType.PDF_OPTIONAL + dtype = DepManager.PDF_OPTIONAL else: - dtype = DepType.PDF_MANDATORY + dtype = DepManager.PDF_MANDATORY kpsewhich = self.which("kpsewhich") for prog, package in self.texlive.items(): # If kpsewhich is not there, just add it to deps if not kpsewhich: - self.add_package(package, dtype) + self.deps.add_package(package, dtype) continue # Check if the package is needed @@ -396,11 +419,11 @@ class SphinxDependencyChecker: # Didn't find. Add it if not result.stdout.strip(): - self.add_package(package, dtype) + self.deps.add_package(package, dtype) except subprocess.CalledProcessError: # kpsewhich returned an error. Add it, just in case - self.add_package(package, dtype) + self.deps.add_package(package, dtype) def get_sphinx_fname(self): if "SPHINXBUILD" in os.environ: @@ -478,6 +501,17 @@ class SphinxDependencyChecker: return f.read().strip() return "" + def check_missing(self, progs): + self.install += self.deps.check_missing(progs) + if self.verbose_warn_install: + self.deps.warn_install() + + if not self.deps.need and not self.deps.optional: + return False + + return True + + # # Distro-specific hints methods # @@ -511,13 +545,11 @@ class SphinxDependencyChecker: } for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, DepType.PDF_MANDATORY) - - self.check_program("dvipng", DepType.PDF_MANDATORY) + self.check_missing_file(files, package, DepManager.PDF_MANDATORY) - self.check_missing(progs) + self.check_program("dvipng", DepManager.PDF_MANDATORY) - if not self.need and not self.optional: + if self.check_missing(progs): return if self.verbose_warn_install: @@ -575,7 +607,7 @@ class SphinxDependencyChecker: # RHEL 8 uses Python 3.6, which is not compatible with # the build system anymore. Suggest Python 3.11 if rel == 8: - self.add_package("python39", DepType.SYSTEM_MANDATORY) + self.deps.add_package("python39", DepManager.SYSTEM_MANDATORY) self.recommend_python = True if self.first_hint: @@ -597,22 +629,18 @@ class SphinxDependencyChecker: "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc", ] - self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepType.PDF_MANDATORY) + self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepManager.PDF_MANDATORY) - self.check_rpm_missing(fedora_tex_pkgs, DepType.PDF_MANDATORY) + self.check_rpm_missing(fedora_tex_pkgs, DepManager.PDF_MANDATORY) - self.check_missing_tex(DepType.PDF_MANDATORY) + self.check_missing_tex(DepManager.PDF_MANDATORY) # There's no texlive-ctex on RHEL 8 repositories. This will # likely affect CJK pdf build only. if not fedora and rel == 8: - if "texlive-ctex" in self.missing: - del self.missing["texlive-ctex"] - - - self.check_missing(progs) + self.deps.del_package("texlive-ctex") - if not self.need and not self.optional: + if self.check_missing(progs): return if self.verbose_warn_install: @@ -659,7 +687,7 @@ class SphinxDependencyChecker: if rel == 15: if not self.which(self.python_cmd): self.recommend_python = True - self.add_package(self.python_cmd, DepType.SYSTEM_MANDATORY) + self.deps.add_package(self.python_cmd, DepManager.SYSTEM_MANDATORY) progs.update({ "python-sphinx": "python311-Sphinx", @@ -681,12 +709,11 @@ class SphinxDependencyChecker: # "Noto Sans CJK SC" on openSUSE if self.pdf: - self.check_rpm_missing(suse_tex_pkgs, DepType.PDF_MANDATORY) + self.check_rpm_missing(suse_tex_pkgs, DepManager.PDF_MANDATORY) if self.pdf: self.check_missing_tex() - self.check_missing(progs) - if not self.need and not self.optional: + if self.check_missing(progs): return if self.verbose_warn_install: @@ -730,13 +757,12 @@ class SphinxDependencyChecker: "/usr/share/fonts/TTF/NotoSans-Regular.ttf", ] - self.check_missing_file(pdf_pkgs, noto_sans, DepType.PDF_MANDATORY) - self.check_rpm_missing(tex_pkgs, DepType.PDF_MANDATORY) - - self.check_missing(progs) + self.check_missing_file(pdf_pkgs, noto_sans, DepManager.PDF_MANDATORY) + self.check_rpm_missing(tex_pkgs, DepManager.PDF_MANDATORY) - if not self.need and not self.optional: + if self.check_missing(progs): return + if self.verbose_warn_install: print("You should run:") print(f"\n\tsudo {packager_cmd} {self.install}") @@ -759,7 +785,7 @@ class SphinxDependencyChecker: ] if self.pdf: - self.check_pacman_missing(archlinux_tex_pkgs, DepType.PDF_MANDATORY) + self.check_pacman_missing(archlinux_tex_pkgs, DepManager.PDF_MANDATORY) self.check_missing_file( ["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], @@ -767,10 +793,9 @@ class SphinxDependencyChecker: 2, ) - self.check_missing(progs) - - if not self.need and not self.optional: + if self.check_missing(progs): return + if self.verbose_warn_install: print("You should run:") print(f"\n\tsudo pacman -S {self.install}") @@ -797,11 +822,9 @@ class SphinxDependencyChecker: ], } for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, DepType.PDF_MANDATORY) - - self.check_missing(progs) + self.check_missing_file(files, package, DepManager.PDF_MANDATORY) - if not self.need and not self.optional: + if self.check_missing(progs): return if self.verbose_warn_install: @@ -1039,16 +1062,18 @@ class SphinxDependencyChecker: print("\n2) As a package with:") - old_need = self.need - old_optional = self.optional - self.missing = {} + old_need = self.deps.need + old_optional = self.deps.optional + self.pdf = False self.optional = 0 self.install = "" old_verbose = self.verbose_warn_install self.verbose_warn_install = 0 - self.add_package("python-sphinx", DepType.PYTHON_MANDATORY) + self.deps.clear_deps() + + self.deps.add_package("python-sphinx", DepManager.PYTHON_MANDATORY) self.check_distros() @@ -1111,7 +1136,7 @@ class SphinxDependencyChecker: print("\n") else: if self.need_sphinx: - self.need += 1 + self.deps.need += 1 # Suggest newer versions if current ones are too old if self.latest_avail_ver and self.latest_avail_ver >= self.min_version: @@ -1190,20 +1215,20 @@ class SphinxDependencyChecker: self.check_python_module("ensurepip") # Check for needed programs/tools - self.check_perl_module("Pod::Usage", DepType.SYSTEM_MANDATORY) + self.check_perl_module("Pod::Usage", DepManager.SYSTEM_MANDATORY) - self.check_program("make", DepType.SYSTEM_MANDATORY) - self.check_program("gcc", DepType.SYSTEM_MANDATORY) + self.check_program("make", DepManager.SYSTEM_MANDATORY) + self.check_program("gcc", DepManager.SYSTEM_MANDATORY) - self.check_program("dot", DepType.SYSTEM_OPTIONAL) - self.check_program("convert", DepType.SYSTEM_OPTIONAL) + self.check_program("dot", DepManager.SYSTEM_OPTIONAL) + self.check_program("convert", DepManager.SYSTEM_OPTIONAL) self.check_python_module("yaml") if self.pdf: - self.check_program("xelatex", DepType.PDF_MANDATORY) - self.check_program("rsvg-convert", DepType.PDF_MANDATORY) - self.check_program("latexmk", DepType.PDF_MANDATORY) + self.check_program("xelatex", DepManager.PDF_MANDATORY) + self.check_program("rsvg-convert", DepManager.PDF_MANDATORY) + self.check_program("latexmk", DepManager.PDF_MANDATORY) # Do distro-specific checks and output distro-install commands self.check_distros() -- cgit From 1e9ba3b6d4cef8b1b87d6226ab7026c9745cc1d7 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:44 +0200 Subject: scripts: sphinx-pre-install: move ancillary checkers to a separate class The code there are just a bunch of static functions that are used by the main class. group them altogether to better organize the code. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/e2671eb14fae7a8510f5305ac44ad8063e237a5f.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 160 +++++++++++++++++++++--------------------- 1 file changed, 81 insertions(+), 79 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 47dce1fcddfb..b00e50028f4d 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -150,85 +150,11 @@ class DepManager: if output_msg: print(f"\n{output_msg}\n") -class SphinxDependencyChecker: - # List of required texlive packages on Fedora and OpenSuse - texlive = { - "amsfonts.sty": "texlive-amsfonts", - "amsmath.sty": "texlive-amsmath", - "amssymb.sty": "texlive-amsfonts", - "amsthm.sty": "texlive-amscls", - "anyfontsize.sty": "texlive-anyfontsize", - "atbegshi.sty": "texlive-oberdiek", - "bm.sty": "texlive-tools", - "capt-of.sty": "texlive-capt-of", - "cmap.sty": "texlive-cmap", - "ctexhook.sty": "texlive-ctex", - "ecrm1000.tfm": "texlive-ec", - "eqparbox.sty": "texlive-eqparbox", - "eu1enc.def": "texlive-euenc", - "fancybox.sty": "texlive-fancybox", - "fancyvrb.sty": "texlive-fancyvrb", - "float.sty": "texlive-float", - "fncychap.sty": "texlive-fncychap", - "footnote.sty": "texlive-mdwtools", - "framed.sty": "texlive-framed", - "luatex85.sty": "texlive-luatex85", - "multirow.sty": "texlive-multirow", - "needspace.sty": "texlive-needspace", - "palatino.sty": "texlive-psnfss", - "parskip.sty": "texlive-parskip", - "polyglossia.sty": "texlive-polyglossia", - "tabulary.sty": "texlive-tabulary", - "threeparttable.sty": "texlive-threeparttable", - "titlesec.sty": "texlive-titlesec", - "ucs.sty": "texlive-ucs", - "upquote.sty": "texlive-upquote", - "wrapfig.sty": "texlive-wrapfig", - } - - def __init__(self, args): - self.pdf = args.pdf - self.virtualenv = args.virtualenv - self.version_check = args.version_check - - self.deps = DepManager(self.pdf) - - self.need_symlink = 0 - self.need_sphinx = 0 - self.need_pip = 0 - self.rec_sphinx_upgrade = 0 - self.verbose_warn_install = 1 - - self.system_release = "" - self.install = "" - self.virtenv_dir = "" - self.python_cmd = "" - self.activate_cmd = "" - - # Some distros may not have a Sphinx shipped package compatible with - # our minimal requirements - self.package_supported = True - - # Recommend a new python version - self.recommend_python = None - - # Certain hints are meant to be shown only once - self.first_hint = True - - self.min_version = (0, 0, 0) - self.cur_version = (0, 0, 0) - self.latest_avail_ver = (0, 0, 0) - self.venv_ver = (0, 0, 0) - - prefix = os.environ.get("srctree", ".") + "/" - - self.conf = prefix + "Documentation/conf.py" - self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" - self.virtenv_prefix = ["sphinx_", "Sphinx_" ] - - # - # Ancillary methods that don't depend on self - # +class AncillaryCheckers: + """ + Ancillary methods that checks for missing dependencies for different + types of types, like binaries, python modules, rpm deps, etc. + """ @staticmethod def which(prog): @@ -335,6 +261,82 @@ class SphinxDependencyChecker: return subprocess.run(*args, **kwargs) +class SphinxDependencyChecker(AncillaryCheckers): + # List of required texlive packages on Fedora and OpenSuse + texlive = { + "amsfonts.sty": "texlive-amsfonts", + "amsmath.sty": "texlive-amsmath", + "amssymb.sty": "texlive-amsfonts", + "amsthm.sty": "texlive-amscls", + "anyfontsize.sty": "texlive-anyfontsize", + "atbegshi.sty": "texlive-oberdiek", + "bm.sty": "texlive-tools", + "capt-of.sty": "texlive-capt-of", + "cmap.sty": "texlive-cmap", + "ctexhook.sty": "texlive-ctex", + "ecrm1000.tfm": "texlive-ec", + "eqparbox.sty": "texlive-eqparbox", + "eu1enc.def": "texlive-euenc", + "fancybox.sty": "texlive-fancybox", + "fancyvrb.sty": "texlive-fancyvrb", + "float.sty": "texlive-float", + "fncychap.sty": "texlive-fncychap", + "footnote.sty": "texlive-mdwtools", + "framed.sty": "texlive-framed", + "luatex85.sty": "texlive-luatex85", + "multirow.sty": "texlive-multirow", + "needspace.sty": "texlive-needspace", + "palatino.sty": "texlive-psnfss", + "parskip.sty": "texlive-parskip", + "polyglossia.sty": "texlive-polyglossia", + "tabulary.sty": "texlive-tabulary", + "threeparttable.sty": "texlive-threeparttable", + "titlesec.sty": "texlive-titlesec", + "ucs.sty": "texlive-ucs", + "upquote.sty": "texlive-upquote", + "wrapfig.sty": "texlive-wrapfig", + } + + def __init__(self, args): + self.pdf = args.pdf + self.virtualenv = args.virtualenv + self.version_check = args.version_check + + self.deps = DepManager(self.pdf) + + self.need_symlink = 0 + self.need_sphinx = 0 + self.need_pip = 0 + self.rec_sphinx_upgrade = 0 + self.verbose_warn_install = 1 + + self.system_release = "" + self.install = "" + self.virtenv_dir = "" + self.python_cmd = "" + self.activate_cmd = "" + + # Some distros may not have a Sphinx shipped package compatible with + # our minimal requirements + self.package_supported = True + + # Recommend a new python version + self.recommend_python = None + + # Certain hints are meant to be shown only once + self.first_hint = True + + self.min_version = (0, 0, 0) + self.cur_version = (0, 0, 0) + self.latest_avail_ver = (0, 0, 0) + self.venv_ver = (0, 0, 0) + + prefix = os.environ.get("srctree", ".") + "/" + + self.conf = prefix + "Documentation/conf.py" + self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" + self.virtenv_prefix = ["sphinx_", "Sphinx_" ] + # # Methods to check if a feature exists # -- cgit From 9bb5f0dc18d037635f4c5075747de1e47493b538 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:45 +0200 Subject: scripts: sphinx-pre-install: add more generic checkers on a class Better organize the code by moving the more generic methods to MissingCheckers. Such class contain only binary and package dependent missing checkers, but no distro-specific data or code. All distro-specific data/code remains at SphinxDependencyChecker class. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/11a252fe816bd7c85583d26ade0666eb2b481bf0.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 142 ++++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 66 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index b00e50028f4d..9127487bd4d7 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -150,7 +150,7 @@ class DepManager: if output_msg: print(f"\n{output_msg}\n") -class AncillaryCheckers: +class AncillaryMethods: """ Ancillary methods that checks for missing dependencies for different types of types, like binaries, python modules, rpm deps, etc. @@ -261,81 +261,24 @@ class AncillaryCheckers: return subprocess.run(*args, **kwargs) -class SphinxDependencyChecker(AncillaryCheckers): - # List of required texlive packages on Fedora and OpenSuse - texlive = { - "amsfonts.sty": "texlive-amsfonts", - "amsmath.sty": "texlive-amsmath", - "amssymb.sty": "texlive-amsfonts", - "amsthm.sty": "texlive-amscls", - "anyfontsize.sty": "texlive-anyfontsize", - "atbegshi.sty": "texlive-oberdiek", - "bm.sty": "texlive-tools", - "capt-of.sty": "texlive-capt-of", - "cmap.sty": "texlive-cmap", - "ctexhook.sty": "texlive-ctex", - "ecrm1000.tfm": "texlive-ec", - "eqparbox.sty": "texlive-eqparbox", - "eu1enc.def": "texlive-euenc", - "fancybox.sty": "texlive-fancybox", - "fancyvrb.sty": "texlive-fancyvrb", - "float.sty": "texlive-float", - "fncychap.sty": "texlive-fncychap", - "footnote.sty": "texlive-mdwtools", - "framed.sty": "texlive-framed", - "luatex85.sty": "texlive-luatex85", - "multirow.sty": "texlive-multirow", - "needspace.sty": "texlive-needspace", - "palatino.sty": "texlive-psnfss", - "parskip.sty": "texlive-parskip", - "polyglossia.sty": "texlive-polyglossia", - "tabulary.sty": "texlive-tabulary", - "threeparttable.sty": "texlive-threeparttable", - "titlesec.sty": "texlive-titlesec", - "ucs.sty": "texlive-ucs", - "upquote.sty": "texlive-upquote", - "wrapfig.sty": "texlive-wrapfig", - } +class MissingCheckers(AncillaryMethods): - def __init__(self, args): + def __init__(self, args, texlive): self.pdf = args.pdf self.virtualenv = args.virtualenv self.version_check = args.version_check + self.texlive = texlive self.deps = DepManager(self.pdf) self.need_symlink = 0 self.need_sphinx = 0 - self.need_pip = 0 - self.rec_sphinx_upgrade = 0 + self.verbose_warn_install = 1 - self.system_release = "" - self.install = "" self.virtenv_dir = "" - self.python_cmd = "" - self.activate_cmd = "" - # Some distros may not have a Sphinx shipped package compatible with - # our minimal requirements - self.package_supported = True - - # Recommend a new python version - self.recommend_python = None - - # Certain hints are meant to be shown only once - self.first_hint = True - - self.min_version = (0, 0, 0) - self.cur_version = (0, 0, 0) - self.latest_avail_ver = (0, 0, 0) - self.venv_ver = (0, 0, 0) - - prefix = os.environ.get("srctree", ".") + "/" - - self.conf = prefix + "Documentation/conf.py" - self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" - self.virtenv_prefix = ["sphinx_", "Sphinx_" ] + self.install = "" # # Methods to check if a feature exists @@ -460,9 +403,9 @@ class SphinxDependencyChecker(AncillaryCheckers): if match: return parse_version(match.group(1)) - def check_sphinx(self): + def check_sphinx(self, conf): try: - with open(self.conf, "r", encoding="utf-8") as f: + with open(conf, "r", encoding="utf-8") as f: for line in f: match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line) if match: @@ -513,6 +456,73 @@ class SphinxDependencyChecker(AncillaryCheckers): return True +class SphinxDependencyChecker(MissingCheckers): + + def __init__(self, args): + # List of required texlive packages on Fedora and OpenSuse + texlive = { + "amsfonts.sty": "texlive-amsfonts", + "amsmath.sty": "texlive-amsmath", + "amssymb.sty": "texlive-amsfonts", + "amsthm.sty": "texlive-amscls", + "anyfontsize.sty": "texlive-anyfontsize", + "atbegshi.sty": "texlive-oberdiek", + "bm.sty": "texlive-tools", + "capt-of.sty": "texlive-capt-of", + "cmap.sty": "texlive-cmap", + "ctexhook.sty": "texlive-ctex", + "ecrm1000.tfm": "texlive-ec", + "eqparbox.sty": "texlive-eqparbox", + "eu1enc.def": "texlive-euenc", + "fancybox.sty": "texlive-fancybox", + "fancyvrb.sty": "texlive-fancyvrb", + "float.sty": "texlive-float", + "fncychap.sty": "texlive-fncychap", + "footnote.sty": "texlive-mdwtools", + "framed.sty": "texlive-framed", + "luatex85.sty": "texlive-luatex85", + "multirow.sty": "texlive-multirow", + "needspace.sty": "texlive-needspace", + "palatino.sty": "texlive-psnfss", + "parskip.sty": "texlive-parskip", + "polyglossia.sty": "texlive-polyglossia", + "tabulary.sty": "texlive-tabulary", + "threeparttable.sty": "texlive-threeparttable", + "titlesec.sty": "texlive-titlesec", + "ucs.sty": "texlive-ucs", + "upquote.sty": "texlive-upquote", + "wrapfig.sty": "texlive-wrapfig", + } + + super().__init__(args, texlive) + + self.need_pip = 0 + self.rec_sphinx_upgrade = 0 + + self.system_release = "" + self.python_cmd = "" + self.activate_cmd = "" + + # Some distros may not have a Sphinx shipped package compatible with + # our minimal requirements + self.package_supported = True + + # Recommend a new python version + self.recommend_python = None + + # Certain hints are meant to be shown only once + self.first_hint = True + + self.min_version = (0, 0, 0) + self.cur_version = (0, 0, 0) + self.latest_avail_ver = (0, 0, 0) + self.venv_ver = (0, 0, 0) + + prefix = os.environ.get("srctree", ".") + "/" + + self.conf = prefix + "Documentation/conf.py" + self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" + self.virtenv_prefix = ["sphinx_", "Sphinx_" ] # # Distro-specific hints methods @@ -1187,7 +1197,7 @@ class SphinxDependencyChecker(AncillaryCheckers): self.python_cmd = sys.executable # Check if Sphinx is already accessible from current environment - self.check_sphinx() + self.check_sphinx(self.conf) if self.system_release: print(f"Detected OS: {self.system_release}.") -- cgit From fb22e438b23eabd828a0ca1076be5ecf9b0262db Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:46 +0200 Subject: scripts: sphinx-pre-install: move get_system_release() The code at get_system_release() is actually a helper function, independent from the actual Sphinx verification checker. Move it to MissingCheckers class, where other checkers are present. With that, the entire distro-specific handler logic, with all its complexity is confined at SphinxDependencyChecker class. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/4b42a85bbb6575bb34a58cf66019038c4afa1d5b.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 115 ++++++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 56 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 9127487bd4d7..593982f350b3 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -456,6 +456,64 @@ class MissingCheckers(AncillaryMethods): return True + def get_system_release(self): + """ + Determine the system type. There's no unique way that would work + with all distros with a minimal package install. So, several + methods are used here. + + By default, it will use lsb_release function. If not available, it will + fail back to reading the known different places where the distro name + is stored. + + Several modern distros now have /etc/os-release, which usually have + a decent coverage. + """ + + system_release = "" + + if self.which("lsb_release"): + result = self.run(["lsb_release", "-d"], capture_output=True, text=True) + system_release = result.stdout.replace("Description:", "").strip() + + release_files = [ + "/etc/system-release", + "/etc/redhat-release", + "/etc/lsb-release", + "/etc/gentoo-release", + ] + + if not system_release: + for f in release_files: + system_release = self.catcheck(f) + if system_release: + break + + # This seems more common than LSB these days + if not system_release: + os_var = {} + try: + with open("/etc/os-release", "r", encoding="utf-8") as f: + for line in f: + match = re.match(r"^([\w\d\_]+)=\"?([^\"]*)\"?\n", line) + if match: + os_var[match.group(1)] = match.group(2) + + system_release = os_var.get("NAME", "") + if "VERSION_ID" in os_var: + system_release += " " + os_var["VERSION_ID"] + elif "VERSION" in os_var: + system_release += " " + os_var["VERSION"] + except IOError: + pass + + if not system_release: + system_release = self.catcheck("/etc/issue") + + system_release = system_release.strip() + + return system_release + class SphinxDependencyChecker(MissingCheckers): def __init__(self, args): @@ -499,7 +557,7 @@ class SphinxDependencyChecker(MissingCheckers): self.need_pip = 0 self.rec_sphinx_upgrade = 0 - self.system_release = "" + self.system_release = self.get_system_release() self.python_cmd = "" self.activate_cmd = "" @@ -1193,7 +1251,6 @@ class SphinxDependencyChecker(MissingCheckers): "\thttps://github.com/sphinx-doc/sphinx/pull/8313") def check_needs(self): - self.get_system_release() self.python_cmd = sys.executable # Check if Sphinx is already accessible from current environment @@ -1270,60 +1327,6 @@ class SphinxDependencyChecker(MissingCheckers): print("Needed package dependencies are met.") - def get_system_release(self): - """ - Determine the system type. There's no unique way that would work - with all distros with a minimal package install. So, several - methods are used here. - - By default, it will use lsb_release function. If not available, it will - fail back to reading the known different places where the distro name - is stored. - - Several modern distros now have /etc/os-release, which usually have - a decent coverage. - """ - - if self.which("lsb_release"): - result = self.run(["lsb_release", "-d"], capture_output=True, text=True) - self.system_release = result.stdout.replace("Description:", "").strip() - - release_files = [ - "/etc/system-release", - "/etc/redhat-release", - "/etc/lsb-release", - "/etc/gentoo-release", - ] - - if not self.system_release: - for f in release_files: - self.system_release = self.catcheck(f) - if self.system_release: - break - - # This seems more common than LSB these days - if not self.system_release: - os_var = {} - try: - with open("/etc/os-release", "r", encoding="utf-8") as f: - for line in f: - match = re.match(r"^([\w\d\_]+)=\"?([^\"]*)\"?\n", line) - if match: - os_var[match.group(1)] = match.group(2) - - self.system_release = os_var.get("NAME", "") - if "VERSION_ID" in os_var: - self.system_release += " " + os_var["VERSION_ID"] - elif "VERSION" in os_var: - self.system_release += " " + os_var["VERSION"] - except IOError: - pass - - if not self.system_release: - self.system_release = self.catcheck("/etc/issue") - - self.system_release = self.system_release.strip() - DESCRIPTION = """ Process some flags related to Sphinx installation and documentation build. """ -- cgit From f477c6d71d3947ab8f2e9e7d5fd8448f7a26c1ab Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:47 +0200 Subject: scripts: sphinx-pre-install: add documentation for the ancillary classes. While here, rename a parameter to have its usage better documented. No functional changes. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/7421112b14edf5c21cc4cf0f2ee320fcaf874b40.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 162 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 140 insertions(+), 22 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 593982f350b3..1dc3f19804ab 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -33,35 +33,62 @@ MIN_PYTHON_VERSION = parse_version("3.7") class DepManager: + """ + Manage package dependencies. There are three types of dependencies: + + - System: dependencies required for docs build; + - Python: python dependencies for a native distro Sphinx install; + - PDF: dependencies needed by PDF builds. + + Each dependency can be mandatory or optional. Not installing an optional + dependency won't break the build, but will cause degradation at the + docs output. + """ - # Internal types of dependencies. + # Internal types of dependencies. Don't use them outside DepManager class. _SYS_TYPE = 0 _PHY_TYPE = 1 _PDF_TYPE = 2 - # Let's define keys as a tuple with the type and mandatory/optional. - # This way, checking for optional or type is easy. + # Dependencies visible outside the class. + # The keys are tuple with: (type, is_mandatory flag). + # + # Currently we're not using all optional dep types. Yet, we'll keep all + # possible combinations here. They're not many, and that makes easier + # if later needed and for the name() method below SYSTEM_MANDATORY = (_SYS_TYPE, True) PYTHON_MANDATORY = (_PHY_TYPE, True) PDF_MANDATORY = (_PDF_TYPE, True) - # Currently we're not using all optional types, but let's keep all - # combinations here, as we may end needing them in the future. Also, - # it allows a name() function that handles all possibilities. SYSTEM_OPTIONAL = (_SYS_TYPE, False) PYTHON_OPTIONAL = (_PHY_TYPE, False) PDF_OPTIONAL = (_PDF_TYPE, True) def __init__(self, pdf): + """ + Initialize internal vars: + + - missing: missing dependencies list, containing a distro-independent + name for a missing dependency and its type. + - missing_pkg: ancillary dict containing missing dependencies in + distro namespace, organized by type. + - need: total number of needed dependencies. Never cleaned. + - optional: total number of optional dependencies. Never cleaned. + - pdf: PDF support is enabled? + """ self.missing = {} - self.run = {} + self.missing_pkg = {} self.need = 0 self.optional = 0 self.pdf = pdf @staticmethod def name(dtype): + """ + Ancillary routine to output a warn/error message reporting + missing dependencies. + """ if dtype[0] == DepManager._SYS_TYPE: msg = "build" elif dtype[0] == DepManager._PHY_TYPE: @@ -76,16 +103,22 @@ class DepManager: @staticmethod def is_optional(dtype): + """Ancillary routine to report if a dependency is optional""" return not dtype[1] @staticmethod def is_pdf(dtype): + """Ancillary routine to report if a dependency is for PDF generation""" if (dtype[0] == DepManager._PDF_TYPE): return True return False def add_package(self, package, dtype): + """ + Add a package at the self.missing() dictionary. + Doesn't update missing_pkg. + """ is_optional = DepManager.is_optional(dtype) self.missing[package] = dtype if is_optional: @@ -94,6 +127,10 @@ class DepManager: self.need += 1 def del_package(self, package): + """ + Remove a package at the self.missing() dictionary. + Doesn't update missing_pkg. + """ if package in self.missing: del self.missing[package] @@ -104,13 +141,22 @@ class DepManager: This is an ackward way to have a separate section to recommend a package after system main dependencies. - TODO: rework the logic to prevent needing it + TODO: rework the logic to prevent needing it. """ self.missing = {} + self.missing_pkg = {} def check_missing(self, progs): - self.run = {} + """ + Update self.missing_pkg, using progs dict to convert from the + agnostic package name to distro-specific one. + + Returns an string with the packages to be installed, sorted and + with eventual duplicates removed. + """ + + self.missing_pkg = {} for prog, dtype in sorted(self.missing.items()): # At least on some LTS distros like CentOS 7, texlive doesn't @@ -123,23 +169,26 @@ class DepManager: self.optional -= 1 continue - if not dtype in self.run: - self.run[dtype] = [] + if not dtype in self.missing_pkg: + self.missing_pkg[dtype] = [] - self.run[dtype].append(progs.get(prog, prog)) + self.missing_pkg[dtype].append(progs.get(prog, prog)) install = [] - for dtype in self.run.keys(): - install += self.run[dtype] + for dtype in self.missing_pkg.keys(): + install += self.missing_pkg[dtype] return " ".join(sorted(set(install))) def warn_install(self): + """ + Emit warnings/errors related to missing packages. + """ output_msg = "" - for dtype in sorted(self.run.keys()): - progs = " ".join(sorted(set(self.run[dtype]))) + for dtype in sorted(self.missing_pkg.keys()): + progs = " ".join(sorted(set(self.missing_pkg[dtype]))) try: name = DepManager.name(dtype) @@ -158,6 +207,11 @@ class AncillaryMethods: @staticmethod def which(prog): + """ + Our own implementation of which(). We could instead use + shutil.which(), but this function is simple enough. + Probably faster to use this implementation than to import shutil. + """ for path in os.environ.get("PATH", "").split(":"): full_path = os.path.join(path, prog) if os.access(full_path, os.X_OK): @@ -167,6 +221,10 @@ class AncillaryMethods: @staticmethod def get_python_version(cmd): + """ + Get python version from a Python binary. As we need to detect if + are out there newer python binaries, we can't rely on sys.release here. + """ result = SphinxDependencyChecker.run([cmd, "--version"], capture_output=True, text=True) @@ -181,7 +239,13 @@ class AncillaryMethods: @staticmethod def find_python(): + """ + Detect if are out there any python 3.xy version newer than the + current one. + Note: this routine is limited to up to 2 digits for python3. We + may need to update it one day, hopefully on a distant future. + """ patterns = [ "python3.[0-9]", "python3.[0-9][0-9]", @@ -200,7 +264,10 @@ class AncillaryMethods: @staticmethod def check_python(): - + """ + Check if the current python binary satisfies our minimal requirement + for Sphinx build. If not, re-run with a newer version if found. + """ cur_ver = sys.version_info[:3] if cur_ver >= MIN_PYTHON_VERSION: ver = ver_str(cur_ver) @@ -240,7 +307,10 @@ class AncillaryMethods: @staticmethod def run(*args, **kwargs): - """Excecute a command, hiding its output by default""" + """ + Excecute a command, hiding its output by default. + Preserve comatibility with older Python versions. + """ capture_output = kwargs.pop('capture_output', False) @@ -262,8 +332,15 @@ class AncillaryMethods: return subprocess.run(*args, **kwargs) class MissingCheckers(AncillaryMethods): + """ + Contains some ancillary checkers for different types of binaries and + package managers. + """ def __init__(self, args, texlive): + """ + Initialize its internal variables + """ self.pdf = args.pdf self.virtualenv = args.virtualenv self.version_check = args.version_check @@ -280,17 +357,20 @@ class MissingCheckers(AncillaryMethods): self.install = "" - # - # Methods to check if a feature exists - # - def check_missing_file(self, files, package, dtype): + """ + Does the file exists? If not, add it to missing dependencies. + """ for f in files: if os.path.exists(f): return self.deps.add_package(package, dtype) def check_program(self, prog, dtype): + """ + Does the program exists and it is at the PATH? + If not, add it to missing dependencies. + """ found = self.which(prog) if found: return found @@ -300,6 +380,18 @@ class MissingCheckers(AncillaryMethods): return None def check_perl_module(self, prog, dtype): + """ + Does perl have a dependency? Is it available? + If not, add it to missing dependencies. + + Right now, we still need Perl for doc build, as it is required + by some tools called at docs or kernel build time, like: + + scripts/documentation-file-ref-check + + Also, checkpatch is on Perl. + """ + # While testing with lxc download template, one of the # distros (Oracle) didn't have perl - nor even an option to install # before installing oraclelinux-release-el9 package. @@ -318,6 +410,10 @@ class MissingCheckers(AncillaryMethods): self.deps.add_package(prog, dtype) def check_python_module(self, module, is_optional=False): + """ + Does a python module exists outside venv? If not, add it to missing + dependencies. + """ if is_optional: dtype = DepManager.PYTHON_OPTIONAL else: @@ -329,6 +425,9 @@ class MissingCheckers(AncillaryMethods): self.deps.add_package(module, dtype) def check_rpm_missing(self, pkgs, dtype): + """ + Does a rpm package exists? If not, add it to missing dependencies. + """ for prog in pkgs: try: self.run(["rpm", "-q", prog], check=True) @@ -336,6 +435,9 @@ class MissingCheckers(AncillaryMethods): self.deps.add_package(prog, dtype) def check_pacman_missing(self, pkgs, dtype): + """ + Does a pacman package exists? If not, add it to missing dependencies. + """ for prog in pkgs: try: self.run(["pacman", "-Q", prog], check=True) @@ -343,6 +445,9 @@ class MissingCheckers(AncillaryMethods): self.deps.add_package(prog, dtype) def check_missing_tex(self, is_optional=False): + """ + Does a LaTeX package exists? If not, add it to missing dependencies. + """ if is_optional: dtype = DepManager.PDF_OPTIONAL else: @@ -371,6 +476,9 @@ class MissingCheckers(AncillaryMethods): self.deps.add_package(package, dtype) def get_sphinx_fname(self): + """ + Gets the binary filename for sphinx-build. + """ if "SPHINXBUILD" in os.environ: return os.environ["SPHINXBUILD"] @@ -386,6 +494,9 @@ class MissingCheckers(AncillaryMethods): return "" def get_sphinx_version(self, cmd): + """ + Gets sphinx-build version. + """ try: result = self.run([cmd, "--version"], stdout=subprocess.PIPE, @@ -404,6 +515,9 @@ class MissingCheckers(AncillaryMethods): return parse_version(match.group(1)) def check_sphinx(self, conf): + """ + Checks Sphinx minimal requirements + """ try: with open(conf, "r", encoding="utf-8") as f: for line in f: @@ -441,6 +555,10 @@ class MissingCheckers(AncillaryMethods): sys.exit(0) def catcheck(self, filename): + """ + Reads a file if it exists, returning as string. + If not found, returns an empty string. + """ if os.path.exists(filename): with open(filename, "r", encoding="utf-8") as f: return f.read().strip() -- cgit From 6d5f4f3da1a82786650d38067bb4c5e4ed84dc61 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:48 +0200 Subject: scripts: sphinx-pre-install: add docstring documentation This program is somewhat complex. Add some docstring documentation, explaining what each function and class is supposed to do. Most of the focus here were to describe the ancillary functions used to detect dependency needs. The main SphinxDependencyChecker still requires a lot of care, and probably need to be reorganized to clearly split the 4 types of output it produces: - Need to upgrade Python binary; - System install needs; - Virtual env install needs; - Python install needs via system packages, to run Sphinx natively. Yet, for now, I'm happy of having it a lot better documented than its Perl version. - While here, rename a parameter to have its usage better documented. No functional changes. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0cadab2cab3f78ae6d9f378e92a45125fbc5188f.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 110 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 102 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 1dc3f19804ab..1c96f6692e9a 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -9,6 +9,22 @@ # Don't add changes not compatible with it, it is meant to report # incompatible python versions. +""" +Dependency checker for Sphinx documentation Kernel build. + +This module provides tools to check for all required dependencies needed to +build documentation using Sphinx, including system packages, Python modules +and LaTeX packages for PDF generation. + +It detect packages for a subset of Linux distributions used by Kernel +maintainers, showing hints and missing dependencies. + +The main class SphinxDependencyChecker handles the dependency checking logic +and provides recommendations for installing missing packages. It supports both +system package installations and Python virtual environments. By default, +system pacage install is recommended. +""" + import argparse import os import re @@ -75,7 +91,7 @@ class DepManager: distro namespace, organized by type. - need: total number of needed dependencies. Never cleaned. - optional: total number of optional dependencies. Never cleaned. - - pdf: PDF support is enabled? + - pdf: Is PDF support enabled? """ self.missing = {} self.missing_pkg = {} @@ -565,6 +581,13 @@ class MissingCheckers(AncillaryMethods): return "" def check_missing(self, progs): + """ + Check for missing dependencies using the provided program mapping. + + The actual distro-specific programs are mapped via progs argument. + + Returns True if there are missing dependencies, False otherwise. + """ self.install += self.deps.check_missing(progs) if self.verbose_warn_install: self.deps.warn_install() @@ -633,8 +656,18 @@ class MissingCheckers(AncillaryMethods): return system_release class SphinxDependencyChecker(MissingCheckers): + """ + Main class for checking Sphinx documentation build dependencies. + - Check for missing system packages; + - Check for missing Python modules; + - Check for missing LaTeX packages needed by PDF generation; + - Propose Sphinx install via Python Virtual environment; + - Propose Sphinx install via distro-specific package install. + """ def __init__(self, args): + """Initialize checker variables""" + # List of required texlive packages on Fedora and OpenSuse texlive = { "amsfonts.sty": "texlive-amsfonts", @@ -705,6 +738,9 @@ class SphinxDependencyChecker(MissingCheckers): # def give_debian_hints(self): + """ + Provide package installation hints for Debian-based distros. + """ progs = { "Pod::Usage": "perl-modules", "convert": "imagemagick", @@ -745,6 +781,10 @@ class SphinxDependencyChecker(MissingCheckers): print(f"\n\tsudo apt-get install {self.install}") def give_redhat_hints(self): + """ + Provide package installation hints for RedHat-based distros + (Fedora, RHEL and RHEL-based variants). + """ progs = { "Pod::Usage": "perl-Pod-Usage", "convert": "ImageMagick", @@ -837,6 +877,10 @@ class SphinxDependencyChecker(MissingCheckers): print(f"\n\tsudo dnf install -y {self.install}") def give_opensuse_hints(self): + """ + Provide package installation hints for openSUSE-based distros + (Leap and Tumbleweed). + """ progs = { "Pod::Usage": "perl-Pod-Usage", "convert": "ImageMagick", @@ -909,6 +953,9 @@ class SphinxDependencyChecker(MissingCheckers): print(f"\n\tsudo zypper install --no-recommends {self.install}") def give_mageia_hints(self): + """ + Provide package installation hints for Mageia and OpenMandriva. + """ progs = { "Pod::Usage": "perl-Pod-Usage", "convert": "ImageMagick", @@ -956,6 +1003,9 @@ class SphinxDependencyChecker(MissingCheckers): print(f"\n\tsudo {packager_cmd} {self.install}") def give_arch_linux_hints(self): + """ + Provide package installation hints for ArchLinux. + """ progs = { "convert": "imagemagick", "dot": "graphviz", @@ -989,6 +1039,9 @@ class SphinxDependencyChecker(MissingCheckers): print(f"\n\tsudo pacman -S {self.install}") def give_gentoo_hints(self): + """ + Provide package installation hints for Gentoo. + """ progs = { "convert": "media-gfx/imagemagick", "dot": "media-gfx/graphviz", @@ -1107,7 +1160,12 @@ class SphinxDependencyChecker(MissingCheckers): # def check_distros(self): - # OS-specific hints logic + """ + OS-specific hints logic. Seeks for a hinter. If found, provide + package-manager-specific install commands. + + Otherwise, just lists the missing dependencies. + """ os_hints = { re.compile("Red Hat Enterprise Linux"): self.give_redhat_hints, re.compile("Fedora"): self.give_redhat_hints, @@ -1159,10 +1217,22 @@ class SphinxDependencyChecker(MissingCheckers): # Common dependencies # def deactivate_help(self): + """ + Print a helper message to disable a virtual environment. + """ + print("\n If you want to exit the virtualenv, you can use:") print("\tdeactivate") def get_virtenv(self): + """ + Give a hint about how to activate an already-existing virtual + environment containing sphinx-build. + + Returns a tuble with (activate_cmd_path, sphinx_version) with + the newest available virtual env. + """ + cwd = os.getcwd() activates = [] @@ -1207,6 +1277,14 @@ class SphinxDependencyChecker(MissingCheckers): return ("", ver) def recommend_sphinx_upgrade(self): + """ + Check if Sphinx needs to be upgraded. + + Returns a tuple with the higest available Sphinx version if found. + Otherwise, returns None to indicate either that no upgrade is needed + or no venv was found. + """ + # Avoid running sphinx-builds from venv if cur_version is good if self.cur_version and self.cur_version >= RECOMMENDED_VERSION: self.latest_avail_ver = self.cur_version @@ -1247,6 +1325,9 @@ class SphinxDependencyChecker(MissingCheckers): return self.latest_avail_ver def recommend_package(self): + """ + Recommend installing Sphinx as a distro-specific package. + """ print("\n2) As a package with:") @@ -1270,11 +1351,19 @@ class SphinxDependencyChecker(MissingCheckers): self.verbose_warn_install = old_verbose def recommend_sphinx_version(self, virtualenv_cmd): - # The logic here is complex, as it have to deal with different versions: - # - minimal supported version; - # - minimal PDF version; - # - recommended version. - # It also needs to work fine with both distro's package and venv/virtualenv + """ + Provide recommendations for installing or upgrading Sphinx based + on current version. + + The logic here is complex, as it have to deal with different versions: + + - minimal supported version; + - minimal PDF version; + - recommended version. + + It also needs to work fine with both distro's package and + venv/virtualenv + """ if self.recommend_python: print("\nPython version is incompatible with doc build.\n" \ @@ -1369,6 +1458,10 @@ class SphinxDependencyChecker(MissingCheckers): "\thttps://github.com/sphinx-doc/sphinx/pull/8313") def check_needs(self): + """ + Main method that checks needed dependencies and provides + recommendations. + """ self.python_cmd = sys.executable # Check if Sphinx is already accessible from current environment @@ -1451,6 +1544,7 @@ Process some flags related to Sphinx installation and documentation build. def main(): + """Main function""" parser = argparse.ArgumentParser(description=DESCRIPTION) parser.add_argument( @@ -1481,6 +1575,6 @@ def main(): checker.check_python() checker.check_needs() - +# Call main if not used as module if __name__ == "__main__": main() -- cgit From 8b45effaa2ce2790e3b391e1ac4bb668e0d37560 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:49 +0200 Subject: scripts: sphinx-pre-install: fix several codingstyle issues Address most pylint issues. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/5139b18535e1436e4b1773706224a9ec3a386697.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 52 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 1c96f6692e9a..1b11162da9fb 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (c) 2017-2025 Mauro Carvalho Chehab # -# pylint: disable=C0103,C0114,C0115,C0116,C0301 -# pylint: disable=R0902,R0904,R0912,R0915,R1705,R1710,E1121 +# pylint: disable=C0103,C0114,C0115,C0116,C0301,C0302 +# pylint: disable=R0902,R0904,R0911,R0912,R0914,R0915,R1705,R1710,E1121 # Note: this script requires at least Python 3.6 to run. # Don't add changes not compatible with it, it is meant to report @@ -115,7 +115,7 @@ class DepManager: if dtype[1]: return f"ERROR: {msg} mandatory deps missing" else: - out = f"Warning: {msg} optional deps missing" + return f"Warning: {msg} optional deps missing" @staticmethod def is_optional(dtype): @@ -125,7 +125,7 @@ class DepManager: @staticmethod def is_pdf(dtype): """Ancillary routine to report if a dependency is for PDF generation""" - if (dtype[0] == DepManager._PDF_TYPE): + if dtype[0] == DepManager._PDF_TYPE: return True return False @@ -191,8 +191,8 @@ class DepManager: self.missing_pkg[dtype].append(progs.get(prog, prog)) install = [] - for dtype in self.missing_pkg.keys(): - install += self.missing_pkg[dtype] + for dtype, pkgs in self.missing_pkg.items(): + install += pkgs return " ".join(sorted(set(install))) @@ -267,8 +267,6 @@ class AncillaryMethods: "python3.[0-9][0-9]", ] - new_python_cmd = None - # Seek for a python binary newer than MIN_PYTHON_VERSION for path in os.getenv("PATH", "").split(":"): for pattern in patterns: @@ -276,7 +274,7 @@ class AncillaryMethods: if os.path.isfile(cmd) and os.access(cmd, os.X_OK): version = SphinxDependencyChecker.get_python_version(cmd) if version >= MIN_PYTHON_VERSION: - return(cmd) + return cmd @staticmethod def check_python(): @@ -306,8 +304,8 @@ class AncillaryMethods: new_python_cmd = SphinxDependencyChecker.find_python() if not new_python_cmd: - print(f"ERROR: Python version {python_ver} is not spported anymore") - print(f" Can't find a new version. This script may fail") + print(f"ERROR: Python version {python_ver} is not spported anymore\n") + print(" Can't find a new version. This script may fail") return # Restart script using the newer version @@ -362,6 +360,9 @@ class MissingCheckers(AncillaryMethods): self.version_check = args.version_check self.texlive = texlive + self.min_version = (0, 0, 0) + self.cur_version = (0, 0, 0) + self.deps = DepManager(self.pdf) self.need_symlink = 0 @@ -370,8 +371,10 @@ class MissingCheckers(AncillaryMethods): self.verbose_warn_install = 1 self.virtenv_dir = "" - self.install = "" + self.python_cmd = "" + + self.virtenv_prefix = ["sphinx_", "Sphinx_" ] def check_missing_file(self, files, package, dtype): """ @@ -542,10 +545,10 @@ class MissingCheckers(AncillaryMethods): self.min_version = parse_version(match.group(1)) break except IOError: - sys.exit(f"Can't open {self.conf}") + sys.exit(f"Can't open {conf}") if not self.min_version: - sys.exit(f"Can't get needs_sphinx version from {self.conf}") + sys.exit(f"Can't get needs_sphinx version from {conf}") self.virtenv_dir = self.virtenv_prefix[0] + "latest" @@ -709,7 +712,6 @@ class SphinxDependencyChecker(MissingCheckers): self.rec_sphinx_upgrade = 0 self.system_release = self.get_system_release() - self.python_cmd = "" self.activate_cmd = "" # Some distros may not have a Sphinx shipped package compatible with @@ -722,8 +724,6 @@ class SphinxDependencyChecker(MissingCheckers): # Certain hints are meant to be shown only once self.first_hint = True - self.min_version = (0, 0, 0) - self.cur_version = (0, 0, 0) self.latest_avail_ver = (0, 0, 0) self.venv_ver = (0, 0, 0) @@ -731,7 +731,6 @@ class SphinxDependencyChecker(MissingCheckers): self.conf = prefix + "Documentation/conf.py" self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" - self.virtenv_prefix = ["sphinx_", "Sphinx_" ] # # Distro-specific hints methods @@ -814,6 +813,7 @@ class SphinxDependencyChecker(MissingCheckers): if not rel: print("Couldn't identify release number") + noto_sans_redhat = None self.pdf = False elif re.search("Fedora", self.system_release): # Fedora 38 and upper use this CJK font @@ -1111,7 +1111,7 @@ class SphinxDependencyChecker(MissingCheckers): for fname, portage in portages.items(): install = False - while install == False: + while install is False: if not files: # No files under package.usage. Install all install = True @@ -1335,7 +1335,7 @@ class SphinxDependencyChecker(MissingCheckers): old_optional = self.deps.optional self.pdf = False - self.optional = 0 + self.deps.optional = 0 self.install = "" old_verbose = self.verbose_warn_install self.verbose_warn_install = 0 @@ -1346,8 +1346,8 @@ class SphinxDependencyChecker(MissingCheckers): self.check_distros() - self.need = old_need - self.optional = old_optional + self.deps.need = old_need + self.deps.optional = old_optional self.verbose_warn_install = old_verbose def recommend_sphinx_version(self, virtualenv_cmd): @@ -1528,13 +1528,13 @@ class SphinxDependencyChecker(MissingCheckers): self.recommend_sphinx_version(virtualenv_cmd) print("") - if not self.optional: + if not self.deps.optional: print("All optional dependencies are met.") - if self.need == 1: + if self.deps.need == 1: sys.exit("Can't build as 1 mandatory dependency is missing") - elif self.need: - sys.exit(f"Can't build as {self.need} mandatory dependencies are missing") + elif self.deps.need: + sys.exit(f"Can't build as {self.deps.need} mandatory dependencies are missing") print("Needed package dependencies are met.") -- cgit From 24a34b3b453dc8fff0366a28c472b603f98a5c20 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:50 +0200 Subject: scripts: sphinx-pre-install: rework install command logic Cleanup the code to remove some redundancy and to let it be clearer about the command install instructions. Ensure that special instructions will be shown only once, before the actual install command. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/a6120449d9cc14346e867d1ef8944ae28ddbf3f6.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 179 +++++++++++++++++++----------------------- 1 file changed, 82 insertions(+), 97 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 1b11162da9fb..7dfe5c2a6cc2 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -213,7 +213,7 @@ class DepManager: raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") if output_msg: - print(f"\n{output_msg}\n") + print(f"\n{output_msg}") class AncillaryMethods: """ @@ -583,23 +583,6 @@ class MissingCheckers(AncillaryMethods): return f.read().strip() return "" - def check_missing(self, progs): - """ - Check for missing dependencies using the provided program mapping. - - The actual distro-specific programs are mapped via progs argument. - - Returns True if there are missing dependencies, False otherwise. - """ - self.install += self.deps.check_missing(progs) - if self.verbose_warn_install: - self.deps.warn_install() - - if not self.deps.need and not self.deps.optional: - return False - - return True - def get_system_release(self): """ Determine the system type. There's no unique way that would work @@ -722,7 +705,7 @@ class SphinxDependencyChecker(MissingCheckers): self.recommend_python = None # Certain hints are meant to be shown only once - self.first_hint = True + self.distro_msg = None self.latest_avail_ver = (0, 0, 0) self.venv_ver = (0, 0, 0) @@ -732,6 +715,33 @@ class SphinxDependencyChecker(MissingCheckers): self.conf = prefix + "Documentation/conf.py" self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" + def get_install_progs(self, progs, cmd, extra=None): + """ + Check for missing dependencies using the provided program mapping. + + The actual distro-specific programs are mapped via progs argument. + """ + install = self.deps.check_missing(progs) + + if self.verbose_warn_install: + self.deps.warn_install() + + if not install: + return + + if cmd: + if self.verbose_warn_install: + msg = "You should run:" + else: + msg = "" + + if extra: + msg += "\n\t" + extra.replace("\n", "\n\t") + + return(msg + "\n\tsudo " + cmd + " " + install) + + return None + # # Distro-specific hints methods # @@ -772,12 +782,7 @@ class SphinxDependencyChecker(MissingCheckers): self.check_program("dvipng", DepManager.PDF_MANDATORY) - if self.check_missing(progs): - return - - if self.verbose_warn_install: - print("You should run:") - print(f"\n\tsudo apt-get install {self.install}") + return self.get_install_progs(progs, "apt-get install") def give_redhat_hints(self): """ @@ -838,18 +843,16 @@ class SphinxDependencyChecker(MissingCheckers): self.deps.add_package("python39", DepManager.SYSTEM_MANDATORY) self.recommend_python = True - if self.first_hint: - print("Note: RHEL-based distros typically require extra repositories.\n" \ - "For most, enabling epel and crb are enough:\n" \ - "\tsudo dnf install -y epel-release", \ - "\tsudo dnf config-manager --set-enabled crb\n" \ - "Yet, some may have other required repositories. Those commands could be useful:\n" \ - "\tsudo dnf repolist all\n" \ - "\tsudo dnf repoquery --available --info \n", - "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want") - - self.first_hint = False - + if not self.distro_msg: + self.distro_msg = \ + "Note: RHEL-based distros typically require extra repositories.\n" \ + "For most, enabling epel and crb are enough:\n" \ + "\tsudo dnf install -y epel-release\n" \ + "\tsudo dnf config-manager --set-enabled crb\n" \ + "Yet, some may have other required repositories. Those commands could be useful:\n" \ + "\tsudo dnf repolist all\n" \ + "\tsudo dnf repoquery --available --info \n" \ + "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want" if self.pdf: pdf_pkgs = [ @@ -868,13 +871,7 @@ class SphinxDependencyChecker(MissingCheckers): if not fedora and rel == 8: self.deps.del_package("texlive-ctex") - if self.check_missing(progs): - return - - if self.verbose_warn_install: - print("You should run:") - - print(f"\n\tsudo dnf install -y {self.install}") + return self.get_install_progs(progs, "dnf install") def give_opensuse_hints(self): """ @@ -945,12 +942,7 @@ class SphinxDependencyChecker(MissingCheckers): if self.pdf: self.check_missing_tex() - if self.check_missing(progs): - return - - if self.verbose_warn_install: - print("You should run:") - print(f"\n\tsudo zypper install --no-recommends {self.install}") + return self.get_install_progs(progs, "zypper install --no-recommends") def give_mageia_hints(self): """ @@ -995,12 +987,7 @@ class SphinxDependencyChecker(MissingCheckers): self.check_missing_file(pdf_pkgs, noto_sans, DepManager.PDF_MANDATORY) self.check_rpm_missing(tex_pkgs, DepManager.PDF_MANDATORY) - if self.check_missing(progs): - return - - if self.verbose_warn_install: - print("You should run:") - print(f"\n\tsudo {packager_cmd} {self.install}") + return self.get_install_progs(progs, packager_cmd) def give_arch_linux_hints(self): """ @@ -1023,20 +1010,15 @@ class SphinxDependencyChecker(MissingCheckers): ] if self.pdf: - self.check_pacman_missing(archlinux_tex_pkgs, DepManager.PDF_MANDATORY) + self.check_pacman_missing(archlinux_tex_pkgs, + DepManager.PDF_MANDATORY) - self.check_missing_file( - ["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], - "noto-fonts-cjk", - 2, - ) + self.check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], + "noto-fonts-cjk", + DepManager.PDF_MANDATORY) - if self.check_missing(progs): - return - if self.verbose_warn_install: - print("You should run:") - print(f"\n\tsudo pacman -S {self.install}") + return self.get_install_progs(progs, "pacman -S") def give_gentoo_hints(self): """ @@ -1065,13 +1047,6 @@ class SphinxDependencyChecker(MissingCheckers): for package, files in pdf_pkgs.items(): self.check_missing_file(files, package, DepManager.PDF_MANDATORY) - if self.check_missing(progs): - return - - if self.verbose_warn_install: - print("You should run:") - print("\n") - # Handling dependencies is a nightmare, as Gentoo refuses to emerge # some packages if there's no package.use file describing them. # To make it worse, compilation flags shall also be present there @@ -1104,7 +1079,10 @@ class SphinxDependencyChecker(MissingCheckers): "zziblib": "dev-libs/zziplib sdl", } - if self.first_hint: + extra_cmds = "" + if not self.distro_msg: + self.distro_msg = "Note: Gentoo requires package.use to be adjusted before emerging packages" + use_base = "/etc/portage/package.use" files = glob(f"{use_base}/*") @@ -1148,23 +1126,23 @@ class SphinxDependencyChecker(MissingCheckers): # emit a code to setup missing USE if install: - print(f"\tsudo su -c 'echo \"{portage}\" > {use_base}/{fname}'") - - self.first_hint = False + extra_cmds += (f"sudo su -c 'echo \"{portage}\" > {use_base}/{fname}'\n") # Now, we can use emerge and let it respect USE - print(f"\tsudo emerge --ask --changed-use --binpkg-respect-use=y {self.install}") + return self.get_install_progs(progs, + "emerge --ask --changed-use --binpkg-respect-use=y", + extra_cmds) - # - # Dispatch the check to an os_specific hinter - # - - def check_distros(self): + def get_install(self): """ - OS-specific hints logic. Seeks for a hinter. If found, provide - package-manager-specific install commands. + OS-specific hints logic. Seeks for a hinter. If found, use it to + provide package-manager specific install commands. + + Otherwise, outputs install instructions for the meta-packages. - Otherwise, just lists the missing dependencies. + Returns a string with the command to be executed to install the + the needed packages, if distro found. Otherwise, return just a + list of packages that require installation. """ os_hints = { re.compile("Red Hat Enterprise Linux"): self.give_redhat_hints, @@ -1195,9 +1173,7 @@ class SphinxDependencyChecker(MissingCheckers): # If the OS is detected, use per-OS hint logic for regex, os_hint in os_hints.items(): if regex.search(self.system_release): - os_hint() - - return + return os_hint() # # Fall-back to generic hint code for other distros @@ -1207,11 +1183,12 @@ class SphinxDependencyChecker(MissingCheckers): if self.pdf: self.check_missing_tex() - self.check_missing(progs) + self.distro_msg = \ + f"I don't know distro {self.system_release}.\n" \ + "So, I can't provide you a hint with the install procedure.\n" \ + "There are likely missing dependencies.\n" - print(f"I don't know distro {self.system_release}.") - print("So, I can't provide you a hint with the install procedure.") - print("There are likely missing dependencies.") + return self.get_install_progs(progs, None) # # Common dependencies @@ -1336,7 +1313,6 @@ class SphinxDependencyChecker(MissingCheckers): self.pdf = False self.deps.optional = 0 - self.install = "" old_verbose = self.verbose_warn_install self.verbose_warn_install = 0 @@ -1344,7 +1320,9 @@ class SphinxDependencyChecker(MissingCheckers): self.deps.add_package("python-sphinx", DepManager.PYTHON_MANDATORY) - self.check_distros() + cmd = self.get_install() + if cmd: + print(cmd) self.deps.need = old_need self.deps.optional = old_optional @@ -1511,7 +1489,14 @@ class SphinxDependencyChecker(MissingCheckers): self.check_program("latexmk", DepManager.PDF_MANDATORY) # Do distro-specific checks and output distro-install commands - self.check_distros() + cmd = self.get_install() + if cmd: + print(cmd) + + # If distro requires some special instructions, print here. + # Please notice that get_install() needs to be called first. + if self.distro_msg: + print("\n" + self.distro_msg) if not self.python_cmd: if self.need == 1: -- cgit From d43cd965f3a646d22851192c659b787dba311d87 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:51 +0200 Subject: docs: Makefile: switch to the new scripts/sphinx-pre-install.py Now that we have a better, improved Python script, use it when checking for documentation build dependencies. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/79508fb071512c33e807f5411bbff1904751b5d3.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 1565 +++++++++++++++++++++++++++++++++++++++++ scripts/sphinx-pre-install.py | 1565 ----------------------------------------- 2 files changed, 1565 insertions(+), 1565 deletions(-) create mode 100755 scripts/sphinx-pre-install delete mode 100755 scripts/sphinx-pre-install.py (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install new file mode 100755 index 000000000000..7dfe5c2a6cc2 --- /dev/null +++ b/scripts/sphinx-pre-install @@ -0,0 +1,1565 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (c) 2017-2025 Mauro Carvalho Chehab +# +# pylint: disable=C0103,C0114,C0115,C0116,C0301,C0302 +# pylint: disable=R0902,R0904,R0911,R0912,R0914,R0915,R1705,R1710,E1121 + +# Note: this script requires at least Python 3.6 to run. +# Don't add changes not compatible with it, it is meant to report +# incompatible python versions. + +""" +Dependency checker for Sphinx documentation Kernel build. + +This module provides tools to check for all required dependencies needed to +build documentation using Sphinx, including system packages, Python modules +and LaTeX packages for PDF generation. + +It detect packages for a subset of Linux distributions used by Kernel +maintainers, showing hints and missing dependencies. + +The main class SphinxDependencyChecker handles the dependency checking logic +and provides recommendations for installing missing packages. It supports both +system package installations and Python virtual environments. By default, +system pacage install is recommended. +""" + +import argparse +import os +import re +import subprocess +import sys +from glob import glob + + +def parse_version(version): + """Convert a major.minor.patch version into a tuple""" + return tuple(int(x) for x in version.split(".")) + + +def ver_str(version): + """Returns a version tuple as major.minor.patch""" + + return ".".join([str(x) for x in version]) + + +RECOMMENDED_VERSION = parse_version("3.4.3") +MIN_PYTHON_VERSION = parse_version("3.7") + + +class DepManager: + """ + Manage package dependencies. There are three types of dependencies: + + - System: dependencies required for docs build; + - Python: python dependencies for a native distro Sphinx install; + - PDF: dependencies needed by PDF builds. + + Each dependency can be mandatory or optional. Not installing an optional + dependency won't break the build, but will cause degradation at the + docs output. + """ + + # Internal types of dependencies. Don't use them outside DepManager class. + _SYS_TYPE = 0 + _PHY_TYPE = 1 + _PDF_TYPE = 2 + + # Dependencies visible outside the class. + # The keys are tuple with: (type, is_mandatory flag). + # + # Currently we're not using all optional dep types. Yet, we'll keep all + # possible combinations here. They're not many, and that makes easier + # if later needed and for the name() method below + + SYSTEM_MANDATORY = (_SYS_TYPE, True) + PYTHON_MANDATORY = (_PHY_TYPE, True) + PDF_MANDATORY = (_PDF_TYPE, True) + + SYSTEM_OPTIONAL = (_SYS_TYPE, False) + PYTHON_OPTIONAL = (_PHY_TYPE, False) + PDF_OPTIONAL = (_PDF_TYPE, True) + + def __init__(self, pdf): + """ + Initialize internal vars: + + - missing: missing dependencies list, containing a distro-independent + name for a missing dependency and its type. + - missing_pkg: ancillary dict containing missing dependencies in + distro namespace, organized by type. + - need: total number of needed dependencies. Never cleaned. + - optional: total number of optional dependencies. Never cleaned. + - pdf: Is PDF support enabled? + """ + self.missing = {} + self.missing_pkg = {} + self.need = 0 + self.optional = 0 + self.pdf = pdf + + @staticmethod + def name(dtype): + """ + Ancillary routine to output a warn/error message reporting + missing dependencies. + """ + if dtype[0] == DepManager._SYS_TYPE: + msg = "build" + elif dtype[0] == DepManager._PHY_TYPE: + msg = "Python" + else: + msg = "PDF" + + if dtype[1]: + return f"ERROR: {msg} mandatory deps missing" + else: + return f"Warning: {msg} optional deps missing" + + @staticmethod + def is_optional(dtype): + """Ancillary routine to report if a dependency is optional""" + return not dtype[1] + + @staticmethod + def is_pdf(dtype): + """Ancillary routine to report if a dependency is for PDF generation""" + if dtype[0] == DepManager._PDF_TYPE: + return True + + return False + + def add_package(self, package, dtype): + """ + Add a package at the self.missing() dictionary. + Doesn't update missing_pkg. + """ + is_optional = DepManager.is_optional(dtype) + self.missing[package] = dtype + if is_optional: + self.optional += 1 + else: + self.need += 1 + + def del_package(self, package): + """ + Remove a package at the self.missing() dictionary. + Doesn't update missing_pkg. + """ + if package in self.missing: + del self.missing[package] + + def clear_deps(self): + """ + Clear dependencies without changing needed/optional. + + This is an ackward way to have a separate section to recommend + a package after system main dependencies. + + TODO: rework the logic to prevent needing it. + """ + + self.missing = {} + self.missing_pkg = {} + + def check_missing(self, progs): + """ + Update self.missing_pkg, using progs dict to convert from the + agnostic package name to distro-specific one. + + Returns an string with the packages to be installed, sorted and + with eventual duplicates removed. + """ + + self.missing_pkg = {} + + for prog, dtype in sorted(self.missing.items()): + # At least on some LTS distros like CentOS 7, texlive doesn't + # provide all packages we need. When such distros are + # detected, we have to disable PDF output. + # + # So, we need to ignore the packages that distros would + # need for LaTeX to work + if DepManager.is_pdf(dtype) and not self.pdf: + self.optional -= 1 + continue + + if not dtype in self.missing_pkg: + self.missing_pkg[dtype] = [] + + self.missing_pkg[dtype].append(progs.get(prog, prog)) + + install = [] + for dtype, pkgs in self.missing_pkg.items(): + install += pkgs + + return " ".join(sorted(set(install))) + + def warn_install(self): + """ + Emit warnings/errors related to missing packages. + """ + + output_msg = "" + + for dtype in sorted(self.missing_pkg.keys()): + progs = " ".join(sorted(set(self.missing_pkg[dtype]))) + + try: + name = DepManager.name(dtype) + output_msg += f'{name}:\t{progs}\n' + except KeyError: + raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") + + if output_msg: + print(f"\n{output_msg}") + +class AncillaryMethods: + """ + Ancillary methods that checks for missing dependencies for different + types of types, like binaries, python modules, rpm deps, etc. + """ + + @staticmethod + def which(prog): + """ + Our own implementation of which(). We could instead use + shutil.which(), but this function is simple enough. + Probably faster to use this implementation than to import shutil. + """ + for path in os.environ.get("PATH", "").split(":"): + full_path = os.path.join(path, prog) + if os.access(full_path, os.X_OK): + return full_path + + return None + + @staticmethod + def get_python_version(cmd): + """ + Get python version from a Python binary. As we need to detect if + are out there newer python binaries, we can't rely on sys.release here. + """ + + result = SphinxDependencyChecker.run([cmd, "--version"], + capture_output=True, text=True) + version = result.stdout.strip() + + match = re.search(r"(\d+\.\d+\.\d+)", version) + if match: + return parse_version(match.group(1)) + + print(f"Can't parse version {version}") + return (0, 0, 0) + + @staticmethod + def find_python(): + """ + Detect if are out there any python 3.xy version newer than the + current one. + + Note: this routine is limited to up to 2 digits for python3. We + may need to update it one day, hopefully on a distant future. + """ + patterns = [ + "python3.[0-9]", + "python3.[0-9][0-9]", + ] + + # Seek for a python binary newer than MIN_PYTHON_VERSION + for path in os.getenv("PATH", "").split(":"): + for pattern in patterns: + for cmd in glob(os.path.join(path, pattern)): + if os.path.isfile(cmd) and os.access(cmd, os.X_OK): + version = SphinxDependencyChecker.get_python_version(cmd) + if version >= MIN_PYTHON_VERSION: + return cmd + + @staticmethod + def check_python(): + """ + Check if the current python binary satisfies our minimal requirement + for Sphinx build. If not, re-run with a newer version if found. + """ + cur_ver = sys.version_info[:3] + if cur_ver >= MIN_PYTHON_VERSION: + ver = ver_str(cur_ver) + print(f"Python version: {ver}") + + # This could be useful for debugging purposes + if SphinxDependencyChecker.which("docutils"): + result = SphinxDependencyChecker.run(["docutils", "--version"], + capture_output=True, text=True) + ver = result.stdout.strip() + match = re.search(r"(\d+\.\d+\.\d+)", ver) + if match: + ver = match.group(1) + + print(f"Docutils version: {ver}") + + return + + python_ver = ver_str(cur_ver) + + new_python_cmd = SphinxDependencyChecker.find_python() + if not new_python_cmd: + print(f"ERROR: Python version {python_ver} is not spported anymore\n") + print(" Can't find a new version. This script may fail") + return + + # Restart script using the newer version + script_path = os.path.abspath(sys.argv[0]) + args = [new_python_cmd, script_path] + sys.argv[1:] + + print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") + + try: + os.execv(new_python_cmd, args) + except OSError as e: + sys.exit(f"Failed to restart with {new_python_cmd}: {e}") + + @staticmethod + def run(*args, **kwargs): + """ + Excecute a command, hiding its output by default. + Preserve comatibility with older Python versions. + """ + + capture_output = kwargs.pop('capture_output', False) + + if capture_output: + if 'stdout' not in kwargs: + kwargs['stdout'] = subprocess.PIPE + if 'stderr' not in kwargs: + kwargs['stderr'] = subprocess.PIPE + else: + if 'stdout' not in kwargs: + kwargs['stdout'] = subprocess.DEVNULL + if 'stderr' not in kwargs: + kwargs['stderr'] = subprocess.DEVNULL + + # Don't break with older Python versions + if 'text' in kwargs and sys.version_info < (3, 7): + kwargs['universal_newlines'] = kwargs.pop('text') + + return subprocess.run(*args, **kwargs) + +class MissingCheckers(AncillaryMethods): + """ + Contains some ancillary checkers for different types of binaries and + package managers. + """ + + def __init__(self, args, texlive): + """ + Initialize its internal variables + """ + self.pdf = args.pdf + self.virtualenv = args.virtualenv + self.version_check = args.version_check + self.texlive = texlive + + self.min_version = (0, 0, 0) + self.cur_version = (0, 0, 0) + + self.deps = DepManager(self.pdf) + + self.need_symlink = 0 + self.need_sphinx = 0 + + self.verbose_warn_install = 1 + + self.virtenv_dir = "" + self.install = "" + self.python_cmd = "" + + self.virtenv_prefix = ["sphinx_", "Sphinx_" ] + + def check_missing_file(self, files, package, dtype): + """ + Does the file exists? If not, add it to missing dependencies. + """ + for f in files: + if os.path.exists(f): + return + self.deps.add_package(package, dtype) + + def check_program(self, prog, dtype): + """ + Does the program exists and it is at the PATH? + If not, add it to missing dependencies. + """ + found = self.which(prog) + if found: + return found + + self.deps.add_package(prog, dtype) + + return None + + def check_perl_module(self, prog, dtype): + """ + Does perl have a dependency? Is it available? + If not, add it to missing dependencies. + + Right now, we still need Perl for doc build, as it is required + by some tools called at docs or kernel build time, like: + + scripts/documentation-file-ref-check + + Also, checkpatch is on Perl. + """ + + # While testing with lxc download template, one of the + # distros (Oracle) didn't have perl - nor even an option to install + # before installing oraclelinux-release-el9 package. + # + # Check it before running an error. If perl is not there, + # add it as a mandatory package, as some parts of the doc builder + # needs it. + if not self.which("perl"): + self.deps.add_package("perl", DepManager.SYSTEM_MANDATORY) + self.deps.add_package(prog, dtype) + return + + try: + self.run(["perl", f"-M{prog}", "-e", "1"], check=True) + except subprocess.CalledProcessError: + self.deps.add_package(prog, dtype) + + def check_python_module(self, module, is_optional=False): + """ + Does a python module exists outside venv? If not, add it to missing + dependencies. + """ + if is_optional: + dtype = DepManager.PYTHON_OPTIONAL + else: + dtype = DepManager.PYTHON_MANDATORY + + try: + self.run([self.python_cmd, "-c", f"import {module}"], check=True) + except subprocess.CalledProcessError: + self.deps.add_package(module, dtype) + + def check_rpm_missing(self, pkgs, dtype): + """ + Does a rpm package exists? If not, add it to missing dependencies. + """ + for prog in pkgs: + try: + self.run(["rpm", "-q", prog], check=True) + except subprocess.CalledProcessError: + self.deps.add_package(prog, dtype) + + def check_pacman_missing(self, pkgs, dtype): + """ + Does a pacman package exists? If not, add it to missing dependencies. + """ + for prog in pkgs: + try: + self.run(["pacman", "-Q", prog], check=True) + except subprocess.CalledProcessError: + self.deps.add_package(prog, dtype) + + def check_missing_tex(self, is_optional=False): + """ + Does a LaTeX package exists? If not, add it to missing dependencies. + """ + if is_optional: + dtype = DepManager.PDF_OPTIONAL + else: + dtype = DepManager.PDF_MANDATORY + + kpsewhich = self.which("kpsewhich") + for prog, package in self.texlive.items(): + + # If kpsewhich is not there, just add it to deps + if not kpsewhich: + self.deps.add_package(package, dtype) + continue + + # Check if the package is needed + try: + result = self.run( + [kpsewhich, prog], stdout=subprocess.PIPE, text=True, check=True + ) + + # Didn't find. Add it + if not result.stdout.strip(): + self.deps.add_package(package, dtype) + + except subprocess.CalledProcessError: + # kpsewhich returned an error. Add it, just in case + self.deps.add_package(package, dtype) + + def get_sphinx_fname(self): + """ + Gets the binary filename for sphinx-build. + """ + if "SPHINXBUILD" in os.environ: + return os.environ["SPHINXBUILD"] + + fname = "sphinx-build" + if self.which(fname): + return fname + + fname = "sphinx-build-3" + if self.which(fname): + self.need_symlink = 1 + return fname + + return "" + + def get_sphinx_version(self, cmd): + """ + Gets sphinx-build version. + """ + try: + result = self.run([cmd, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, check=True) + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + for line in result.stdout.split("\n"): + match = re.match(r"^sphinx-build\s+([\d\.]+)(?:\+(?:/[\da-f]+)|b\d+)?\s*$", line) + if match: + return parse_version(match.group(1)) + + match = re.match(r"^Sphinx.*\s+([\d\.]+)\s*$", line) + if match: + return parse_version(match.group(1)) + + def check_sphinx(self, conf): + """ + Checks Sphinx minimal requirements + """ + try: + with open(conf, "r", encoding="utf-8") as f: + for line in f: + match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line) + if match: + self.min_version = parse_version(match.group(1)) + break + except IOError: + sys.exit(f"Can't open {conf}") + + if not self.min_version: + sys.exit(f"Can't get needs_sphinx version from {conf}") + + self.virtenv_dir = self.virtenv_prefix[0] + "latest" + + sphinx = self.get_sphinx_fname() + if not sphinx: + self.need_sphinx = 1 + return + + self.cur_version = self.get_sphinx_version(sphinx) + if not self.cur_version: + sys.exit(f"{sphinx} didn't return its version") + + if self.cur_version < self.min_version: + curver = ver_str(self.cur_version) + minver = ver_str(self.min_version) + + print(f"ERROR: Sphinx version is {curver}. It should be >= {minver}") + self.need_sphinx = 1 + return + + # On version check mode, just assume Sphinx has all mandatory deps + if self.version_check and self.cur_version >= RECOMMENDED_VERSION: + sys.exit(0) + + def catcheck(self, filename): + """ + Reads a file if it exists, returning as string. + If not found, returns an empty string. + """ + if os.path.exists(filename): + with open(filename, "r", encoding="utf-8") as f: + return f.read().strip() + return "" + + def get_system_release(self): + """ + Determine the system type. There's no unique way that would work + with all distros with a minimal package install. So, several + methods are used here. + + By default, it will use lsb_release function. If not available, it will + fail back to reading the known different places where the distro name + is stored. + + Several modern distros now have /etc/os-release, which usually have + a decent coverage. + """ + + system_release = "" + + if self.which("lsb_release"): + result = self.run(["lsb_release", "-d"], capture_output=True, text=True) + system_release = result.stdout.replace("Description:", "").strip() + + release_files = [ + "/etc/system-release", + "/etc/redhat-release", + "/etc/lsb-release", + "/etc/gentoo-release", + ] + + if not system_release: + for f in release_files: + system_release = self.catcheck(f) + if system_release: + break + + # This seems more common than LSB these days + if not system_release: + os_var = {} + try: + with open("/etc/os-release", "r", encoding="utf-8") as f: + for line in f: + match = re.match(r"^([\w\d\_]+)=\"?([^\"]*)\"?\n", line) + if match: + os_var[match.group(1)] = match.group(2) + + system_release = os_var.get("NAME", "") + if "VERSION_ID" in os_var: + system_release += " " + os_var["VERSION_ID"] + elif "VERSION" in os_var: + system_release += " " + os_var["VERSION"] + except IOError: + pass + + if not system_release: + system_release = self.catcheck("/etc/issue") + + system_release = system_release.strip() + + return system_release + +class SphinxDependencyChecker(MissingCheckers): + """ + Main class for checking Sphinx documentation build dependencies. + + - Check for missing system packages; + - Check for missing Python modules; + - Check for missing LaTeX packages needed by PDF generation; + - Propose Sphinx install via Python Virtual environment; + - Propose Sphinx install via distro-specific package install. + """ + def __init__(self, args): + """Initialize checker variables""" + + # List of required texlive packages on Fedora and OpenSuse + texlive = { + "amsfonts.sty": "texlive-amsfonts", + "amsmath.sty": "texlive-amsmath", + "amssymb.sty": "texlive-amsfonts", + "amsthm.sty": "texlive-amscls", + "anyfontsize.sty": "texlive-anyfontsize", + "atbegshi.sty": "texlive-oberdiek", + "bm.sty": "texlive-tools", + "capt-of.sty": "texlive-capt-of", + "cmap.sty": "texlive-cmap", + "ctexhook.sty": "texlive-ctex", + "ecrm1000.tfm": "texlive-ec", + "eqparbox.sty": "texlive-eqparbox", + "eu1enc.def": "texlive-euenc", + "fancybox.sty": "texlive-fancybox", + "fancyvrb.sty": "texlive-fancyvrb", + "float.sty": "texlive-float", + "fncychap.sty": "texlive-fncychap", + "footnote.sty": "texlive-mdwtools", + "framed.sty": "texlive-framed", + "luatex85.sty": "texlive-luatex85", + "multirow.sty": "texlive-multirow", + "needspace.sty": "texlive-needspace", + "palatino.sty": "texlive-psnfss", + "parskip.sty": "texlive-parskip", + "polyglossia.sty": "texlive-polyglossia", + "tabulary.sty": "texlive-tabulary", + "threeparttable.sty": "texlive-threeparttable", + "titlesec.sty": "texlive-titlesec", + "ucs.sty": "texlive-ucs", + "upquote.sty": "texlive-upquote", + "wrapfig.sty": "texlive-wrapfig", + } + + super().__init__(args, texlive) + + self.need_pip = 0 + self.rec_sphinx_upgrade = 0 + + self.system_release = self.get_system_release() + self.activate_cmd = "" + + # Some distros may not have a Sphinx shipped package compatible with + # our minimal requirements + self.package_supported = True + + # Recommend a new python version + self.recommend_python = None + + # Certain hints are meant to be shown only once + self.distro_msg = None + + self.latest_avail_ver = (0, 0, 0) + self.venv_ver = (0, 0, 0) + + prefix = os.environ.get("srctree", ".") + "/" + + self.conf = prefix + "Documentation/conf.py" + self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" + + def get_install_progs(self, progs, cmd, extra=None): + """ + Check for missing dependencies using the provided program mapping. + + The actual distro-specific programs are mapped via progs argument. + """ + install = self.deps.check_missing(progs) + + if self.verbose_warn_install: + self.deps.warn_install() + + if not install: + return + + if cmd: + if self.verbose_warn_install: + msg = "You should run:" + else: + msg = "" + + if extra: + msg += "\n\t" + extra.replace("\n", "\n\t") + + return(msg + "\n\tsudo " + cmd + " " + install) + + return None + + # + # Distro-specific hints methods + # + + def give_debian_hints(self): + """ + Provide package installation hints for Debian-based distros. + """ + progs = { + "Pod::Usage": "perl-modules", + "convert": "imagemagick", + "dot": "graphviz", + "ensurepip": "python3-venv", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2-bin", + "virtualenv": "virtualenv", + "xelatex": "texlive-xetex", + "yaml": "python3-yaml", + } + + if self.pdf: + pdf_pkgs = { + "texlive-lang-chinese": [ + "/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty", + ], + "fonts-dejavu": [ + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + ], + "fonts-noto-cjk": [ + "/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc", + ], + } + + for package, files in pdf_pkgs.items(): + self.check_missing_file(files, package, DepManager.PDF_MANDATORY) + + self.check_program("dvipng", DepManager.PDF_MANDATORY) + + return self.get_install_progs(progs, "apt-get install") + + def give_redhat_hints(self): + """ + Provide package installation hints for RedHat-based distros + (Fedora, RHEL and RHEL-based variants). + """ + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2-tools", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive-xetex-bin", + "yaml": "python3-pyyaml", + } + + fedora_tex_pkgs = [ + "dejavu-sans-fonts", + "dejavu-sans-mono-fonts", + "dejavu-serif-fonts", + "texlive-collection-fontsrecommended", + "texlive-collection-latex", + "texlive-xecjk", + ] + + fedora = False + rel = None + + match = re.search(r"(release|Linux)\s+(\d+)", self.system_release) + if match: + rel = int(match.group(2)) + + if not rel: + print("Couldn't identify release number") + noto_sans_redhat = None + self.pdf = False + elif re.search("Fedora", self.system_release): + # Fedora 38 and upper use this CJK font + + noto_sans_redhat = "google-noto-sans-cjk-fonts" + fedora = True + else: + # Almalinux, CentOS, RHEL, ... + + # at least up to version 9 (and Fedora < 38), that's the CJK font + noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts" + + progs["virtualenv"] = "python-virtualenv" + + if not rel or rel < 8: + print("ERROR: Distro not supported. Too old?") + return + + # RHEL 8 uses Python 3.6, which is not compatible with + # the build system anymore. Suggest Python 3.11 + if rel == 8: + self.deps.add_package("python39", DepManager.SYSTEM_MANDATORY) + self.recommend_python = True + + if not self.distro_msg: + self.distro_msg = \ + "Note: RHEL-based distros typically require extra repositories.\n" \ + "For most, enabling epel and crb are enough:\n" \ + "\tsudo dnf install -y epel-release\n" \ + "\tsudo dnf config-manager --set-enabled crb\n" \ + "Yet, some may have other required repositories. Those commands could be useful:\n" \ + "\tsudo dnf repolist all\n" \ + "\tsudo dnf repoquery --available --info \n" \ + "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want" + + if self.pdf: + pdf_pkgs = [ + "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc", + ] + + self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepManager.PDF_MANDATORY) + + self.check_rpm_missing(fedora_tex_pkgs, DepManager.PDF_MANDATORY) + + self.check_missing_tex(DepManager.PDF_MANDATORY) + + # There's no texlive-ctex on RHEL 8 repositories. This will + # likely affect CJK pdf build only. + if not fedora and rel == 8: + self.deps.del_package("texlive-ctex") + + return self.get_install_progs(progs, "dnf install") + + def give_opensuse_hints(self): + """ + Provide package installation hints for openSUSE-based distros + (Leap and Tumbleweed). + """ + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive-xetex-bin", + "yaml": "python3-pyyaml", + } + + suse_tex_pkgs = [ + "texlive-babel-english", + "texlive-caption", + "texlive-colortbl", + "texlive-courier", + "texlive-dvips", + "texlive-helvetic", + "texlive-makeindex", + "texlive-metafont", + "texlive-metapost", + "texlive-palatino", + "texlive-preview", + "texlive-times", + "texlive-zapfchan", + "texlive-zapfding", + ] + + progs["latexmk"] = "texlive-latexmk-bin" + + match = re.search(r"(Leap)\s+(\d+).(\d)", self.system_release) + if match: + rel = int(match.group(2)) + + # Leap 15.x uses Python 3.6, which is not compatible with + # the build system anymore. Suggest Python 3.11 + if rel == 15: + if not self.which(self.python_cmd): + self.recommend_python = True + self.deps.add_package(self.python_cmd, DepManager.SYSTEM_MANDATORY) + + progs.update({ + "python-sphinx": "python311-Sphinx", + "virtualenv": "python311-virtualenv", + "yaml": "python311-PyYAML", + }) + else: + # Tumbleweed defaults to Python 3.11 + + progs.update({ + "python-sphinx": "python313-Sphinx", + "virtualenv": "python313-virtualenv", + "yaml": "python313-PyYAML", + }) + + # FIXME: add support for installing CJK fonts + # + # I tried hard, but was unable to find a way to install + # "Noto Sans CJK SC" on openSUSE + + if self.pdf: + self.check_rpm_missing(suse_tex_pkgs, DepManager.PDF_MANDATORY) + if self.pdf: + self.check_missing_tex() + + return self.get_install_progs(progs, "zypper install --no-recommends") + + def give_mageia_hints(self): + """ + Provide package installation hints for Mageia and OpenMandriva. + """ + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive", + "yaml": "python3-yaml", + } + + tex_pkgs = [ + "texlive-fontsextra", + ] + + if re.search(r"OpenMandriva", self.system_release): + packager_cmd = "dnf install" + noto_sans = "noto-sans-cjk-fonts" + tex_pkgs = ["texlive-collection-fontsextra"] + + # Tested on OpenMandriva Lx 4.3 + progs["convert"] = "imagemagick" + progs["yaml"] = "python-pyyaml" + + else: + packager_cmd = "urpmi" + noto_sans = "google-noto-sans-cjk-ttc-fonts" + + progs["latexmk"] = "texlive-collection-basic" + + if self.pdf: + pdf_pkgs = [ + "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/TTF/NotoSans-Regular.ttf", + ] + + self.check_missing_file(pdf_pkgs, noto_sans, DepManager.PDF_MANDATORY) + self.check_rpm_missing(tex_pkgs, DepManager.PDF_MANDATORY) + + return self.get_install_progs(progs, packager_cmd) + + def give_arch_linux_hints(self): + """ + Provide package installation hints for ArchLinux. + """ + progs = { + "convert": "imagemagick", + "dot": "graphviz", + "latexmk": "texlive-core", + "rsvg-convert": "extra/librsvg", + "virtualenv": "python-virtualenv", + "xelatex": "texlive-xetex", + "yaml": "python-yaml", + } + + archlinux_tex_pkgs = [ + "texlive-core", + "texlive-latexextra", + "ttf-dejavu", + ] + + if self.pdf: + self.check_pacman_missing(archlinux_tex_pkgs, + DepManager.PDF_MANDATORY) + + self.check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], + "noto-fonts-cjk", + DepManager.PDF_MANDATORY) + + + return self.get_install_progs(progs, "pacman -S") + + def give_gentoo_hints(self): + """ + Provide package installation hints for Gentoo. + """ + progs = { + "convert": "media-gfx/imagemagick", + "dot": "media-gfx/graphviz", + "rsvg-convert": "gnome-base/librsvg", + "virtualenv": "dev-python/virtualenv", + "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", + "yaml": "dev-python/pyyaml", + "python-sphinx": "dev-python/sphinx", + } + + if self.pdf: + pdf_pkgs = { + "media-fonts/dejavu": [ + "/usr/share/fonts/dejavu/DejaVuSans.ttf", + ], + "media-fonts/noto-cjk": [ + "/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", + "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc", + ], + } + for package, files in pdf_pkgs.items(): + self.check_missing_file(files, package, DepManager.PDF_MANDATORY) + + # Handling dependencies is a nightmare, as Gentoo refuses to emerge + # some packages if there's no package.use file describing them. + # To make it worse, compilation flags shall also be present there + # for some packages. If USE is not perfect, error/warning messages + # like those are shown: + # + # !!! The following binary packages have been ignored due to non matching USE: + # + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_13 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf python_single_target_python3_12 -python_single_target_python3_13 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 python_single_target_python3_12 -python_single_target_python3_13 qt6 svg + # =media-fonts/noto-cjk-20190416 X + # =app-text/texlive-core-2024-r1 X cjk -xetex + # =app-text/texlive-core-2024-r1 X -xetex + # =app-text/texlive-core-2024-r1 -xetex + # =dev-libs/zziplib-0.13.79-r1 sdl + # + # And will ignore such packages, installing the remaining ones. That + # affects mostly the image extension and PDF generation. + + # Package dependencies and the minimal needed args: + portages = { + "graphviz": "media-gfx/graphviz", + "imagemagick": "media-gfx/imagemagick", + "media-libs": "media-libs/harfbuzz icu", + "media-fonts": "media-fonts/noto-cjk", + "texlive": "app-text/texlive-core xetex", + "zziblib": "dev-libs/zziplib sdl", + } + + extra_cmds = "" + if not self.distro_msg: + self.distro_msg = "Note: Gentoo requires package.use to be adjusted before emerging packages" + + use_base = "/etc/portage/package.use" + files = glob(f"{use_base}/*") + + for fname, portage in portages.items(): + install = False + + while install is False: + if not files: + # No files under package.usage. Install all + install = True + break + + args = portage.split(" ") + + name = args.pop(0) + + cmd = ["grep", "-l", "-E", rf"^{name}\b" ] + files + result = self.run(cmd, stdout=subprocess.PIPE, text=True) + if result.returncode or not result.stdout.strip(): + # File containing portage name not found + install = True + break + + # Ensure that needed USE flags are present + if args: + match_fname = result.stdout.strip() + with open(match_fname, 'r', encoding='utf8', + errors='backslashreplace') as fp: + for line in fp: + for arg in args: + if arg.startswith("-"): + continue + + if not re.search(rf"\s*{arg}\b", line): + # Needed file argument not found + install = True + break + + # Everything looks ok, don't install + break + + # emit a code to setup missing USE + if install: + extra_cmds += (f"sudo su -c 'echo \"{portage}\" > {use_base}/{fname}'\n") + + # Now, we can use emerge and let it respect USE + return self.get_install_progs(progs, + "emerge --ask --changed-use --binpkg-respect-use=y", + extra_cmds) + + def get_install(self): + """ + OS-specific hints logic. Seeks for a hinter. If found, use it to + provide package-manager specific install commands. + + Otherwise, outputs install instructions for the meta-packages. + + Returns a string with the command to be executed to install the + the needed packages, if distro found. Otherwise, return just a + list of packages that require installation. + """ + os_hints = { + re.compile("Red Hat Enterprise Linux"): self.give_redhat_hints, + re.compile("Fedora"): self.give_redhat_hints, + re.compile("AlmaLinux"): self.give_redhat_hints, + re.compile("Amazon Linux"): self.give_redhat_hints, + re.compile("CentOS"): self.give_redhat_hints, + re.compile("openEuler"): self.give_redhat_hints, + re.compile("Oracle Linux Server"): self.give_redhat_hints, + re.compile("Rocky Linux"): self.give_redhat_hints, + re.compile("Springdale Open Enterprise"): self.give_redhat_hints, + + re.compile("Ubuntu"): self.give_debian_hints, + re.compile("Debian"): self.give_debian_hints, + re.compile("Devuan"): self.give_debian_hints, + re.compile("Kali"): self.give_debian_hints, + re.compile("Mint"): self.give_debian_hints, + + re.compile("openSUSE"): self.give_opensuse_hints, + + re.compile("Mageia"): self.give_mageia_hints, + re.compile("OpenMandriva"): self.give_mageia_hints, + + re.compile("Arch Linux"): self.give_arch_linux_hints, + re.compile("Gentoo"): self.give_gentoo_hints, + } + + # If the OS is detected, use per-OS hint logic + for regex, os_hint in os_hints.items(): + if regex.search(self.system_release): + return os_hint() + + # + # Fall-back to generic hint code for other distros + # That's far from ideal, specially for LaTeX dependencies. + # + progs = {"sphinx-build": "sphinx"} + if self.pdf: + self.check_missing_tex() + + self.distro_msg = \ + f"I don't know distro {self.system_release}.\n" \ + "So, I can't provide you a hint with the install procedure.\n" \ + "There are likely missing dependencies.\n" + + return self.get_install_progs(progs, None) + + # + # Common dependencies + # + def deactivate_help(self): + """ + Print a helper message to disable a virtual environment. + """ + + print("\n If you want to exit the virtualenv, you can use:") + print("\tdeactivate") + + def get_virtenv(self): + """ + Give a hint about how to activate an already-existing virtual + environment containing sphinx-build. + + Returns a tuble with (activate_cmd_path, sphinx_version) with + the newest available virtual env. + """ + + cwd = os.getcwd() + + activates = [] + + # Add all sphinx prefixes with possible version numbers + for p in self.virtenv_prefix: + activates += glob(f"{cwd}/{p}[0-9]*/bin/activate") + + activates.sort(reverse=True, key=str.lower) + + # Place sphinx_latest first, if it exists + for p in self.virtenv_prefix: + activates = glob(f"{cwd}/{p}*latest/bin/activate") + activates + + ver = (0, 0, 0) + for f in activates: + # Discard too old Sphinx virtual environments + match = re.search(r"(\d+)\.(\d+)\.(\d+)", f) + if match: + ver = (int(match.group(1)), int(match.group(2)), int(match.group(3))) + + if ver < self.min_version: + continue + + sphinx_cmd = f.replace("activate", "sphinx-build") + if not os.path.isfile(sphinx_cmd): + continue + + ver = self.get_sphinx_version(sphinx_cmd) + + if not ver: + venv_dir = f.replace("/bin/activate", "") + print(f"Warning: virtual environment {venv_dir} is not working.\n" \ + "Python version upgrade? Remove it with:\n\n" \ + "\trm -rf {venv_dir}\n\n") + else: + if self.need_sphinx and ver >= self.min_version: + return (f, ver) + elif parse_version(ver) > self.cur_version: + return (f, ver) + + return ("", ver) + + def recommend_sphinx_upgrade(self): + """ + Check if Sphinx needs to be upgraded. + + Returns a tuple with the higest available Sphinx version if found. + Otherwise, returns None to indicate either that no upgrade is needed + or no venv was found. + """ + + # Avoid running sphinx-builds from venv if cur_version is good + if self.cur_version and self.cur_version >= RECOMMENDED_VERSION: + self.latest_avail_ver = self.cur_version + return None + + # Get the highest version from sphinx_*/bin/sphinx-build and the + # corresponding command to activate the venv/virtenv + self.activate_cmd, self.venv_ver = self.get_virtenv() + + # Store the highest version from Sphinx existing virtualenvs + if self.activate_cmd and self.venv_ver > self.cur_version: + self.latest_avail_ver = self.venv_ver + else: + if self.cur_version: + self.latest_avail_ver = self.cur_version + else: + self.latest_avail_ver = (0, 0, 0) + + # As we don't know package version of Sphinx, and there's no + # virtual environments, don't check if upgrades are needed + if not self.virtualenv: + if not self.latest_avail_ver: + return None + + return self.latest_avail_ver + + # Either there are already a virtual env or a new one should be created + self.need_pip = 1 + + if not self.latest_avail_ver: + return None + + # Return if the reason is due to an upgrade or not + if self.latest_avail_ver != (0, 0, 0): + if self.latest_avail_ver < RECOMMENDED_VERSION: + self.rec_sphinx_upgrade = 1 + + return self.latest_avail_ver + + def recommend_package(self): + """ + Recommend installing Sphinx as a distro-specific package. + """ + + print("\n2) As a package with:") + + old_need = self.deps.need + old_optional = self.deps.optional + + self.pdf = False + self.deps.optional = 0 + old_verbose = self.verbose_warn_install + self.verbose_warn_install = 0 + + self.deps.clear_deps() + + self.deps.add_package("python-sphinx", DepManager.PYTHON_MANDATORY) + + cmd = self.get_install() + if cmd: + print(cmd) + + self.deps.need = old_need + self.deps.optional = old_optional + self.verbose_warn_install = old_verbose + + def recommend_sphinx_version(self, virtualenv_cmd): + """ + Provide recommendations for installing or upgrading Sphinx based + on current version. + + The logic here is complex, as it have to deal with different versions: + + - minimal supported version; + - minimal PDF version; + - recommended version. + + It also needs to work fine with both distro's package and + venv/virtualenv + """ + + if self.recommend_python: + print("\nPython version is incompatible with doc build.\n" \ + "Please upgrade it and re-run.\n") + return + + + # Version is OK. Nothing to do. + if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: + return + + if self.latest_avail_ver: + latest_avail_ver = ver_str(self.latest_avail_ver) + + if not self.need_sphinx: + # sphinx-build is present and its version is >= $min_version + + # only recommend enabling a newer virtenv version if makes sense. + if self.latest_avail_ver and self.latest_avail_ver > self.cur_version: + print(f"\nYou may also use the newer Sphinx version {latest_avail_ver} with:") + if f"{self.virtenv_prefix}" in os.getcwd(): + print("\tdeactivate") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + + if self.latest_avail_ver and self.latest_avail_ver >= RECOMMENDED_VERSION: + return + + if not self.virtualenv: + # No sphinx either via package or via virtenv. As we can't + # Compare the versions here, just return, recommending the + # user to install it from the package distro. + if not self.latest_avail_ver or self.latest_avail_ver == (0, 0, 0): + return + + # User doesn't want a virtenv recommendation, but he already + # installed one via virtenv with a newer version. + # So, print commands to enable it + if self.latest_avail_ver > self.cur_version: + print(f"\nYou may also use the Sphinx virtualenv version {latest_avail_ver} with:") + if f"{self.virtenv_prefix}" in os.getcwd(): + print("\tdeactivate") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + print("\n") + else: + if self.need_sphinx: + self.deps.need += 1 + + # Suggest newer versions if current ones are too old + if self.latest_avail_ver and self.latest_avail_ver >= self.min_version: + if self.latest_avail_ver >= RECOMMENDED_VERSION: + print(f"\nNeed to activate Sphinx (version {latest_avail_ver}) on virtualenv with:") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + + # Version is above the minimal required one, but may be + # below the recommended one. So, print warnings/notes + if self.latest_avail_ver < RECOMMENDED_VERSION: + print(f"Warning: It is recommended at least Sphinx version {RECOMMENDED_VERSION}.") + + # At this point, either it needs Sphinx or upgrade is recommended, + # both via pip + + if self.rec_sphinx_upgrade: + if not self.virtualenv: + print("Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n") + else: + print("To upgrade Sphinx, use:\n\n") + else: + print("\nSphinx needs to be installed either:\n1) via pip/pypi with:\n") + + if not virtualenv_cmd: + print(" Currently not possible.\n") + print(" Please upgrade Python to a newer version and run this script again") + else: + print(f"\t{virtualenv_cmd} {self.virtenv_dir}") + print(f"\t. {self.virtenv_dir}/bin/activate") + print(f"\tpip install -r {self.requirement_file}") + self.deactivate_help() + + if self.package_supported: + self.recommend_package() + + print("\n" \ + " Please note that Sphinx currentlys produce false-positive\n" \ + " warnings when the same name is used for more than one type (functions,\n" \ + " structs, enums,...). This is known Sphinx bug. For more details, see:\n" \ + "\thttps://github.com/sphinx-doc/sphinx/pull/8313") + + def check_needs(self): + """ + Main method that checks needed dependencies and provides + recommendations. + """ + self.python_cmd = sys.executable + + # Check if Sphinx is already accessible from current environment + self.check_sphinx(self.conf) + + if self.system_release: + print(f"Detected OS: {self.system_release}.") + else: + print("Unknown OS") + if self.cur_version != (0, 0, 0): + ver = ver_str(self.cur_version) + print(f"Sphinx version: {ver}\n") + + # Check the type of virtual env, depending on Python version + virtualenv_cmd = None + + if sys.version_info < MIN_PYTHON_VERSION: + min_ver = ver_str(MIN_PYTHON_VERSION) + print(f"ERROR: at least python {min_ver} is required to build the kernel docs") + self.need_sphinx = 1 + + self.venv_ver = self.recommend_sphinx_upgrade() + + if self.need_pip: + if sys.version_info < MIN_PYTHON_VERSION: + self.need_pip = False + print("Warning: python version is not supported.") + + else: + virtualenv_cmd = f"{self.python_cmd} -m venv" + self.check_python_module("ensurepip") + + # Check for needed programs/tools + self.check_perl_module("Pod::Usage", DepManager.SYSTEM_MANDATORY) + + self.check_program("make", DepManager.SYSTEM_MANDATORY) + self.check_program("gcc", DepManager.SYSTEM_MANDATORY) + + self.check_program("dot", DepManager.SYSTEM_OPTIONAL) + self.check_program("convert", DepManager.SYSTEM_OPTIONAL) + + self.check_python_module("yaml") + + if self.pdf: + self.check_program("xelatex", DepManager.PDF_MANDATORY) + self.check_program("rsvg-convert", DepManager.PDF_MANDATORY) + self.check_program("latexmk", DepManager.PDF_MANDATORY) + + # Do distro-specific checks and output distro-install commands + cmd = self.get_install() + if cmd: + print(cmd) + + # If distro requires some special instructions, print here. + # Please notice that get_install() needs to be called first. + if self.distro_msg: + print("\n" + self.distro_msg) + + if not self.python_cmd: + if self.need == 1: + sys.exit("Can't build as 1 mandatory dependency is missing") + elif self.need: + sys.exit(f"Can't build as {self.need} mandatory dependencies are missing") + + # Check if sphinx-build is called sphinx-build-3 + if self.need_symlink: + sphinx_path = self.which("sphinx-build-3") + if sphinx_path: + print(f"\tsudo ln -sf {sphinx_path} /usr/bin/sphinx-build\n") + + self.recommend_sphinx_version(virtualenv_cmd) + print("") + + if not self.deps.optional: + print("All optional dependencies are met.") + + if self.deps.need == 1: + sys.exit("Can't build as 1 mandatory dependency is missing") + elif self.deps.need: + sys.exit(f"Can't build as {self.deps.need} mandatory dependencies are missing") + + print("Needed package dependencies are met.") + +DESCRIPTION = """ +Process some flags related to Sphinx installation and documentation build. +""" + + +def main(): + """Main function""" + parser = argparse.ArgumentParser(description=DESCRIPTION) + + parser.add_argument( + "--no-virtualenv", + action="store_false", + dest="virtualenv", + help="Recommend installing Sphinx instead of using a virtualenv", + ) + + parser.add_argument( + "--no-pdf", + action="store_false", + dest="pdf", + help="Don't check for dependencies required to build PDF docs", + ) + + parser.add_argument( + "--version-check", + action="store_true", + dest="version_check", + help="If version is compatible, don't check for missing dependencies", + ) + + args = parser.parse_args() + + checker = SphinxDependencyChecker(args) + + checker.check_python() + checker.check_needs() + +# Call main if not used as module +if __name__ == "__main__": + main() diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py deleted file mode 100755 index 7dfe5c2a6cc2..000000000000 --- a/scripts/sphinx-pre-install.py +++ /dev/null @@ -1,1565 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-or-later -# Copyright (c) 2017-2025 Mauro Carvalho Chehab -# -# pylint: disable=C0103,C0114,C0115,C0116,C0301,C0302 -# pylint: disable=R0902,R0904,R0911,R0912,R0914,R0915,R1705,R1710,E1121 - -# Note: this script requires at least Python 3.6 to run. -# Don't add changes not compatible with it, it is meant to report -# incompatible python versions. - -""" -Dependency checker for Sphinx documentation Kernel build. - -This module provides tools to check for all required dependencies needed to -build documentation using Sphinx, including system packages, Python modules -and LaTeX packages for PDF generation. - -It detect packages for a subset of Linux distributions used by Kernel -maintainers, showing hints and missing dependencies. - -The main class SphinxDependencyChecker handles the dependency checking logic -and provides recommendations for installing missing packages. It supports both -system package installations and Python virtual environments. By default, -system pacage install is recommended. -""" - -import argparse -import os -import re -import subprocess -import sys -from glob import glob - - -def parse_version(version): - """Convert a major.minor.patch version into a tuple""" - return tuple(int(x) for x in version.split(".")) - - -def ver_str(version): - """Returns a version tuple as major.minor.patch""" - - return ".".join([str(x) for x in version]) - - -RECOMMENDED_VERSION = parse_version("3.4.3") -MIN_PYTHON_VERSION = parse_version("3.7") - - -class DepManager: - """ - Manage package dependencies. There are three types of dependencies: - - - System: dependencies required for docs build; - - Python: python dependencies for a native distro Sphinx install; - - PDF: dependencies needed by PDF builds. - - Each dependency can be mandatory or optional. Not installing an optional - dependency won't break the build, but will cause degradation at the - docs output. - """ - - # Internal types of dependencies. Don't use them outside DepManager class. - _SYS_TYPE = 0 - _PHY_TYPE = 1 - _PDF_TYPE = 2 - - # Dependencies visible outside the class. - # The keys are tuple with: (type, is_mandatory flag). - # - # Currently we're not using all optional dep types. Yet, we'll keep all - # possible combinations here. They're not many, and that makes easier - # if later needed and for the name() method below - - SYSTEM_MANDATORY = (_SYS_TYPE, True) - PYTHON_MANDATORY = (_PHY_TYPE, True) - PDF_MANDATORY = (_PDF_TYPE, True) - - SYSTEM_OPTIONAL = (_SYS_TYPE, False) - PYTHON_OPTIONAL = (_PHY_TYPE, False) - PDF_OPTIONAL = (_PDF_TYPE, True) - - def __init__(self, pdf): - """ - Initialize internal vars: - - - missing: missing dependencies list, containing a distro-independent - name for a missing dependency and its type. - - missing_pkg: ancillary dict containing missing dependencies in - distro namespace, organized by type. - - need: total number of needed dependencies. Never cleaned. - - optional: total number of optional dependencies. Never cleaned. - - pdf: Is PDF support enabled? - """ - self.missing = {} - self.missing_pkg = {} - self.need = 0 - self.optional = 0 - self.pdf = pdf - - @staticmethod - def name(dtype): - """ - Ancillary routine to output a warn/error message reporting - missing dependencies. - """ - if dtype[0] == DepManager._SYS_TYPE: - msg = "build" - elif dtype[0] == DepManager._PHY_TYPE: - msg = "Python" - else: - msg = "PDF" - - if dtype[1]: - return f"ERROR: {msg} mandatory deps missing" - else: - return f"Warning: {msg} optional deps missing" - - @staticmethod - def is_optional(dtype): - """Ancillary routine to report if a dependency is optional""" - return not dtype[1] - - @staticmethod - def is_pdf(dtype): - """Ancillary routine to report if a dependency is for PDF generation""" - if dtype[0] == DepManager._PDF_TYPE: - return True - - return False - - def add_package(self, package, dtype): - """ - Add a package at the self.missing() dictionary. - Doesn't update missing_pkg. - """ - is_optional = DepManager.is_optional(dtype) - self.missing[package] = dtype - if is_optional: - self.optional += 1 - else: - self.need += 1 - - def del_package(self, package): - """ - Remove a package at the self.missing() dictionary. - Doesn't update missing_pkg. - """ - if package in self.missing: - del self.missing[package] - - def clear_deps(self): - """ - Clear dependencies without changing needed/optional. - - This is an ackward way to have a separate section to recommend - a package after system main dependencies. - - TODO: rework the logic to prevent needing it. - """ - - self.missing = {} - self.missing_pkg = {} - - def check_missing(self, progs): - """ - Update self.missing_pkg, using progs dict to convert from the - agnostic package name to distro-specific one. - - Returns an string with the packages to be installed, sorted and - with eventual duplicates removed. - """ - - self.missing_pkg = {} - - for prog, dtype in sorted(self.missing.items()): - # At least on some LTS distros like CentOS 7, texlive doesn't - # provide all packages we need. When such distros are - # detected, we have to disable PDF output. - # - # So, we need to ignore the packages that distros would - # need for LaTeX to work - if DepManager.is_pdf(dtype) and not self.pdf: - self.optional -= 1 - continue - - if not dtype in self.missing_pkg: - self.missing_pkg[dtype] = [] - - self.missing_pkg[dtype].append(progs.get(prog, prog)) - - install = [] - for dtype, pkgs in self.missing_pkg.items(): - install += pkgs - - return " ".join(sorted(set(install))) - - def warn_install(self): - """ - Emit warnings/errors related to missing packages. - """ - - output_msg = "" - - for dtype in sorted(self.missing_pkg.keys()): - progs = " ".join(sorted(set(self.missing_pkg[dtype]))) - - try: - name = DepManager.name(dtype) - output_msg += f'{name}:\t{progs}\n' - except KeyError: - raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") - - if output_msg: - print(f"\n{output_msg}") - -class AncillaryMethods: - """ - Ancillary methods that checks for missing dependencies for different - types of types, like binaries, python modules, rpm deps, etc. - """ - - @staticmethod - def which(prog): - """ - Our own implementation of which(). We could instead use - shutil.which(), but this function is simple enough. - Probably faster to use this implementation than to import shutil. - """ - for path in os.environ.get("PATH", "").split(":"): - full_path = os.path.join(path, prog) - if os.access(full_path, os.X_OK): - return full_path - - return None - - @staticmethod - def get_python_version(cmd): - """ - Get python version from a Python binary. As we need to detect if - are out there newer python binaries, we can't rely on sys.release here. - """ - - result = SphinxDependencyChecker.run([cmd, "--version"], - capture_output=True, text=True) - version = result.stdout.strip() - - match = re.search(r"(\d+\.\d+\.\d+)", version) - if match: - return parse_version(match.group(1)) - - print(f"Can't parse version {version}") - return (0, 0, 0) - - @staticmethod - def find_python(): - """ - Detect if are out there any python 3.xy version newer than the - current one. - - Note: this routine is limited to up to 2 digits for python3. We - may need to update it one day, hopefully on a distant future. - """ - patterns = [ - "python3.[0-9]", - "python3.[0-9][0-9]", - ] - - # Seek for a python binary newer than MIN_PYTHON_VERSION - for path in os.getenv("PATH", "").split(":"): - for pattern in patterns: - for cmd in glob(os.path.join(path, pattern)): - if os.path.isfile(cmd) and os.access(cmd, os.X_OK): - version = SphinxDependencyChecker.get_python_version(cmd) - if version >= MIN_PYTHON_VERSION: - return cmd - - @staticmethod - def check_python(): - """ - Check if the current python binary satisfies our minimal requirement - for Sphinx build. If not, re-run with a newer version if found. - """ - cur_ver = sys.version_info[:3] - if cur_ver >= MIN_PYTHON_VERSION: - ver = ver_str(cur_ver) - print(f"Python version: {ver}") - - # This could be useful for debugging purposes - if SphinxDependencyChecker.which("docutils"): - result = SphinxDependencyChecker.run(["docutils", "--version"], - capture_output=True, text=True) - ver = result.stdout.strip() - match = re.search(r"(\d+\.\d+\.\d+)", ver) - if match: - ver = match.group(1) - - print(f"Docutils version: {ver}") - - return - - python_ver = ver_str(cur_ver) - - new_python_cmd = SphinxDependencyChecker.find_python() - if not new_python_cmd: - print(f"ERROR: Python version {python_ver} is not spported anymore\n") - print(" Can't find a new version. This script may fail") - return - - # Restart script using the newer version - script_path = os.path.abspath(sys.argv[0]) - args = [new_python_cmd, script_path] + sys.argv[1:] - - print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") - - try: - os.execv(new_python_cmd, args) - except OSError as e: - sys.exit(f"Failed to restart with {new_python_cmd}: {e}") - - @staticmethod - def run(*args, **kwargs): - """ - Excecute a command, hiding its output by default. - Preserve comatibility with older Python versions. - """ - - capture_output = kwargs.pop('capture_output', False) - - if capture_output: - if 'stdout' not in kwargs: - kwargs['stdout'] = subprocess.PIPE - if 'stderr' not in kwargs: - kwargs['stderr'] = subprocess.PIPE - else: - if 'stdout' not in kwargs: - kwargs['stdout'] = subprocess.DEVNULL - if 'stderr' not in kwargs: - kwargs['stderr'] = subprocess.DEVNULL - - # Don't break with older Python versions - if 'text' in kwargs and sys.version_info < (3, 7): - kwargs['universal_newlines'] = kwargs.pop('text') - - return subprocess.run(*args, **kwargs) - -class MissingCheckers(AncillaryMethods): - """ - Contains some ancillary checkers for different types of binaries and - package managers. - """ - - def __init__(self, args, texlive): - """ - Initialize its internal variables - """ - self.pdf = args.pdf - self.virtualenv = args.virtualenv - self.version_check = args.version_check - self.texlive = texlive - - self.min_version = (0, 0, 0) - self.cur_version = (0, 0, 0) - - self.deps = DepManager(self.pdf) - - self.need_symlink = 0 - self.need_sphinx = 0 - - self.verbose_warn_install = 1 - - self.virtenv_dir = "" - self.install = "" - self.python_cmd = "" - - self.virtenv_prefix = ["sphinx_", "Sphinx_" ] - - def check_missing_file(self, files, package, dtype): - """ - Does the file exists? If not, add it to missing dependencies. - """ - for f in files: - if os.path.exists(f): - return - self.deps.add_package(package, dtype) - - def check_program(self, prog, dtype): - """ - Does the program exists and it is at the PATH? - If not, add it to missing dependencies. - """ - found = self.which(prog) - if found: - return found - - self.deps.add_package(prog, dtype) - - return None - - def check_perl_module(self, prog, dtype): - """ - Does perl have a dependency? Is it available? - If not, add it to missing dependencies. - - Right now, we still need Perl for doc build, as it is required - by some tools called at docs or kernel build time, like: - - scripts/documentation-file-ref-check - - Also, checkpatch is on Perl. - """ - - # While testing with lxc download template, one of the - # distros (Oracle) didn't have perl - nor even an option to install - # before installing oraclelinux-release-el9 package. - # - # Check it before running an error. If perl is not there, - # add it as a mandatory package, as some parts of the doc builder - # needs it. - if not self.which("perl"): - self.deps.add_package("perl", DepManager.SYSTEM_MANDATORY) - self.deps.add_package(prog, dtype) - return - - try: - self.run(["perl", f"-M{prog}", "-e", "1"], check=True) - except subprocess.CalledProcessError: - self.deps.add_package(prog, dtype) - - def check_python_module(self, module, is_optional=False): - """ - Does a python module exists outside venv? If not, add it to missing - dependencies. - """ - if is_optional: - dtype = DepManager.PYTHON_OPTIONAL - else: - dtype = DepManager.PYTHON_MANDATORY - - try: - self.run([self.python_cmd, "-c", f"import {module}"], check=True) - except subprocess.CalledProcessError: - self.deps.add_package(module, dtype) - - def check_rpm_missing(self, pkgs, dtype): - """ - Does a rpm package exists? If not, add it to missing dependencies. - """ - for prog in pkgs: - try: - self.run(["rpm", "-q", prog], check=True) - except subprocess.CalledProcessError: - self.deps.add_package(prog, dtype) - - def check_pacman_missing(self, pkgs, dtype): - """ - Does a pacman package exists? If not, add it to missing dependencies. - """ - for prog in pkgs: - try: - self.run(["pacman", "-Q", prog], check=True) - except subprocess.CalledProcessError: - self.deps.add_package(prog, dtype) - - def check_missing_tex(self, is_optional=False): - """ - Does a LaTeX package exists? If not, add it to missing dependencies. - """ - if is_optional: - dtype = DepManager.PDF_OPTIONAL - else: - dtype = DepManager.PDF_MANDATORY - - kpsewhich = self.which("kpsewhich") - for prog, package in self.texlive.items(): - - # If kpsewhich is not there, just add it to deps - if not kpsewhich: - self.deps.add_package(package, dtype) - continue - - # Check if the package is needed - try: - result = self.run( - [kpsewhich, prog], stdout=subprocess.PIPE, text=True, check=True - ) - - # Didn't find. Add it - if not result.stdout.strip(): - self.deps.add_package(package, dtype) - - except subprocess.CalledProcessError: - # kpsewhich returned an error. Add it, just in case - self.deps.add_package(package, dtype) - - def get_sphinx_fname(self): - """ - Gets the binary filename for sphinx-build. - """ - if "SPHINXBUILD" in os.environ: - return os.environ["SPHINXBUILD"] - - fname = "sphinx-build" - if self.which(fname): - return fname - - fname = "sphinx-build-3" - if self.which(fname): - self.need_symlink = 1 - return fname - - return "" - - def get_sphinx_version(self, cmd): - """ - Gets sphinx-build version. - """ - try: - result = self.run([cmd, "--version"], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, check=True) - except (subprocess.CalledProcessError, FileNotFoundError): - return None - - for line in result.stdout.split("\n"): - match = re.match(r"^sphinx-build\s+([\d\.]+)(?:\+(?:/[\da-f]+)|b\d+)?\s*$", line) - if match: - return parse_version(match.group(1)) - - match = re.match(r"^Sphinx.*\s+([\d\.]+)\s*$", line) - if match: - return parse_version(match.group(1)) - - def check_sphinx(self, conf): - """ - Checks Sphinx minimal requirements - """ - try: - with open(conf, "r", encoding="utf-8") as f: - for line in f: - match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line) - if match: - self.min_version = parse_version(match.group(1)) - break - except IOError: - sys.exit(f"Can't open {conf}") - - if not self.min_version: - sys.exit(f"Can't get needs_sphinx version from {conf}") - - self.virtenv_dir = self.virtenv_prefix[0] + "latest" - - sphinx = self.get_sphinx_fname() - if not sphinx: - self.need_sphinx = 1 - return - - self.cur_version = self.get_sphinx_version(sphinx) - if not self.cur_version: - sys.exit(f"{sphinx} didn't return its version") - - if self.cur_version < self.min_version: - curver = ver_str(self.cur_version) - minver = ver_str(self.min_version) - - print(f"ERROR: Sphinx version is {curver}. It should be >= {minver}") - self.need_sphinx = 1 - return - - # On version check mode, just assume Sphinx has all mandatory deps - if self.version_check and self.cur_version >= RECOMMENDED_VERSION: - sys.exit(0) - - def catcheck(self, filename): - """ - Reads a file if it exists, returning as string. - If not found, returns an empty string. - """ - if os.path.exists(filename): - with open(filename, "r", encoding="utf-8") as f: - return f.read().strip() - return "" - - def get_system_release(self): - """ - Determine the system type. There's no unique way that would work - with all distros with a minimal package install. So, several - methods are used here. - - By default, it will use lsb_release function. If not available, it will - fail back to reading the known different places where the distro name - is stored. - - Several modern distros now have /etc/os-release, which usually have - a decent coverage. - """ - - system_release = "" - - if self.which("lsb_release"): - result = self.run(["lsb_release", "-d"], capture_output=True, text=True) - system_release = result.stdout.replace("Description:", "").strip() - - release_files = [ - "/etc/system-release", - "/etc/redhat-release", - "/etc/lsb-release", - "/etc/gentoo-release", - ] - - if not system_release: - for f in release_files: - system_release = self.catcheck(f) - if system_release: - break - - # This seems more common than LSB these days - if not system_release: - os_var = {} - try: - with open("/etc/os-release", "r", encoding="utf-8") as f: - for line in f: - match = re.match(r"^([\w\d\_]+)=\"?([^\"]*)\"?\n", line) - if match: - os_var[match.group(1)] = match.group(2) - - system_release = os_var.get("NAME", "") - if "VERSION_ID" in os_var: - system_release += " " + os_var["VERSION_ID"] - elif "VERSION" in os_var: - system_release += " " + os_var["VERSION"] - except IOError: - pass - - if not system_release: - system_release = self.catcheck("/etc/issue") - - system_release = system_release.strip() - - return system_release - -class SphinxDependencyChecker(MissingCheckers): - """ - Main class for checking Sphinx documentation build dependencies. - - - Check for missing system packages; - - Check for missing Python modules; - - Check for missing LaTeX packages needed by PDF generation; - - Propose Sphinx install via Python Virtual environment; - - Propose Sphinx install via distro-specific package install. - """ - def __init__(self, args): - """Initialize checker variables""" - - # List of required texlive packages on Fedora and OpenSuse - texlive = { - "amsfonts.sty": "texlive-amsfonts", - "amsmath.sty": "texlive-amsmath", - "amssymb.sty": "texlive-amsfonts", - "amsthm.sty": "texlive-amscls", - "anyfontsize.sty": "texlive-anyfontsize", - "atbegshi.sty": "texlive-oberdiek", - "bm.sty": "texlive-tools", - "capt-of.sty": "texlive-capt-of", - "cmap.sty": "texlive-cmap", - "ctexhook.sty": "texlive-ctex", - "ecrm1000.tfm": "texlive-ec", - "eqparbox.sty": "texlive-eqparbox", - "eu1enc.def": "texlive-euenc", - "fancybox.sty": "texlive-fancybox", - "fancyvrb.sty": "texlive-fancyvrb", - "float.sty": "texlive-float", - "fncychap.sty": "texlive-fncychap", - "footnote.sty": "texlive-mdwtools", - "framed.sty": "texlive-framed", - "luatex85.sty": "texlive-luatex85", - "multirow.sty": "texlive-multirow", - "needspace.sty": "texlive-needspace", - "palatino.sty": "texlive-psnfss", - "parskip.sty": "texlive-parskip", - "polyglossia.sty": "texlive-polyglossia", - "tabulary.sty": "texlive-tabulary", - "threeparttable.sty": "texlive-threeparttable", - "titlesec.sty": "texlive-titlesec", - "ucs.sty": "texlive-ucs", - "upquote.sty": "texlive-upquote", - "wrapfig.sty": "texlive-wrapfig", - } - - super().__init__(args, texlive) - - self.need_pip = 0 - self.rec_sphinx_upgrade = 0 - - self.system_release = self.get_system_release() - self.activate_cmd = "" - - # Some distros may not have a Sphinx shipped package compatible with - # our minimal requirements - self.package_supported = True - - # Recommend a new python version - self.recommend_python = None - - # Certain hints are meant to be shown only once - self.distro_msg = None - - self.latest_avail_ver = (0, 0, 0) - self.venv_ver = (0, 0, 0) - - prefix = os.environ.get("srctree", ".") + "/" - - self.conf = prefix + "Documentation/conf.py" - self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" - - def get_install_progs(self, progs, cmd, extra=None): - """ - Check for missing dependencies using the provided program mapping. - - The actual distro-specific programs are mapped via progs argument. - """ - install = self.deps.check_missing(progs) - - if self.verbose_warn_install: - self.deps.warn_install() - - if not install: - return - - if cmd: - if self.verbose_warn_install: - msg = "You should run:" - else: - msg = "" - - if extra: - msg += "\n\t" + extra.replace("\n", "\n\t") - - return(msg + "\n\tsudo " + cmd + " " + install) - - return None - - # - # Distro-specific hints methods - # - - def give_debian_hints(self): - """ - Provide package installation hints for Debian-based distros. - """ - progs = { - "Pod::Usage": "perl-modules", - "convert": "imagemagick", - "dot": "graphviz", - "ensurepip": "python3-venv", - "python-sphinx": "python3-sphinx", - "rsvg-convert": "librsvg2-bin", - "virtualenv": "virtualenv", - "xelatex": "texlive-xetex", - "yaml": "python3-yaml", - } - - if self.pdf: - pdf_pkgs = { - "texlive-lang-chinese": [ - "/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty", - ], - "fonts-dejavu": [ - "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", - ], - "fonts-noto-cjk": [ - "/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc", - ], - } - - for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, DepManager.PDF_MANDATORY) - - self.check_program("dvipng", DepManager.PDF_MANDATORY) - - return self.get_install_progs(progs, "apt-get install") - - def give_redhat_hints(self): - """ - Provide package installation hints for RedHat-based distros - (Fedora, RHEL and RHEL-based variants). - """ - progs = { - "Pod::Usage": "perl-Pod-Usage", - "convert": "ImageMagick", - "dot": "graphviz", - "python-sphinx": "python3-sphinx", - "rsvg-convert": "librsvg2-tools", - "virtualenv": "python3-virtualenv", - "xelatex": "texlive-xetex-bin", - "yaml": "python3-pyyaml", - } - - fedora_tex_pkgs = [ - "dejavu-sans-fonts", - "dejavu-sans-mono-fonts", - "dejavu-serif-fonts", - "texlive-collection-fontsrecommended", - "texlive-collection-latex", - "texlive-xecjk", - ] - - fedora = False - rel = None - - match = re.search(r"(release|Linux)\s+(\d+)", self.system_release) - if match: - rel = int(match.group(2)) - - if not rel: - print("Couldn't identify release number") - noto_sans_redhat = None - self.pdf = False - elif re.search("Fedora", self.system_release): - # Fedora 38 and upper use this CJK font - - noto_sans_redhat = "google-noto-sans-cjk-fonts" - fedora = True - else: - # Almalinux, CentOS, RHEL, ... - - # at least up to version 9 (and Fedora < 38), that's the CJK font - noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts" - - progs["virtualenv"] = "python-virtualenv" - - if not rel or rel < 8: - print("ERROR: Distro not supported. Too old?") - return - - # RHEL 8 uses Python 3.6, which is not compatible with - # the build system anymore. Suggest Python 3.11 - if rel == 8: - self.deps.add_package("python39", DepManager.SYSTEM_MANDATORY) - self.recommend_python = True - - if not self.distro_msg: - self.distro_msg = \ - "Note: RHEL-based distros typically require extra repositories.\n" \ - "For most, enabling epel and crb are enough:\n" \ - "\tsudo dnf install -y epel-release\n" \ - "\tsudo dnf config-manager --set-enabled crb\n" \ - "Yet, some may have other required repositories. Those commands could be useful:\n" \ - "\tsudo dnf repolist all\n" \ - "\tsudo dnf repoquery --available --info \n" \ - "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want" - - if self.pdf: - pdf_pkgs = [ - "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc", - ] - - self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepManager.PDF_MANDATORY) - - self.check_rpm_missing(fedora_tex_pkgs, DepManager.PDF_MANDATORY) - - self.check_missing_tex(DepManager.PDF_MANDATORY) - - # There's no texlive-ctex on RHEL 8 repositories. This will - # likely affect CJK pdf build only. - if not fedora and rel == 8: - self.deps.del_package("texlive-ctex") - - return self.get_install_progs(progs, "dnf install") - - def give_opensuse_hints(self): - """ - Provide package installation hints for openSUSE-based distros - (Leap and Tumbleweed). - """ - progs = { - "Pod::Usage": "perl-Pod-Usage", - "convert": "ImageMagick", - "dot": "graphviz", - "python-sphinx": "python3-sphinx", - "virtualenv": "python3-virtualenv", - "xelatex": "texlive-xetex-bin", - "yaml": "python3-pyyaml", - } - - suse_tex_pkgs = [ - "texlive-babel-english", - "texlive-caption", - "texlive-colortbl", - "texlive-courier", - "texlive-dvips", - "texlive-helvetic", - "texlive-makeindex", - "texlive-metafont", - "texlive-metapost", - "texlive-palatino", - "texlive-preview", - "texlive-times", - "texlive-zapfchan", - "texlive-zapfding", - ] - - progs["latexmk"] = "texlive-latexmk-bin" - - match = re.search(r"(Leap)\s+(\d+).(\d)", self.system_release) - if match: - rel = int(match.group(2)) - - # Leap 15.x uses Python 3.6, which is not compatible with - # the build system anymore. Suggest Python 3.11 - if rel == 15: - if not self.which(self.python_cmd): - self.recommend_python = True - self.deps.add_package(self.python_cmd, DepManager.SYSTEM_MANDATORY) - - progs.update({ - "python-sphinx": "python311-Sphinx", - "virtualenv": "python311-virtualenv", - "yaml": "python311-PyYAML", - }) - else: - # Tumbleweed defaults to Python 3.11 - - progs.update({ - "python-sphinx": "python313-Sphinx", - "virtualenv": "python313-virtualenv", - "yaml": "python313-PyYAML", - }) - - # FIXME: add support for installing CJK fonts - # - # I tried hard, but was unable to find a way to install - # "Noto Sans CJK SC" on openSUSE - - if self.pdf: - self.check_rpm_missing(suse_tex_pkgs, DepManager.PDF_MANDATORY) - if self.pdf: - self.check_missing_tex() - - return self.get_install_progs(progs, "zypper install --no-recommends") - - def give_mageia_hints(self): - """ - Provide package installation hints for Mageia and OpenMandriva. - """ - progs = { - "Pod::Usage": "perl-Pod-Usage", - "convert": "ImageMagick", - "dot": "graphviz", - "python-sphinx": "python3-sphinx", - "rsvg-convert": "librsvg2", - "virtualenv": "python3-virtualenv", - "xelatex": "texlive", - "yaml": "python3-yaml", - } - - tex_pkgs = [ - "texlive-fontsextra", - ] - - if re.search(r"OpenMandriva", self.system_release): - packager_cmd = "dnf install" - noto_sans = "noto-sans-cjk-fonts" - tex_pkgs = ["texlive-collection-fontsextra"] - - # Tested on OpenMandriva Lx 4.3 - progs["convert"] = "imagemagick" - progs["yaml"] = "python-pyyaml" - - else: - packager_cmd = "urpmi" - noto_sans = "google-noto-sans-cjk-ttc-fonts" - - progs["latexmk"] = "texlive-collection-basic" - - if self.pdf: - pdf_pkgs = [ - "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/TTF/NotoSans-Regular.ttf", - ] - - self.check_missing_file(pdf_pkgs, noto_sans, DepManager.PDF_MANDATORY) - self.check_rpm_missing(tex_pkgs, DepManager.PDF_MANDATORY) - - return self.get_install_progs(progs, packager_cmd) - - def give_arch_linux_hints(self): - """ - Provide package installation hints for ArchLinux. - """ - progs = { - "convert": "imagemagick", - "dot": "graphviz", - "latexmk": "texlive-core", - "rsvg-convert": "extra/librsvg", - "virtualenv": "python-virtualenv", - "xelatex": "texlive-xetex", - "yaml": "python-yaml", - } - - archlinux_tex_pkgs = [ - "texlive-core", - "texlive-latexextra", - "ttf-dejavu", - ] - - if self.pdf: - self.check_pacman_missing(archlinux_tex_pkgs, - DepManager.PDF_MANDATORY) - - self.check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], - "noto-fonts-cjk", - DepManager.PDF_MANDATORY) - - - return self.get_install_progs(progs, "pacman -S") - - def give_gentoo_hints(self): - """ - Provide package installation hints for Gentoo. - """ - progs = { - "convert": "media-gfx/imagemagick", - "dot": "media-gfx/graphviz", - "rsvg-convert": "gnome-base/librsvg", - "virtualenv": "dev-python/virtualenv", - "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", - "yaml": "dev-python/pyyaml", - "python-sphinx": "dev-python/sphinx", - } - - if self.pdf: - pdf_pkgs = { - "media-fonts/dejavu": [ - "/usr/share/fonts/dejavu/DejaVuSans.ttf", - ], - "media-fonts/noto-cjk": [ - "/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", - "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc", - ], - } - for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, DepManager.PDF_MANDATORY) - - # Handling dependencies is a nightmare, as Gentoo refuses to emerge - # some packages if there's no package.use file describing them. - # To make it worse, compilation flags shall also be present there - # for some packages. If USE is not perfect, error/warning messages - # like those are shown: - # - # !!! The following binary packages have been ignored due to non matching USE: - # - # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_13 qt6 svg - # =media-gfx/graphviz-12.2.1-r1 X pdf python_single_target_python3_12 -python_single_target_python3_13 qt6 svg - # =media-gfx/graphviz-12.2.1-r1 X pdf qt6 svg - # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 qt6 svg - # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 python_single_target_python3_12 -python_single_target_python3_13 qt6 svg - # =media-fonts/noto-cjk-20190416 X - # =app-text/texlive-core-2024-r1 X cjk -xetex - # =app-text/texlive-core-2024-r1 X -xetex - # =app-text/texlive-core-2024-r1 -xetex - # =dev-libs/zziplib-0.13.79-r1 sdl - # - # And will ignore such packages, installing the remaining ones. That - # affects mostly the image extension and PDF generation. - - # Package dependencies and the minimal needed args: - portages = { - "graphviz": "media-gfx/graphviz", - "imagemagick": "media-gfx/imagemagick", - "media-libs": "media-libs/harfbuzz icu", - "media-fonts": "media-fonts/noto-cjk", - "texlive": "app-text/texlive-core xetex", - "zziblib": "dev-libs/zziplib sdl", - } - - extra_cmds = "" - if not self.distro_msg: - self.distro_msg = "Note: Gentoo requires package.use to be adjusted before emerging packages" - - use_base = "/etc/portage/package.use" - files = glob(f"{use_base}/*") - - for fname, portage in portages.items(): - install = False - - while install is False: - if not files: - # No files under package.usage. Install all - install = True - break - - args = portage.split(" ") - - name = args.pop(0) - - cmd = ["grep", "-l", "-E", rf"^{name}\b" ] + files - result = self.run(cmd, stdout=subprocess.PIPE, text=True) - if result.returncode or not result.stdout.strip(): - # File containing portage name not found - install = True - break - - # Ensure that needed USE flags are present - if args: - match_fname = result.stdout.strip() - with open(match_fname, 'r', encoding='utf8', - errors='backslashreplace') as fp: - for line in fp: - for arg in args: - if arg.startswith("-"): - continue - - if not re.search(rf"\s*{arg}\b", line): - # Needed file argument not found - install = True - break - - # Everything looks ok, don't install - break - - # emit a code to setup missing USE - if install: - extra_cmds += (f"sudo su -c 'echo \"{portage}\" > {use_base}/{fname}'\n") - - # Now, we can use emerge and let it respect USE - return self.get_install_progs(progs, - "emerge --ask --changed-use --binpkg-respect-use=y", - extra_cmds) - - def get_install(self): - """ - OS-specific hints logic. Seeks for a hinter. If found, use it to - provide package-manager specific install commands. - - Otherwise, outputs install instructions for the meta-packages. - - Returns a string with the command to be executed to install the - the needed packages, if distro found. Otherwise, return just a - list of packages that require installation. - """ - os_hints = { - re.compile("Red Hat Enterprise Linux"): self.give_redhat_hints, - re.compile("Fedora"): self.give_redhat_hints, - re.compile("AlmaLinux"): self.give_redhat_hints, - re.compile("Amazon Linux"): self.give_redhat_hints, - re.compile("CentOS"): self.give_redhat_hints, - re.compile("openEuler"): self.give_redhat_hints, - re.compile("Oracle Linux Server"): self.give_redhat_hints, - re.compile("Rocky Linux"): self.give_redhat_hints, - re.compile("Springdale Open Enterprise"): self.give_redhat_hints, - - re.compile("Ubuntu"): self.give_debian_hints, - re.compile("Debian"): self.give_debian_hints, - re.compile("Devuan"): self.give_debian_hints, - re.compile("Kali"): self.give_debian_hints, - re.compile("Mint"): self.give_debian_hints, - - re.compile("openSUSE"): self.give_opensuse_hints, - - re.compile("Mageia"): self.give_mageia_hints, - re.compile("OpenMandriva"): self.give_mageia_hints, - - re.compile("Arch Linux"): self.give_arch_linux_hints, - re.compile("Gentoo"): self.give_gentoo_hints, - } - - # If the OS is detected, use per-OS hint logic - for regex, os_hint in os_hints.items(): - if regex.search(self.system_release): - return os_hint() - - # - # Fall-back to generic hint code for other distros - # That's far from ideal, specially for LaTeX dependencies. - # - progs = {"sphinx-build": "sphinx"} - if self.pdf: - self.check_missing_tex() - - self.distro_msg = \ - f"I don't know distro {self.system_release}.\n" \ - "So, I can't provide you a hint with the install procedure.\n" \ - "There are likely missing dependencies.\n" - - return self.get_install_progs(progs, None) - - # - # Common dependencies - # - def deactivate_help(self): - """ - Print a helper message to disable a virtual environment. - """ - - print("\n If you want to exit the virtualenv, you can use:") - print("\tdeactivate") - - def get_virtenv(self): - """ - Give a hint about how to activate an already-existing virtual - environment containing sphinx-build. - - Returns a tuble with (activate_cmd_path, sphinx_version) with - the newest available virtual env. - """ - - cwd = os.getcwd() - - activates = [] - - # Add all sphinx prefixes with possible version numbers - for p in self.virtenv_prefix: - activates += glob(f"{cwd}/{p}[0-9]*/bin/activate") - - activates.sort(reverse=True, key=str.lower) - - # Place sphinx_latest first, if it exists - for p in self.virtenv_prefix: - activates = glob(f"{cwd}/{p}*latest/bin/activate") + activates - - ver = (0, 0, 0) - for f in activates: - # Discard too old Sphinx virtual environments - match = re.search(r"(\d+)\.(\d+)\.(\d+)", f) - if match: - ver = (int(match.group(1)), int(match.group(2)), int(match.group(3))) - - if ver < self.min_version: - continue - - sphinx_cmd = f.replace("activate", "sphinx-build") - if not os.path.isfile(sphinx_cmd): - continue - - ver = self.get_sphinx_version(sphinx_cmd) - - if not ver: - venv_dir = f.replace("/bin/activate", "") - print(f"Warning: virtual environment {venv_dir} is not working.\n" \ - "Python version upgrade? Remove it with:\n\n" \ - "\trm -rf {venv_dir}\n\n") - else: - if self.need_sphinx and ver >= self.min_version: - return (f, ver) - elif parse_version(ver) > self.cur_version: - return (f, ver) - - return ("", ver) - - def recommend_sphinx_upgrade(self): - """ - Check if Sphinx needs to be upgraded. - - Returns a tuple with the higest available Sphinx version if found. - Otherwise, returns None to indicate either that no upgrade is needed - or no venv was found. - """ - - # Avoid running sphinx-builds from venv if cur_version is good - if self.cur_version and self.cur_version >= RECOMMENDED_VERSION: - self.latest_avail_ver = self.cur_version - return None - - # Get the highest version from sphinx_*/bin/sphinx-build and the - # corresponding command to activate the venv/virtenv - self.activate_cmd, self.venv_ver = self.get_virtenv() - - # Store the highest version from Sphinx existing virtualenvs - if self.activate_cmd and self.venv_ver > self.cur_version: - self.latest_avail_ver = self.venv_ver - else: - if self.cur_version: - self.latest_avail_ver = self.cur_version - else: - self.latest_avail_ver = (0, 0, 0) - - # As we don't know package version of Sphinx, and there's no - # virtual environments, don't check if upgrades are needed - if not self.virtualenv: - if not self.latest_avail_ver: - return None - - return self.latest_avail_ver - - # Either there are already a virtual env or a new one should be created - self.need_pip = 1 - - if not self.latest_avail_ver: - return None - - # Return if the reason is due to an upgrade or not - if self.latest_avail_ver != (0, 0, 0): - if self.latest_avail_ver < RECOMMENDED_VERSION: - self.rec_sphinx_upgrade = 1 - - return self.latest_avail_ver - - def recommend_package(self): - """ - Recommend installing Sphinx as a distro-specific package. - """ - - print("\n2) As a package with:") - - old_need = self.deps.need - old_optional = self.deps.optional - - self.pdf = False - self.deps.optional = 0 - old_verbose = self.verbose_warn_install - self.verbose_warn_install = 0 - - self.deps.clear_deps() - - self.deps.add_package("python-sphinx", DepManager.PYTHON_MANDATORY) - - cmd = self.get_install() - if cmd: - print(cmd) - - self.deps.need = old_need - self.deps.optional = old_optional - self.verbose_warn_install = old_verbose - - def recommend_sphinx_version(self, virtualenv_cmd): - """ - Provide recommendations for installing or upgrading Sphinx based - on current version. - - The logic here is complex, as it have to deal with different versions: - - - minimal supported version; - - minimal PDF version; - - recommended version. - - It also needs to work fine with both distro's package and - venv/virtualenv - """ - - if self.recommend_python: - print("\nPython version is incompatible with doc build.\n" \ - "Please upgrade it and re-run.\n") - return - - - # Version is OK. Nothing to do. - if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: - return - - if self.latest_avail_ver: - latest_avail_ver = ver_str(self.latest_avail_ver) - - if not self.need_sphinx: - # sphinx-build is present and its version is >= $min_version - - # only recommend enabling a newer virtenv version if makes sense. - if self.latest_avail_ver and self.latest_avail_ver > self.cur_version: - print(f"\nYou may also use the newer Sphinx version {latest_avail_ver} with:") - if f"{self.virtenv_prefix}" in os.getcwd(): - print("\tdeactivate") - print(f"\t. {self.activate_cmd}") - self.deactivate_help() - return - - if self.latest_avail_ver and self.latest_avail_ver >= RECOMMENDED_VERSION: - return - - if not self.virtualenv: - # No sphinx either via package or via virtenv. As we can't - # Compare the versions here, just return, recommending the - # user to install it from the package distro. - if not self.latest_avail_ver or self.latest_avail_ver == (0, 0, 0): - return - - # User doesn't want a virtenv recommendation, but he already - # installed one via virtenv with a newer version. - # So, print commands to enable it - if self.latest_avail_ver > self.cur_version: - print(f"\nYou may also use the Sphinx virtualenv version {latest_avail_ver} with:") - if f"{self.virtenv_prefix}" in os.getcwd(): - print("\tdeactivate") - print(f"\t. {self.activate_cmd}") - self.deactivate_help() - return - print("\n") - else: - if self.need_sphinx: - self.deps.need += 1 - - # Suggest newer versions if current ones are too old - if self.latest_avail_ver and self.latest_avail_ver >= self.min_version: - if self.latest_avail_ver >= RECOMMENDED_VERSION: - print(f"\nNeed to activate Sphinx (version {latest_avail_ver}) on virtualenv with:") - print(f"\t. {self.activate_cmd}") - self.deactivate_help() - return - - # Version is above the minimal required one, but may be - # below the recommended one. So, print warnings/notes - if self.latest_avail_ver < RECOMMENDED_VERSION: - print(f"Warning: It is recommended at least Sphinx version {RECOMMENDED_VERSION}.") - - # At this point, either it needs Sphinx or upgrade is recommended, - # both via pip - - if self.rec_sphinx_upgrade: - if not self.virtualenv: - print("Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n") - else: - print("To upgrade Sphinx, use:\n\n") - else: - print("\nSphinx needs to be installed either:\n1) via pip/pypi with:\n") - - if not virtualenv_cmd: - print(" Currently not possible.\n") - print(" Please upgrade Python to a newer version and run this script again") - else: - print(f"\t{virtualenv_cmd} {self.virtenv_dir}") - print(f"\t. {self.virtenv_dir}/bin/activate") - print(f"\tpip install -r {self.requirement_file}") - self.deactivate_help() - - if self.package_supported: - self.recommend_package() - - print("\n" \ - " Please note that Sphinx currentlys produce false-positive\n" \ - " warnings when the same name is used for more than one type (functions,\n" \ - " structs, enums,...). This is known Sphinx bug. For more details, see:\n" \ - "\thttps://github.com/sphinx-doc/sphinx/pull/8313") - - def check_needs(self): - """ - Main method that checks needed dependencies and provides - recommendations. - """ - self.python_cmd = sys.executable - - # Check if Sphinx is already accessible from current environment - self.check_sphinx(self.conf) - - if self.system_release: - print(f"Detected OS: {self.system_release}.") - else: - print("Unknown OS") - if self.cur_version != (0, 0, 0): - ver = ver_str(self.cur_version) - print(f"Sphinx version: {ver}\n") - - # Check the type of virtual env, depending on Python version - virtualenv_cmd = None - - if sys.version_info < MIN_PYTHON_VERSION: - min_ver = ver_str(MIN_PYTHON_VERSION) - print(f"ERROR: at least python {min_ver} is required to build the kernel docs") - self.need_sphinx = 1 - - self.venv_ver = self.recommend_sphinx_upgrade() - - if self.need_pip: - if sys.version_info < MIN_PYTHON_VERSION: - self.need_pip = False - print("Warning: python version is not supported.") - - else: - virtualenv_cmd = f"{self.python_cmd} -m venv" - self.check_python_module("ensurepip") - - # Check for needed programs/tools - self.check_perl_module("Pod::Usage", DepManager.SYSTEM_MANDATORY) - - self.check_program("make", DepManager.SYSTEM_MANDATORY) - self.check_program("gcc", DepManager.SYSTEM_MANDATORY) - - self.check_program("dot", DepManager.SYSTEM_OPTIONAL) - self.check_program("convert", DepManager.SYSTEM_OPTIONAL) - - self.check_python_module("yaml") - - if self.pdf: - self.check_program("xelatex", DepManager.PDF_MANDATORY) - self.check_program("rsvg-convert", DepManager.PDF_MANDATORY) - self.check_program("latexmk", DepManager.PDF_MANDATORY) - - # Do distro-specific checks and output distro-install commands - cmd = self.get_install() - if cmd: - print(cmd) - - # If distro requires some special instructions, print here. - # Please notice that get_install() needs to be called first. - if self.distro_msg: - print("\n" + self.distro_msg) - - if not self.python_cmd: - if self.need == 1: - sys.exit("Can't build as 1 mandatory dependency is missing") - elif self.need: - sys.exit(f"Can't build as {self.need} mandatory dependencies are missing") - - # Check if sphinx-build is called sphinx-build-3 - if self.need_symlink: - sphinx_path = self.which("sphinx-build-3") - if sphinx_path: - print(f"\tsudo ln -sf {sphinx_path} /usr/bin/sphinx-build\n") - - self.recommend_sphinx_version(virtualenv_cmd) - print("") - - if not self.deps.optional: - print("All optional dependencies are met.") - - if self.deps.need == 1: - sys.exit("Can't build as 1 mandatory dependency is missing") - elif self.deps.need: - sys.exit(f"Can't build as {self.deps.need} mandatory dependencies are missing") - - print("Needed package dependencies are met.") - -DESCRIPTION = """ -Process some flags related to Sphinx installation and documentation build. -""" - - -def main(): - """Main function""" - parser = argparse.ArgumentParser(description=DESCRIPTION) - - parser.add_argument( - "--no-virtualenv", - action="store_false", - dest="virtualenv", - help="Recommend installing Sphinx instead of using a virtualenv", - ) - - parser.add_argument( - "--no-pdf", - action="store_false", - dest="pdf", - help="Don't check for dependencies required to build PDF docs", - ) - - parser.add_argument( - "--version-check", - action="store_true", - dest="version_check", - help="If version is compatible, don't check for missing dependencies", - ) - - args = parser.parse_args() - - checker = SphinxDependencyChecker(args) - - checker.check_python() - checker.check_needs() - -# Call main if not used as module -if __name__ == "__main__": - main() -- cgit From 29e71d96837dd49ab2f35f3524a1ceda0ff21ea7 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:52 +0200 Subject: scripts: sphinx-pre-install.pl: get rid of the old script All features were ported to the Python version. Plus, it supports more variants and contain fixes. So, drop the old version. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/6900872e6b89b7ff304e70f5d1c23cbb3c757d28.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.pl | 1056 ----------------------------------------- 1 file changed, 1056 deletions(-) delete mode 100755 scripts/sphinx-pre-install.pl (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.pl b/scripts/sphinx-pre-install.pl deleted file mode 100755 index 07234d482fa8..000000000000 --- a/scripts/sphinx-pre-install.pl +++ /dev/null @@ -1,1056 +0,0 @@ -#!/usr/bin/env perl -# SPDX-License-Identifier: GPL-2.0-or-later -use strict; - -# Copyright (c) 2017-2020 Mauro Carvalho Chehab -# - -my $prefix = "./"; -$prefix = "$ENV{'srctree'}/" if ($ENV{'srctree'}); - -my $conf = $prefix . "Documentation/conf.py"; -my $requirement_file = $prefix . "Documentation/sphinx/requirements.txt"; -my $virtenv_prefix = "sphinx_"; - -# -# Static vars -# - -my %missing; -my $system_release; -my $need = 0; -my $optional = 0; -my $need_symlink = 0; -my $need_sphinx = 0; -my $need_pip = 0; -my $need_virtualenv = 0; -my $rec_sphinx_upgrade = 0; -my $verbose_warn_install = 1; -my $install = ""; -my $virtenv_dir = ""; -my $python_cmd = ""; -my $activate_cmd; -my $min_version; -my $cur_version; -my $rec_version = "3.4.3"; -my $latest_avail_ver; - -# -# Command line arguments -# - -my $pdf = 1; -my $virtualenv = 1; -my $version_check = 0; - -# -# List of required texlive packages on Fedora and OpenSuse -# - -my %texlive = ( - 'amsfonts.sty' => 'texlive-amsfonts', - 'amsmath.sty' => 'texlive-amsmath', - 'amssymb.sty' => 'texlive-amsfonts', - 'amsthm.sty' => 'texlive-amscls', - 'anyfontsize.sty' => 'texlive-anyfontsize', - 'atbegshi.sty' => 'texlive-oberdiek', - 'bm.sty' => 'texlive-tools', - 'capt-of.sty' => 'texlive-capt-of', - 'cmap.sty' => 'texlive-cmap', - 'ecrm1000.tfm' => 'texlive-ec', - 'eqparbox.sty' => 'texlive-eqparbox', - 'eu1enc.def' => 'texlive-euenc', - 'fancybox.sty' => 'texlive-fancybox', - 'fancyvrb.sty' => 'texlive-fancyvrb', - 'float.sty' => 'texlive-float', - 'fncychap.sty' => 'texlive-fncychap', - 'footnote.sty' => 'texlive-mdwtools', - 'framed.sty' => 'texlive-framed', - 'luatex85.sty' => 'texlive-luatex85', - 'multirow.sty' => 'texlive-multirow', - 'needspace.sty' => 'texlive-needspace', - 'palatino.sty' => 'texlive-psnfss', - 'parskip.sty' => 'texlive-parskip', - 'polyglossia.sty' => 'texlive-polyglossia', - 'tabulary.sty' => 'texlive-tabulary', - 'threeparttable.sty' => 'texlive-threeparttable', - 'titlesec.sty' => 'texlive-titlesec', - 'ucs.sty' => 'texlive-ucs', - 'upquote.sty' => 'texlive-upquote', - 'wrapfig.sty' => 'texlive-wrapfig', - 'ctexhook.sty' => 'texlive-ctex', -); - -# -# Subroutines that checks if a feature exists -# - -sub check_missing(%) -{ - my %map = %{$_[0]}; - - foreach my $prog (sort keys %missing) { - my $is_optional = $missing{$prog}; - - # At least on some LTS distros like CentOS 7, texlive doesn't - # provide all packages we need. When such distros are - # detected, we have to disable PDF output. - # - # So, we need to ignore the packages that distros would - # need for LaTeX to work - if ($is_optional == 2 && !$pdf) { - $optional--; - next; - } - - if ($verbose_warn_install) { - if ($is_optional) { - print "Warning: better to also install \"$prog\".\n"; - } else { - print "ERROR: please install \"$prog\", otherwise, build won't work.\n"; - } - } - if (defined($map{$prog})) { - $install .= " " . $map{$prog}; - } else { - $install .= " " . $prog; - } - } - - $install =~ s/^\s//; -} - -sub add_package($$) -{ - my $package = shift; - my $is_optional = shift; - - $missing{$package} = $is_optional; - if ($is_optional) { - $optional++; - } else { - $need++; - } -} - -sub check_missing_file($$$) -{ - my $files = shift; - my $package = shift; - my $is_optional = shift; - - for (@$files) { - return if(-e $_); - } - - add_package($package, $is_optional); -} - -sub findprog($) -{ - foreach(split(/:/, $ENV{PATH})) { - return "$_/$_[0]" if(-x "$_/$_[0]"); - } -} - -sub find_python_no_venv() -{ - my $prog = shift; - - my $cur_dir = qx(pwd); - $cur_dir =~ s/\s+$//; - - foreach my $dir (split(/:/, $ENV{PATH})) { - next if ($dir =~ m,($cur_dir)/sphinx,); - return "$dir/python3" if(-x "$dir/python3"); - } - foreach my $dir (split(/:/, $ENV{PATH})) { - next if ($dir =~ m,($cur_dir)/sphinx,); - return "$dir/python" if(-x "$dir/python"); - } - return "python"; -} - -sub check_program($$) -{ - my $prog = shift; - my $is_optional = shift; - - return $prog if findprog($prog); - - add_package($prog, $is_optional); -} - -sub check_perl_module($$) -{ - my $prog = shift; - my $is_optional = shift; - - my $err = system("perl -M$prog -e 1 2>/dev/null /dev/null"); - return if ($err == 0); - - add_package($prog, $is_optional); -} - -sub check_python_module($$) -{ - my $prog = shift; - my $is_optional = shift; - - return if (!$python_cmd); - - my $err = system("$python_cmd -c 'import $prog' 2>/dev/null /dev/null"); - return if ($err == 0); - - add_package($prog, $is_optional); -} - -sub check_rpm_missing($$) -{ - my @pkgs = @{$_[0]}; - my $is_optional = $_[1]; - - foreach my $prog(@pkgs) { - my $err = system("rpm -q '$prog' 2>/dev/null >/dev/null"); - add_package($prog, $is_optional) if ($err); - } -} - -sub check_pacman_missing($$) -{ - my @pkgs = @{$_[0]}; - my $is_optional = $_[1]; - - foreach my $prog(@pkgs) { - my $err = system("pacman -Q '$prog' 2>/dev/null >/dev/null"); - add_package($prog, $is_optional) if ($err); - } -} - -sub check_missing_tex($) -{ - my $is_optional = shift; - my $kpsewhich = findprog("kpsewhich"); - - foreach my $prog(keys %texlive) { - my $package = $texlive{$prog}; - if (!$kpsewhich) { - add_package($package, $is_optional); - next; - } - my $file = qx($kpsewhich $prog); - add_package($package, $is_optional) if ($file =~ /^\s*$/); - } -} - -sub get_sphinx_fname() -{ - if ($ENV{'SPHINXBUILD'}) { - return $ENV{'SPHINXBUILD'}; - } - - my $fname = "sphinx-build"; - return $fname if findprog($fname); - - $fname = "sphinx-build-3"; - if (findprog($fname)) { - $need_symlink = 1; - return $fname; - } - - return ""; -} - -sub get_sphinx_version($) -{ - my $cmd = shift; - my $ver; - - open IN, "$cmd --version 2>&1 |"; - while () { - if (m/^\s*sphinx-build\s+([\d\.]+)((\+\/[\da-f]+)|(b\d+))?$/) { - $ver=$1; - last; - } - # Sphinx 1.2.x uses a different format - if (m/^\s*Sphinx.*\s+([\d\.]+)$/) { - $ver=$1; - last; - } - } - close IN; - return $ver; -} - -sub check_sphinx() -{ - open IN, $conf or die "Can't open $conf"; - while () { - if (m/^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]/) { - $min_version=$1; - last; - } - } - close IN; - - die "Can't get needs_sphinx version from $conf" if (!$min_version); - - $virtenv_dir = $virtenv_prefix . "latest"; - - my $sphinx = get_sphinx_fname(); - if ($sphinx eq "") { - $need_sphinx = 1; - return; - } - - $cur_version = get_sphinx_version($sphinx); - die "$sphinx didn't return its version" if (!$cur_version); - - if ($cur_version lt $min_version) { - printf "ERROR: Sphinx version is %s. It should be >= %s\n", - $cur_version, $min_version; - $need_sphinx = 1; - return; - } - - return if ($cur_version lt $rec_version); - - # On version check mode, just assume Sphinx has all mandatory deps - exit (0) if ($version_check); -} - -# -# Ancillary subroutines -# - -sub catcheck($) -{ - my $res = ""; - $res = qx(cat $_[0]) if (-r $_[0]); - return $res; -} - -sub which($) -{ - my $file = shift; - my @path = split ":", $ENV{PATH}; - - foreach my $dir(@path) { - my $name = $dir.'/'.$file; - return $name if (-x $name ); - } - return undef; -} - -# -# Subroutines that check distro-specific hints -# - -sub give_debian_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-yaml", - "ensurepip" => "python3-venv", - "virtualenv" => "virtualenv", - "dot" => "graphviz", - "convert" => "imagemagick", - "Pod::Usage" => "perl-modules", - "xelatex" => "texlive-xetex", - "rsvg-convert" => "librsvg2-bin", - ); - - if ($pdf) { - check_missing_file(["/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty"], - "texlive-lang-chinese", 2); - - check_missing_file(["/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"], - "fonts-dejavu", 2); - - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc"], - "fonts-noto-cjk", 2); - } - - check_program("dvipng", 2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo apt-get install $install\n"); -} - -sub give_redhat_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-pyyaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive-xetex-bin", - "rsvg-convert" => "librsvg2-tools", - ); - - my @fedora26_opt_pkgs = ( - "graphviz-gd", # Fedora 26: needed for PDF support - ); - - my @fedora_tex_pkgs = ( - "texlive-collection-fontsrecommended", - "texlive-collection-latex", - "texlive-xecjk", - "dejavu-sans-fonts", - "dejavu-serif-fonts", - "dejavu-sans-mono-fonts", - ); - - # - # Checks valid for RHEL/CentOS version 7.x. - # - my $old = 0; - my $rel; - my $noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts"; - $rel = $2 if ($system_release =~ /(release|Linux)\s+(\d+)/); - - if (!($system_release =~ /Fedora/)) { - $map{"virtualenv"} = "python-virtualenv"; - - if ($rel && $rel < 8) { - $old = 1; - $pdf = 0; - - printf("Note: texlive packages on RHEL/CENTOS <= 7 are incomplete. Can't support PDF output\n"); - printf("If you want to build PDF, please read:\n"); - printf("\thttps://www.systutorials.com/241660/how-to-install-tex-live-on-centos-7-linux/\n"); - } - } else { - if ($rel && $rel < 26) { - $old = 1; - } - if ($rel && $rel >= 38) { - $noto_sans_redhat = "google-noto-sans-cjk-fonts"; - } - } - if (!$rel) { - printf("Couldn't identify release number\n"); - $old = 1; - $pdf = 0; - } - - if ($pdf) { - check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc"], - $noto_sans_redhat, 2); - } - - check_rpm_missing(\@fedora26_opt_pkgs, 2) if ($pdf && !$old); - check_rpm_missing(\@fedora_tex_pkgs, 2) if ($pdf); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - - if (!$old) { - # dnf, for Fedora 18+ - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo dnf install -y $install\n"); - } else { - # yum, for RHEL (and clones) or Fedora version < 18 - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo yum install -y $install\n"); - } -} - -sub give_opensuse_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-pyyaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive-xetex-bin", - ); - - # On Tumbleweed, this package is also named rsvg-convert - $map{"rsvg-convert"} = "rsvg-view" if (!($system_release =~ /Tumbleweed/)); - - my @suse_tex_pkgs = ( - "texlive-babel-english", - "texlive-caption", - "texlive-colortbl", - "texlive-courier", - "texlive-dvips", - "texlive-helvetic", - "texlive-makeindex", - "texlive-metafont", - "texlive-metapost", - "texlive-palatino", - "texlive-preview", - "texlive-times", - "texlive-zapfchan", - "texlive-zapfding", - ); - - $map{"latexmk"} = "texlive-latexmk-bin"; - - # FIXME: add support for installing CJK fonts - # - # I tried hard, but was unable to find a way to install - # "Noto Sans CJK SC" on openSUSE - - check_rpm_missing(\@suse_tex_pkgs, 2) if ($pdf); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo zypper install --no-recommends $install\n"); -} - -sub give_mageia_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-yaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive", - "rsvg-convert" => "librsvg2", - ); - - my @tex_pkgs = ( - "texlive-fontsextra", - ); - - $map{"latexmk"} = "texlive-collection-basic"; - - my $packager_cmd; - my $noto_sans; - if ($system_release =~ /OpenMandriva/) { - $packager_cmd = "dnf install"; - $noto_sans = "noto-sans-cjk-fonts"; - @tex_pkgs = ( "texlive-collection-fontsextra" ); - } else { - $packager_cmd = "urpmi"; - $noto_sans = "google-noto-sans-cjk-ttc-fonts"; - } - - - if ($pdf) { - check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/TTF/NotoSans-Regular.ttf"], - $noto_sans, 2); - } - - check_rpm_missing(\@tex_pkgs, 2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo $packager_cmd $install\n"); -} - -sub give_arch_linux_hints() -{ - my %map = ( - "yaml" => "python-yaml", - "virtualenv" => "python-virtualenv", - "dot" => "graphviz", - "convert" => "imagemagick", - "xelatex" => "texlive-xetex", - "latexmk" => "texlive-core", - "rsvg-convert" => "extra/librsvg", - ); - - my @archlinux_tex_pkgs = ( - "texlive-core", - "texlive-latexextra", - "ttf-dejavu", - ); - check_pacman_missing(\@archlinux_tex_pkgs, 2) if ($pdf); - - if ($pdf) { - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], - "noto-fonts-cjk", 2); - } - - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo pacman -S $install\n"); -} - -sub give_gentoo_hints() -{ - my %map = ( - "yaml" => "dev-python/pyyaml", - "virtualenv" => "dev-python/virtualenv", - "dot" => "media-gfx/graphviz", - "convert" => "media-gfx/imagemagick", - "xelatex" => "dev-texlive/texlive-xetex media-fonts/dejavu", - "rsvg-convert" => "gnome-base/librsvg", - ); - - check_missing_file(["/usr/share/fonts/dejavu/DejaVuSans.ttf"], - "media-fonts/dejavu", 2) if ($pdf); - - if ($pdf) { - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", - "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc"], - "media-fonts/noto-cjk", 2); - } - - check_missing(\%map); - - return if (!$need && !$optional); - - printf("You should run:\n") if ($verbose_warn_install); - printf("\n"); - - my $imagemagick = "media-gfx/imagemagick svg png"; - my $cairo = "media-gfx/graphviz cairo pdf"; - my $portage_imagemagick = "/etc/portage/package.use/imagemagick"; - my $portage_cairo = "/etc/portage/package.use/graphviz"; - - if (qx(grep imagemagick $portage_imagemagick 2>/dev/null) eq "") { - printf("\tsudo su -c 'echo \"$imagemagick\" > $portage_imagemagick'\n") - } - if (qx(grep graphviz $portage_cairo 2>/dev/null) eq "") { - printf("\tsudo su -c 'echo \"$cairo\" > $portage_cairo'\n"); - } - - printf("\tsudo emerge --ask $install\n"); - -} - -sub check_distros() -{ - # Distro-specific hints - if ($system_release =~ /Red Hat Enterprise Linux/) { - give_redhat_hints; - return; - } - if ($system_release =~ /CentOS/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Scientific Linux/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Oracle Linux Server/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Fedora/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Ubuntu/) { - give_debian_hints; - return; - } - if ($system_release =~ /Debian/) { - give_debian_hints; - return; - } - if ($system_release =~ /openSUSE/) { - give_opensuse_hints; - return; - } - if ($system_release =~ /Mageia/) { - give_mageia_hints; - return; - } - if ($system_release =~ /OpenMandriva/) { - give_mageia_hints; - return; - } - if ($system_release =~ /Arch Linux/) { - give_arch_linux_hints; - return; - } - if ($system_release =~ /Gentoo/) { - give_gentoo_hints; - return; - } - - # - # Fall-back to generic hint code for other distros - # That's far from ideal, specially for LaTeX dependencies. - # - my %map = ( - "sphinx-build" => "sphinx" - ); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - print "I don't know distro $system_release.\n"; - print "So, I can't provide you a hint with the install procedure.\n"; - print "There are likely missing dependencies.\n"; -} - -# -# Common dependencies -# - -sub deactivate_help() -{ - printf "\n If you want to exit the virtualenv, you can use:\n"; - printf "\tdeactivate\n"; -} - -sub get_virtenv() -{ - my $ver; - my $min_activate = "$ENV{'PWD'}/${virtenv_prefix}${min_version}/bin/activate"; - my @activates = glob "$ENV{'PWD'}/${virtenv_prefix}*/bin/activate"; - - @activates = sort {$b cmp $a} @activates; - - foreach my $f (@activates) { - next if ($f lt $min_activate); - - my $sphinx_cmd = $f; - $sphinx_cmd =~ s/activate/sphinx-build/; - next if (! -f $sphinx_cmd); - - my $ver = get_sphinx_version($sphinx_cmd); - - if (!$ver) { - $f =~ s#/bin/activate##; - print("Warning: virtual environment $f is not working.\nPython version upgrade? Remove it with:\n\n\trm -rf $f\n\n"); - } - - if ($need_sphinx && ($ver ge $min_version)) { - return ($f, $ver); - } elsif ($ver gt $cur_version) { - return ($f, $ver); - } - } - return ("", ""); -} - -sub recommend_sphinx_upgrade() -{ - my $venv_ver; - - # Avoid running sphinx-builds from venv if $cur_version is good - if ($cur_version && ($cur_version ge $rec_version)) { - $latest_avail_ver = $cur_version; - return; - } - - # Get the highest version from sphinx_*/bin/sphinx-build and the - # corresponding command to activate the venv/virtenv - ($activate_cmd, $venv_ver) = get_virtenv(); - - # Store the highest version from Sphinx existing virtualenvs - if (($activate_cmd ne "") && ($venv_ver gt $cur_version)) { - $latest_avail_ver = $venv_ver; - } else { - $latest_avail_ver = $cur_version if ($cur_version); - } - - # As we don't know package version of Sphinx, and there's no - # virtual environments, don't check if upgrades are needed - if (!$virtualenv) { - return if (!$latest_avail_ver); - } - - # Either there are already a virtual env or a new one should be created - $need_pip = 1; - - return if (!$latest_avail_ver); - - # Return if the reason is due to an upgrade or not - if ($latest_avail_ver lt $rec_version) { - $rec_sphinx_upgrade = 1; - } - - return $latest_avail_ver; -} - -# -# The logic here is complex, as it have to deal with different versions: -# - minimal supported version; -# - minimal PDF version; -# - recommended version. -# It also needs to work fine with both distro's package and venv/virtualenv -sub recommend_sphinx_version($) -{ - my $virtualenv_cmd = shift; - - # Version is OK. Nothing to do. - if ($cur_version && ($cur_version ge $rec_version)) { - return; - }; - - if (!$need_sphinx) { - # sphinx-build is present and its version is >= $min_version - - #only recommend enabling a newer virtenv version if makes sense. - if ($latest_avail_ver gt $cur_version) { - printf "\nYou may also use the newer Sphinx version $latest_avail_ver with:\n"; - printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - return if ($latest_avail_ver ge $rec_version); - } - - if (!$virtualenv) { - # No sphinx either via package or via virtenv. As we can't - # Compare the versions here, just return, recommending the - # user to install it from the package distro. - return if (!$latest_avail_ver); - - # User doesn't want a virtenv recommendation, but he already - # installed one via virtenv with a newer version. - # So, print commands to enable it - if ($latest_avail_ver gt $cur_version) { - printf "\nYou may also use the Sphinx virtualenv version $latest_avail_ver with:\n"; - printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - print "\n"; - } else { - $need++ if ($need_sphinx); - } - - # Suggest newer versions if current ones are too old - if ($latest_avail_ver && $latest_avail_ver ge $min_version) { - # If there's a good enough version, ask the user to enable it - if ($latest_avail_ver ge $rec_version) { - printf "\nNeed to activate Sphinx (version $latest_avail_ver) on virtualenv with:\n"; - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - - # Version is above the minimal required one, but may be - # below the recommended one. So, print warnings/notes - - if ($latest_avail_ver lt $rec_version) { - print "Warning: It is recommended at least Sphinx version $rec_version.\n"; - } - } - - # At this point, either it needs Sphinx or upgrade is recommended, - # both via pip - - if ($rec_sphinx_upgrade) { - if (!$virtualenv) { - print "Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n"; - } else { - print "To upgrade Sphinx, use:\n\n"; - } - } else { - print "\nSphinx needs to be installed either:\n1) via pip/pypi with:\n\n"; - } - - $python_cmd = find_python_no_venv(); - - printf "\t$virtualenv_cmd $virtenv_dir\n"; - - printf "\t. $virtenv_dir/bin/activate\n"; - printf "\tpip install -r $requirement_file\n"; - deactivate_help(); - - printf "\n2) As a package with:\n"; - - my $old_need = $need; - my $old_optional = $optional; - %missing = (); - $pdf = 0; - $optional = 0; - $install = ""; - $verbose_warn_install = 0; - - add_package("python-sphinx", 0); - - check_distros(); - - $need = $old_need; - $optional = $old_optional; - - printf "\n Please note that Sphinx >= 3.0 will currently produce false-positive\n"; - printf " warning when the same name is used for more than one type (functions,\n"; - printf " structs, enums,...). This is known Sphinx bug. For more details, see:\n"; - printf "\thttps://github.com/sphinx-doc/sphinx/pull/8313\n"; -} - -sub check_needs() -{ - # Check if Sphinx is already accessible from current environment - check_sphinx(); - - if ($system_release) { - print "Detected OS: $system_release.\n"; - } else { - print "Unknown OS\n"; - } - printf "Sphinx version: %s\n\n", $cur_version if ($cur_version); - - # Check python command line, trying first python3 - $python_cmd = findprog("python3"); - $python_cmd = check_program("python", 0) if (!$python_cmd); - - # Check the type of virtual env, depending on Python version - if ($python_cmd) { - if ($virtualenv) { - my $tmp = qx($python_cmd --version 2>&1); - if ($tmp =~ m/(\d+\.)(\d+\.)/) { - if ($1 < 3) { - # Fail if it finds python2 (or worse) - die "Python 3 is required to build the kernel docs\n"; - } - if ($1 == 3 && $2 < 3) { - # Need Python 3.3 or upper for venv - $need_virtualenv = 1; - } - } else { - die "Warning: couldn't identify $python_cmd version!"; - } - } else { - add_package("python-sphinx", 0); - } - } - - my $venv_ver = recommend_sphinx_upgrade(); - - my $virtualenv_cmd; - - if ($need_pip) { - # Set virtualenv command line, if python < 3.3 - if ($need_virtualenv) { - $virtualenv_cmd = findprog("virtualenv-3"); - $virtualenv_cmd = findprog("virtualenv-3.5") if (!$virtualenv_cmd); - if (!$virtualenv_cmd) { - check_program("virtualenv", 0); - $virtualenv_cmd = "virtualenv"; - } - } else { - $virtualenv_cmd = "$python_cmd -m venv"; - check_python_module("ensurepip", 0); - } - } - - # Check for needed programs/tools - check_perl_module("Pod::Usage", 0); - check_python_module("yaml", 0); - check_program("make", 0); - check_program("gcc", 0); - check_program("dot", 1); - check_program("convert", 1); - - # Extra PDF files - should use 2 for is_optional - check_program("xelatex", 2) if ($pdf); - check_program("rsvg-convert", 2) if ($pdf); - check_program("latexmk", 2) if ($pdf); - - # Do distro-specific checks and output distro-install commands - check_distros(); - - if (!$python_cmd) { - if ($need == 1) { - die "Can't build as $need mandatory dependency is missing"; - } elsif ($need) { - die "Can't build as $need mandatory dependencies are missing"; - } - } - - # Check if sphinx-build is called sphinx-build-3 - if ($need_symlink) { - printf "\tsudo ln -sf %s /usr/bin/sphinx-build\n\n", - which("sphinx-build-3"); - } - - recommend_sphinx_version($virtualenv_cmd); - printf "\n"; - - print "All optional dependencies are met.\n" if (!$optional); - - if ($need == 1) { - die "Can't build as $need mandatory dependency is missing"; - } elsif ($need) { - die "Can't build as $need mandatory dependencies are missing"; - } - - print "Needed package dependencies are met.\n"; -} - -# -# Main -# - -while (@ARGV) { - my $arg = shift(@ARGV); - - if ($arg eq "--no-virtualenv") { - $virtualenv = 0; - } elsif ($arg eq "--no-pdf"){ - $pdf = 0; - } elsif ($arg eq "--version-check"){ - $version_check = 1; - } else { - print "Usage:\n\t$0 <--no-virtualenv> <--no-pdf> <--version-check>\n\n"; - print "Where:\n"; - print "\t--no-virtualenv\t- Recommend installing Sphinx instead of using a virtualenv\n"; - print "\t--version-check\t- if version is compatible, don't check for missing dependencies\n"; - print "\t--no-pdf\t- don't check for dependencies required to build PDF docs\n\n"; - exit -1; - } -} - -# -# Determine the system type. There's no standard unique way that would -# work with all distros with a minimal package install. So, several -# methods are used here. -# -# By default, it will use lsb_release function. If not available, it will -# fail back to reading the known different places where the distro name -# is stored -# - -$system_release = qx(lsb_release -d) if which("lsb_release"); -$system_release =~ s/Description:\s*// if ($system_release); -$system_release = catcheck("/etc/system-release") if !$system_release; -$system_release = catcheck("/etc/redhat-release") if !$system_release; -$system_release = catcheck("/etc/lsb-release") if !$system_release; -$system_release = catcheck("/etc/gentoo-release") if !$system_release; - -# This seems more common than LSB these days -if (!$system_release) { - my %os_var; - if (open IN, "cat /etc/os-release|") { - while () { - if (m/^([\w\d\_]+)=\"?([^\"]*)\"?\n/) { - $os_var{$1}=$2; - } - } - $system_release = $os_var{"NAME"}; - if (defined($os_var{"VERSION_ID"})) { - $system_release .= " " . $os_var{"VERSION_ID"} if (defined($os_var{"VERSION_ID"})); - } else { - $system_release .= " " . $os_var{"VERSION"}; - } - } -} -$system_release = catcheck("/etc/issue") if !$system_release; -$system_release =~ s/\s+$//; - -check_needs; -- cgit From df4d2f966354297c94ba5e198533cc31548a12a8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:53 +0200 Subject: scripts: sphinx-pre-install: update mandatory system deps To build docs, gcc is not needed. Also, Kernel can be built nowadays with clang. So, drop it. On the other hand, which is needed. Add a system dependency for it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/4ec979e4692c9e4acd6c31424c0e2f4bf5b80e71.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 7dfe5c2a6cc2..fc9dc45054d7 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -1476,7 +1476,7 @@ class SphinxDependencyChecker(MissingCheckers): self.check_perl_module("Pod::Usage", DepManager.SYSTEM_MANDATORY) self.check_program("make", DepManager.SYSTEM_MANDATORY) - self.check_program("gcc", DepManager.SYSTEM_MANDATORY) + self.check_program("which", DepManager.SYSTEM_MANDATORY) self.check_program("dot", DepManager.SYSTEM_OPTIONAL) self.check_program("convert", DepManager.SYSTEM_OPTIONAL) -- cgit From 9f51a1d6966760dadf62ff1f62c614a342fb7e62 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:54 +0200 Subject: scripts: sphinx-pre-install: add support for RHEL8-based distros On RHEL8, only installing with a venv is supported, as there's no Sphinx package using Python 3.7 or upper. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/bcdde20edab07be6bf447eac18eecdd88c7f947c.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index fc9dc45054d7..324baa98a395 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -691,7 +691,7 @@ class SphinxDependencyChecker(MissingCheckers): super().__init__(args, texlive) - self.need_pip = 0 + self.need_pip = False self.rec_sphinx_upgrade = 0 self.system_release = self.get_system_release() @@ -840,9 +840,15 @@ class SphinxDependencyChecker(MissingCheckers): # RHEL 8 uses Python 3.6, which is not compatible with # the build system anymore. Suggest Python 3.11 if rel == 8: - self.deps.add_package("python39", DepManager.SYSTEM_MANDATORY) + self.check_program("python3.9", DepManager.SYSTEM_MANDATORY) + progs["python3.9"] = "python39" + progs["yaml"] = "python39-pyyaml" + self.recommend_python = True + # There's no python39-sphinx package. Only pip is supported + self.package_supported = False + if not self.distro_msg: self.distro_msg = \ "Note: RHEL-based distros typically require extra repositories.\n" \ @@ -915,8 +921,9 @@ class SphinxDependencyChecker(MissingCheckers): # the build system anymore. Suggest Python 3.11 if rel == 15: if not self.which(self.python_cmd): + self.check_program("python3.11", DepManager.SYSTEM_MANDATORY) + progs["python3.11"] = "python311" self.recommend_python = True - self.deps.add_package(self.python_cmd, DepManager.SYSTEM_MANDATORY) progs.update({ "python-sphinx": "python311-Sphinx", @@ -1289,7 +1296,7 @@ class SphinxDependencyChecker(MissingCheckers): return self.latest_avail_ver # Either there are already a virtual env or a new one should be created - self.need_pip = 1 + self.need_pip = True if not self.latest_avail_ver: return None @@ -1344,10 +1351,11 @@ class SphinxDependencyChecker(MissingCheckers): """ if self.recommend_python: - print("\nPython version is incompatible with doc build.\n" \ - "Please upgrade it and re-run.\n") - return - + cur_ver = sys.version_info[:3] + if cur_ver < MIN_PYTHON_VERSION: + print(f"\nPython version {cur_ver} is incompatible with doc build.\n" \ + "Please upgrade it and re-run.\n") + return # Version is OK. Nothing to do. if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: @@ -1467,7 +1475,6 @@ class SphinxDependencyChecker(MissingCheckers): if sys.version_info < MIN_PYTHON_VERSION: self.need_pip = False print("Warning: python version is not supported.") - else: virtualenv_cmd = f"{self.python_cmd} -m venv" self.check_python_module("ensurepip") -- cgit From 491a99511eaf333855caf28170cfb3f6f4460b54 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:55 +0200 Subject: scripts: sphinx-pre-install: add a warning for Debian-based distros On Some Debian-based distros, ImageMagick package has a broken policy that causes LaTeX to fail while building docs. Add a note about that. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/628d01784e8c24e3d93c69c436f12398e00165b3.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 324baa98a395..09a337509b23 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -782,6 +782,11 @@ class SphinxDependencyChecker(MissingCheckers): self.check_program("dvipng", DepManager.PDF_MANDATORY) + if not self.distro_msg: + self.distro_msg = \ + "Note: ImageMagick is broken on some distros, affecting PDF output. For more details:\n" \ + "\thttps://askubuntu.com/questions/1158894/imagemagick-still-broken-using-with-usr-bin-convert" + return self.get_install_progs(progs, "apt-get install") def give_redhat_hints(self): @@ -1193,7 +1198,7 @@ class SphinxDependencyChecker(MissingCheckers): self.distro_msg = \ f"I don't know distro {self.system_release}.\n" \ "So, I can't provide you a hint with the install procedure.\n" \ - "There are likely missing dependencies.\n" + "There are likely missing dependencies." return self.get_install_progs(progs, None) -- cgit From 6170b1eacac881af5732af78f33b76bd560c441b Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:56 +0200 Subject: scripts: sphinx-pre-install: some adjustments related to venv While nothing was really needed for virtualenv to work on most distros, we had an issue with OpenMandriva. While checking for it, it was noticed that there was no check if python-virtualenv was installed. This didn't solve the issues we faced there: at least with the half-broken OpenMandriva Lx 4.0 docker container we used, ensurepip was not available anywhere, causing venv to fail. Add a distro-specific note about that. Note: at least at the time we did our tests, OpenMandriva Lx 4.0 docker was shipped with wrong dnf repositories. Also, there was no repos available for it anymore. So, we had to do some hacks to upgrade to 4.3 before being able to run any tests. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/e3a0e5eccd50eb506846e3e8487a2d9124ef83e2.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 09a337509b23..b8474848df4e 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -983,6 +983,22 @@ class SphinxDependencyChecker(MissingCheckers): # Tested on OpenMandriva Lx 4.3 progs["convert"] = "imagemagick" progs["yaml"] = "python-pyyaml" + progs["python-virtualenv"] = "python-virtualenv" + progs["python-sphinx"] = "python-sphinx" + + self.check_program("python-virtualenv", DepManager.PYTHON_MANDATORY) + + # On my tests with openMandriva LX 4.0 docker image, upgraded + # to 4.3, python-virtualenv package is broken: it is missing + # ensurepip. Without it, the alternative would be to run: + # python3 -m venv --without-pip ~/sphinx_latest, but running + # pip there won't install sphinx at venv. + # + # Add a note about that. + + if not self.distro_msg: + self.distro_msg = \ + "Note: for venv, ensurepip could be broken, preventing its install method." else: packager_cmd = "urpmi" -- cgit From 670ec7333a2c4823ac777b70f045cf731525ae5e Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:29 -0600 Subject: docs: kdoc: remove dead code create_parameter_list() tests an argument against the same regex twice, in two different locations; remove the pointless extra tests and the never-executed error cases that go with them. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-2-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 9e65948f8254..96e3fe4ec431 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -564,28 +564,18 @@ class KernelDoc: args.insert(0, first_arg.pop()) dtype = ' '.join(first_arg) + bitfield_re = KernRe(r'(.*?):(\w+)') for param in args: - if KernRe(r'^(\*+)\s*(.*)').match(param): - r = KernRe(r'^(\*+)\s*(.*)') - if not r.match(param): - self.emit_msg(ln, f"Invalid param: {param}") - continue - - param = r.group(1) - + r = KernRe(r'^(\*+)\s*(.*)') + if r.match(param): self.push_parameter(ln, decl_type, r.group(2), f"{dtype} {r.group(1)}", arg, declaration_name) - elif KernRe(r'(.*?):(\w+)').search(param): - r = KernRe(r'(.*?):(\w+)') - if not r.match(param): - self.emit_msg(ln, f"Invalid param: {param}") - continue - + elif bitfield_re.search(param): if dtype != "": # Skip unnamed bit-fields - self.push_parameter(ln, decl_type, r.group(1), - f"{dtype}:{r.group(2)}", + self.push_parameter(ln, decl_type, bitfield_re.group(1), + f"{dtype}:{bitfield_re.group(2)}", arg, declaration_name) else: self.push_parameter(ln, decl_type, param, dtype, -- cgit From f51b42b99e1d35698e0277337fde2c15ccc29a2b Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:30 -0600 Subject: docs: kdoc: tidy up space removal in create_parameter_list() Remove a redundant test and add a comment describing what the space removal is doing. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-3-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 96e3fe4ec431..53051ce831ba 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -545,12 +545,14 @@ class KernelDoc: arg, declaration_name) elif arg: + # + # Clean up extraneous spaces and split the string at commas; the first + # element of the resulting list will also include the type information. + # arg = KernRe(r'\s*:\s*').sub(":", arg) arg = KernRe(r'\s*\[').sub('[', arg) - args = KernRe(r'\s*,\s*').split(arg) - if args[0] and '*' in args[0]: - args[0] = re.sub(r'(\*+)\s*', r' \1', args[0]) + args[0] = re.sub(r'(\*+)\s*', r' \1', args[0]) first_arg = [] r = KernRe(r'^(.*\s+)(.*?\[.*\].*)$') -- cgit From 05d72fe07242a8e4535aa52e0858f9198e668a41 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:31 -0600 Subject: docs: kdoc: clean up the create_parameter_list() "first arg" logic The logic for finding the name of the first in a series of variable names is somewhat convoluted and, in the use of .extend(), actively buggy. Document what is happening and simplify the logic. Acked-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-4-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 53051ce831ba..07234ce04409 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -553,18 +553,18 @@ class KernelDoc: arg = KernRe(r'\s*\[').sub('[', arg) args = KernRe(r'\s*,\s*').split(arg) args[0] = re.sub(r'(\*+)\s*', r' \1', args[0]) - - first_arg = [] - r = KernRe(r'^(.*\s+)(.*?\[.*\].*)$') - if args[0] and r.match(args[0]): - args.pop(0) - first_arg.extend(r.group(1)) - first_arg.append(r.group(2)) + # + # args[0] has a string of "type a". If "a" includes an [array] + # declaration, we want to not be fooled by any white space inside + # the brackets, so detect and handle that case specially. + # + r = KernRe(r'^([^[\]]*\s+)(.*)$') + if r.match(args[0]): + args[0] = r.group(2) + dtype = r.group(1) else: - first_arg = KernRe(r'\s+').split(args.pop(0)) - - args.insert(0, first_arg.pop()) - dtype = ' '.join(first_arg) + # No space in args[0]; this seems wrong but preserves previous behavior + dtype = '' bitfield_re = KernRe(r'(.*?):(\w+)') for param in args: -- cgit From 8f05fbc5afb86f0d4dcae33f3cb0cda561d4d93e Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:32 -0600 Subject: docs: kdoc: add a couple more comments in create_parameter_list() Make what the final code is doing a bit more clear to slow readers like me. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-5-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 07234ce04409..29881757bf1c 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -568,12 +568,18 @@ class KernelDoc: bitfield_re = KernRe(r'(.*?):(\w+)') for param in args: + # + # For pointers, shift the star(s) from the variable name to the + # type declaration. + # r = KernRe(r'^(\*+)\s*(.*)') if r.match(param): self.push_parameter(ln, decl_type, r.group(2), f"{dtype} {r.group(1)}", arg, declaration_name) - + # + # Perform a similar shift for bitfields. + # elif bitfield_re.search(param): if dtype != "": # Skip unnamed bit-fields self.push_parameter(ln, decl_type, bitfield_re.group(1), -- cgit From bf6b310d1b7e31a1cd6951eb75608a1d2876c04a Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:33 -0600 Subject: docs: kdoc: tighten up the array-of-pointers case Simplify one gnarly regex and remove another altogether; add a comment describing what is going on. There will be no #-substituted commas in this case, so don't bother trying to put them back. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-6-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 29881757bf1c..7f4d95dd47d4 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -527,23 +527,21 @@ class KernelDoc: dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg) self.push_parameter(ln, decl_type, param, dtype, arg, declaration_name) - + # + # The array-of-pointers case. Dig the parameter name out from the middle + # of the declaration. + # elif KernRe(r'\(.+\)\s*\[').search(arg): - # Array-of-pointers - - arg = arg.replace('#', ',') - r = KernRe(r'[^\(]+\(\s*\*\s*([\w\[\].]*?)\s*(\s*\[\s*[\w]+\s*\]\s*)*\)') + r = KernRe(r'[^\(]+\(\s*\*\s*' # Up to "(" and maybe "*" + r'([\w.]*?)' # The actual pointer name + r'\s*(\[\s*\w+\s*\]\s*)*\)') # The [array portion] if r.match(arg): param = r.group(1) else: self.emit_msg(ln, f"Invalid param: {arg}") param = arg - - dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg) - - self.push_parameter(ln, decl_type, param, dtype, - arg, declaration_name) - + dtype = arg.replace(param, '') + self.push_parameter(ln, decl_type, param, dtype, arg, declaration_name) elif arg: # # Clean up extraneous spaces and split the string at commas; the first -- cgit From e5d91662fcbac251dd17f04dbacf4d997939316e Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:34 -0600 Subject: docs: kdoc: tighten up the pointer-to-function case Tighten up the code and remove an unneeded regex operation. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-7-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 7f4d95dd47d4..998b1ece932a 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -511,22 +511,21 @@ class KernelDoc: # Treat preprocessor directive as a typeless variable self.push_parameter(ln, decl_type, arg, "", "", declaration_name) - + # + # The pointer-to-function case. + # elif KernRe(r'\(.+\)\s*\(').search(arg): - # Pointer-to-function - arg = arg.replace('#', ',') - - r = KernRe(r'[^\(]+\(\*?\s*([\w\[\].]*)\s*\)') + r = KernRe(r'[^\(]+\(\*?\s*' # Everything up to "(*" + r'([\w\[\].]*)' # Capture the name and possible [array] + r'\s*\)') # Make sure the trailing ")" is there if r.match(arg): param = r.group(1) else: self.emit_msg(ln, f"Invalid param: {arg}") param = arg - - dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg) - self.push_parameter(ln, decl_type, param, dtype, - arg, declaration_name) + dtype = arg.replace(param, '') + self.push_parameter(ln, decl_type, param, dtype, arg, declaration_name) # # The array-of-pointers case. Dig the parameter name out from the middle # of the declaration. -- cgit From 1d8125e27323d8a378cb38f88a6c5a0d7fdb2f6c Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:35 -0600 Subject: docs: kdoc: remove redundant comment stripping By the time stuff gets to create_parameter_list(), comments have long since been stripped out, so we do not need to do it again here. Acked-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-8-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 998b1ece932a..a560546c1867 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -493,9 +493,6 @@ class KernelDoc: args = arg_expr.sub(r"\1#", args) for arg in args.split(splitter): - # Strip comments - arg = KernRe(r'/\*.*\*/').sub('', arg) - # Ignore argument attributes arg = KernRe(r'\sPOS0?\s').sub(' ', arg) -- cgit From 37c52167b007d9d0bb8c5ed53dd6efc4969a1356 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 13 Aug 2025 12:00:52 +0200 Subject: docs: Remove remainders of reiserfs Reiserfs has been removed in 6.13, there are still some mentions in the documentation about it and the tools. Remove those that don't seem relevant anymore but keep references to reiserfs' r5 hash used by some code. There's one change in a script scripts/selinux/install_policy.sh but it does not seem to be relevant either. Signed-off-by: David Sterba Acked-by: Paul Moore Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250813100053.1291961-1-dsterba@suse.com --- scripts/selinux/install_policy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/selinux/install_policy.sh b/scripts/selinux/install_policy.sh index db40237e60ce..77368a73f111 100755 --- a/scripts/selinux/install_policy.sh +++ b/scripts/selinux/install_policy.sh @@ -74,7 +74,7 @@ cd /etc/selinux/dummy/contexts/files $SF -F file_contexts / mounts=`cat /proc/$$/mounts | \ - grep -E "ext[234]|jfs|xfs|reiserfs|jffs2|gfs2|btrfs|f2fs|ocfs2" | \ + grep -E "ext[234]|jfs|xfs|jffs2|gfs2|btrfs|f2fs|ocfs2" | \ awk '{ print $2 '}` $SF -F file_contexts $mounts -- cgit From 4e9a563f0774fdd6f1bd68235314b4b8298fc0a9 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:44 +0200 Subject: scripts: sphinx-pre-install: fix PDF build issues on Ubuntu PDF output with current Debian-based distros require other packages for it to work. The main one is pzdr.tfm. Without that, \sphinxhyphen{} won't work, affecting multiple docs. For CJK, tex-gyre is required. Add the missing packages to the list. After the change, all PDF files build on latest Ubuntu: Ubuntu 25.04: ------------- PASSED: OS detection: Ubuntu 25.04 SKIPPED (Sphinx Sphinx 8.1.3): System packages SKIPPED (Sphinx already installed either as venv or as native package): Sphinx on venv SKIPPED (Sphinx already installed either as venv or as native package): Sphinx package PASSED: Clean documentation: Build time: 0:00, return code: 0 PASSED: Build HTML documentation: Build time: 3:28, return code: 0 PASSED: Build PDF documentation: Build time: 11:08, return code: 0 PDF docs: --------- PASSED: dev-tools: pdf/dev-tools.pdf PASSED: tools: pdf/tools.pdf PASSED: filesystems: pdf/filesystems.pdf PASSED: w1: pdf/w1.pdf PASSED: maintainer: pdf/maintainer.pdf PASSED: process: pdf/process.pdf PASSED: isdn: pdf/isdn.pdf PASSED: fault-injection: pdf/fault-injection.pdf PASSED: iio: pdf/iio.pdf PASSED: scheduler: pdf/scheduler.pdf PASSED: staging: pdf/staging.pdf PASSED: fpga: pdf/fpga.pdf PASSED: power: pdf/power.pdf PASSED: leds: pdf/leds.pdf PASSED: edac: pdf/edac.pdf PASSED: PCI: pdf/PCI.pdf PASSED: firmware-guide: pdf/firmware-guide.pdf PASSED: cpu-freq: pdf/cpu-freq.pdf PASSED: mhi: pdf/mhi.pdf PASSED: wmi: pdf/wmi.pdf PASSED: timers: pdf/timers.pdf PASSED: accel: pdf/accel.pdf PASSED: hid: pdf/hid.pdf PASSED: userspace-api: pdf/userspace-api.pdf PASSED: spi: pdf/spi.pdf PASSED: networking: pdf/networking.pdf PASSED: virt: pdf/virt.pdf PASSED: nvme: pdf/nvme.pdf PASSED: translations: pdf/translations.pdf PASSED: input: pdf/input.pdf PASSED: tee: pdf/tee.pdf PASSED: doc-guide: pdf/doc-guide.pdf PASSED: cdrom: pdf/cdrom.pdf PASSED: gpu: pdf/gpu.pdf PASSED: i2c: pdf/i2c.pdf PASSED: RCU: pdf/RCU.pdf PASSED: watchdog: pdf/watchdog.pdf PASSED: usb: pdf/usb.pdf PASSED: rust: pdf/rust.pdf PASSED: crypto: pdf/crypto.pdf PASSED: kbuild: pdf/kbuild.pdf PASSED: livepatch: pdf/livepatch.pdf PASSED: mm: pdf/mm.pdf PASSED: locking: pdf/locking.pdf PASSED: infiniband: pdf/infiniband.pdf PASSED: driver-api: pdf/driver-api.pdf PASSED: bpf: pdf/bpf.pdf PASSED: devicetree: pdf/devicetree.pdf PASSED: block: pdf/block.pdf PASSED: target: pdf/target.pdf PASSED: arch: pdf/arch.pdf PASSED: pcmcia: pdf/pcmcia.pdf PASSED: scsi: pdf/scsi.pdf PASSED: netlabel: pdf/netlabel.pdf PASSED: sound: pdf/sound.pdf PASSED: security: pdf/security.pdf PASSED: accounting: pdf/accounting.pdf PASSED: admin-guide: pdf/admin-guide.pdf PASSED: core-api: pdf/core-api.pdf PASSED: fb: pdf/fb.pdf PASSED: peci: pdf/peci.pdf PASSED: trace: pdf/trace.pdf PASSED: misc-devices: pdf/misc-devices.pdf PASSED: kernel-hacking: pdf/kernel-hacking.pdf PASSED: hwmon: pdf/hwmon.pdf Reported-by: Akira Yokosawa Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/b5e2e0df68b377b148fdbdd721f6c1cbefe6f861.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index b8474848df4e..b24a6f91ec0a 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -764,9 +764,6 @@ class SphinxDependencyChecker(MissingCheckers): if self.pdf: pdf_pkgs = { - "texlive-lang-chinese": [ - "/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty", - ], "fonts-dejavu": [ "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", ], @@ -775,6 +772,15 @@ class SphinxDependencyChecker(MissingCheckers): "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc", ], + "tex-gyre": [ + "/usr/share/texmf/tex/latex/tex-gyre/tgtermes.sty" + ], + "texlive-fonts-recommended": [ + "/usr/share/texlive/texmf-dist/fonts/tfm/adobe/zapfding/pzdr.tfm", + ], + "texlive-lang-chinese": [ + "/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty", + ], } for package, files in pdf_pkgs.items(): -- cgit From 9ff5c2f51da284906c2f89e7d594f31be69e7abe Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:45 +0200 Subject: scripts: sphinx-pre-install: add missing gentoo pdf dependencies There are two packages that are required to build PDF at gentoo: dev-texlive/texlive-latexextra media-fonts/lm Place latex_dependencies on a list to make it easier to maintain and add the missing ones. With that, most PDF documents now build on Gentoo: Gentoo Base System release 2.17: -------------------------------- PASSED: OS detection: Gentoo Base System release 2.17 SKIPPED (Sphinx Sphinx 8.2.3): System packages SKIPPED (Sphinx already installed either as venv or as native package): Sphinx on venv SKIPPED (Sphinx already installed either as venv or as native package): Sphinx package PASSED: Clean documentation: Build time: 0:00, return code: 0 PASSED: Build HTML documentation: Build time: 5:28, return code: 0 PARTIAL: Build PDF documentation: Test failed (Build time: 9:19, return code: 2) PDF docs: --------- PASSED: dev-tools: pdf/dev-tools.pdf PASSED: tools: pdf/tools.pdf PASSED: filesystems: pdf/filesystems.pdf PASSED: w1: pdf/w1.pdf PASSED: maintainer: pdf/maintainer.pdf PASSED: process: pdf/process.pdf PASSED: isdn: pdf/isdn.pdf PASSED: fault-injection: pdf/fault-injection.pdf PASSED: iio: pdf/iio.pdf PASSED: scheduler: pdf/scheduler.pdf PASSED: staging: pdf/staging.pdf PASSED: fpga: pdf/fpga.pdf PASSED: power: pdf/power.pdf PASSED: leds: pdf/leds.pdf PASSED: edac: pdf/edac.pdf PASSED: PCI: pdf/PCI.pdf PASSED: firmware-guide: pdf/firmware-guide.pdf PASSED: cpu-freq: pdf/cpu-freq.pdf PASSED: mhi: pdf/mhi.pdf PASSED: wmi: pdf/wmi.pdf PASSED: timers: pdf/timers.pdf PASSED: accel: pdf/accel.pdf PASSED: hid: pdf/hid.pdf FAILED: userspace-api: Build failed (FAILED) PASSED: spi: pdf/spi.pdf PASSED: networking: pdf/networking.pdf PASSED: virt: pdf/virt.pdf PASSED: nvme: pdf/nvme.pdf FAILED: translations: Build failed (FAILED) PASSED: input: pdf/input.pdf PASSED: tee: pdf/tee.pdf PASSED: doc-guide: pdf/doc-guide.pdf PASSED: cdrom: pdf/cdrom.pdf FAILED: gpu: Build failed (FAILED) FAILED: i2c: Build failed (FAILED) FAILED: RCU: Build failed (FAILED) PASSED: watchdog: pdf/watchdog.pdf PASSED: usb: pdf/usb.pdf PASSED: rust: pdf/rust.pdf PASSED: crypto: pdf/crypto.pdf PASSED: kbuild: pdf/kbuild.pdf PASSED: livepatch: pdf/livepatch.pdf PASSED: mm: pdf/mm.pdf PASSED: locking: pdf/locking.pdf PASSED: infiniband: pdf/infiniband.pdf PASSED: driver-api: pdf/driver-api.pdf PASSED: bpf: pdf/bpf.pdf PASSED: devicetree: pdf/devicetree.pdf PASSED: block: pdf/block.pdf PASSED: target: pdf/target.pdf FAILED: arch: Build failed (FAILED) PASSED: pcmcia: pdf/pcmcia.pdf PASSED: scsi: pdf/scsi.pdf PASSED: netlabel: pdf/netlabel.pdf PASSED: sound: pdf/sound.pdf PASSED: security: pdf/security.pdf PASSED: accounting: pdf/accounting.pdf PASSED: admin-guide: pdf/admin-guide.pdf FAILED: core-api: Build failed (FAILED) PASSED: fb: pdf/fb.pdf PASSED: peci: pdf/peci.pdf PASSED: trace: pdf/trace.pdf PASSED: misc-devices: pdf/misc-devices.pdf PASSED: kernel-hacking: pdf/kernel-hacking.pdf PASSED: hwmon: pdf/hwmon.pdf Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0ac8d6b7484aaf930917c8edde53742d425e7e8f.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index b24a6f91ec0a..f987abfec802 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -1058,12 +1058,19 @@ class SphinxDependencyChecker(MissingCheckers): """ Provide package installation hints for Gentoo. """ + texlive_deps = [ + "dev-texlive/texlive-latexextra", + "dev-texlive/texlive-xetex", + "media-fonts/dejavu", + "media-fonts/lm", + ] + progs = { "convert": "media-gfx/imagemagick", "dot": "media-gfx/graphviz", "rsvg-convert": "gnome-base/librsvg", "virtualenv": "dev-python/virtualenv", - "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", + "xelatex": " ".join(texlive_deps), "yaml": "dev-python/pyyaml", "python-sphinx": "dev-python/sphinx", } -- cgit From b2d5d61c1371dc6e0ddda69a0a2c921e3ceb928e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:46 +0200 Subject: scripts: sphinx-pre-install: fix PDF dependencies for openSuse The dependencies are outdated: both versions need texlive-dejavu fonts. Also, for PDF generation, python311-Sphinx-latex is required. With that, all PDF files are now tuilt on both: openSUSE Leap 15.6: ------------------- PASSED: OS detection: openSUSE Leap 15.6 SKIPPED (Sphinx Sphinx 7.2.6): System packages SKIPPED (Sphinx already installed either as venv or as native package): Sphinx on venv SKIPPED (Sphinx already installed either as venv or as native package): Sphinx package PASSED: Clean documentation: Build time: 0:00, return code: 0 PASSED: Build HTML documentation: Build time: 5:29, return code: 0 PASSED: Build PDF documentation: Build time: 13:45, return code: 0 openSUSE Tumbleweed: -------------------- PASSED: OS detection: openSUSE Tumbleweed SKIPPED (Sphinx Sphinx 8.2.3): System packages SKIPPED (Sphinx already installed either as venv or as native package): Sphinx on venv SKIPPED (Sphinx already installed either as venv or as native package): Sphinx package PASSED: Clean documentation: Build time: 0:00, return code: 0 PASSED: Build HTML documentation: Build time: 4:33, return code: 0 PASSED: Build PDF documentation: Build time: 13:18, return code: 0 Summary ======= PASSED - openSUSE Leap 15.6 (7 tests) PASSED - openSUSE Tumbleweed (7 tests) Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/d78457376f9dfd24cb7ac3a32895c654412715f3.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index f987abfec802..86f129c76ecd 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -901,7 +901,7 @@ class SphinxDependencyChecker(MissingCheckers): "dot": "graphviz", "python-sphinx": "python3-sphinx", "virtualenv": "python3-virtualenv", - "xelatex": "texlive-xetex-bin", + "xelatex": "texlive-xetex-bin texlive-dejavu", "yaml": "python3-pyyaml", } @@ -937,7 +937,7 @@ class SphinxDependencyChecker(MissingCheckers): self.recommend_python = True progs.update({ - "python-sphinx": "python311-Sphinx", + "python-sphinx": "python311-Sphinx python311-Sphinx-latex", "virtualenv": "python311-virtualenv", "yaml": "python311-PyYAML", }) @@ -945,7 +945,7 @@ class SphinxDependencyChecker(MissingCheckers): # Tumbleweed defaults to Python 3.11 progs.update({ - "python-sphinx": "python313-Sphinx", + "python-sphinx": "python313-Sphinx python313-Sphinx-latex", "virtualenv": "python313-virtualenv", "yaml": "python313-PyYAML", }) -- cgit From b51f8c12d16bcf29496ebaf1d7cf3587ca28ba0a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:47 +0200 Subject: scripts: sphinx-pre-install: fix dependencies for OpenMandriva The dependeny list for OpenMandriva is wrong. Update it. Yet, on my tests with OpenMandriva LX 4.3, the texlive packages are broken: xelatex can't build anything there, as it lacks xelatex.sfm. Yet, this could be a problem at the way I created the container. Just in case, add a note about that. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/669e759ba366328e5c8d5b14a591ba45a1f58176.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 86f129c76ecd..224db3af17db 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -984,13 +984,19 @@ class SphinxDependencyChecker(MissingCheckers): if re.search(r"OpenMandriva", self.system_release): packager_cmd = "dnf install" noto_sans = "noto-sans-cjk-fonts" - tex_pkgs = ["texlive-collection-fontsextra"] + tex_pkgs = [ + "texlive-collection-basic", + "texlive-collection-langcjk", + "texlive-collection-fontsextra", + "texlive-collection-fontsrecommended" + ] # Tested on OpenMandriva Lx 4.3 progs["convert"] = "imagemagick" progs["yaml"] = "python-pyyaml" progs["python-virtualenv"] = "python-virtualenv" progs["python-sphinx"] = "python-sphinx" + progs["xelatex"] = "texlive" self.check_program("python-virtualenv", DepManager.PYTHON_MANDATORY) @@ -1004,7 +1010,9 @@ class SphinxDependencyChecker(MissingCheckers): if not self.distro_msg: self.distro_msg = \ - "Note: for venv, ensurepip could be broken, preventing its install method." + "Notes:\n"\ + "1. for venv, ensurepip could be broken, preventing its install method.\n" \ + "2. at least on OpenMandriva LX 4.3, texlive packages seem broken" else: packager_cmd = "urpmi" -- cgit From c71c5d6dcb34423c0f15c146240b25a0e78f65aa Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:48 +0200 Subject: scripts: sphinx-pre-install: fix pdf dependencies for Mageia 9 On Mageia 9, two packages are missing. Add them. With that, all PDF packages now build: Mageia 9: --------- PASSED: OS detection: Mageia 9 PASSED: System packages: Packages installed PASSED: Sphinx on venv: Sphinx Sphinx 8.1.3 PASSED: Sphinx package: Sphinx Sphinx 6.1.3 PASSED: Clean documentation: Build time: 0:00, return code: 0 PASSED: Build HTML documentation: Build time: 5:17, return code: 0 PASSED: Build PDF documentation: Build time: 14:28, return code: 0 PDF docs: --------- PASSED: dev-tools: pdf/dev-tools.pdf PASSED: tools: pdf/tools.pdf PASSED: filesystems: pdf/filesystems.pdf PASSED: w1: pdf/w1.pdf PASSED: maintainer: pdf/maintainer.pdf PASSED: process: pdf/process.pdf PASSED: isdn: pdf/isdn.pdf PASSED: fault-injection: pdf/fault-injection.pdf PASSED: iio: pdf/iio.pdf PASSED: scheduler: pdf/scheduler.pdf PASSED: staging: pdf/staging.pdf PASSED: fpga: pdf/fpga.pdf PASSED: power: pdf/power.pdf PASSED: leds: pdf/leds.pdf PASSED: edac: pdf/edac.pdf PASSED: PCI: pdf/PCI.pdf PASSED: firmware-guide: pdf/firmware-guide.pdf PASSED: cpu-freq: pdf/cpu-freq.pdf PASSED: mhi: pdf/mhi.pdf PASSED: wmi: pdf/wmi.pdf PASSED: timers: pdf/timers.pdf PASSED: accel: pdf/accel.pdf PASSED: hid: pdf/hid.pdf PASSED: userspace-api: pdf/userspace-api.pdf PASSED: spi: pdf/spi.pdf PASSED: networking: pdf/networking.pdf PASSED: virt: pdf/virt.pdf PASSED: nvme: pdf/nvme.pdf PASSED: translations: pdf/translations.pdf PASSED: input: pdf/input.pdf PASSED: tee: pdf/tee.pdf PASSED: doc-guide: pdf/doc-guide.pdf PASSED: cdrom: pdf/cdrom.pdf PASSED: gpu: pdf/gpu.pdf PASSED: i2c: pdf/i2c.pdf PASSED: RCU: pdf/RCU.pdf PASSED: watchdog: pdf/watchdog.pdf PASSED: usb: pdf/usb.pdf PASSED: rust: pdf/rust.pdf PASSED: crypto: pdf/crypto.pdf PASSED: kbuild: pdf/kbuild.pdf PASSED: livepatch: pdf/livepatch.pdf PASSED: mm: pdf/mm.pdf PASSED: locking: pdf/locking.pdf PASSED: infiniband: pdf/infiniband.pdf PASSED: driver-api: pdf/driver-api.pdf PASSED: bpf: pdf/bpf.pdf PASSED: devicetree: pdf/devicetree.pdf PASSED: block: pdf/block.pdf PASSED: target: pdf/target.pdf PASSED: arch: pdf/arch.pdf PASSED: pcmcia: pdf/pcmcia.pdf PASSED: scsi: pdf/scsi.pdf PASSED: netlabel: pdf/netlabel.pdf PASSED: sound: pdf/sound.pdf PASSED: security: pdf/security.pdf PASSED: accounting: pdf/accounting.pdf PASSED: admin-guide: pdf/admin-guide.pdf PASSED: core-api: pdf/core-api.pdf PASSED: fb: pdf/fb.pdf PASSED: peci: pdf/peci.pdf PASSED: trace: pdf/trace.pdf PASSED: misc-devices: pdf/misc-devices.pdf PASSED: kernel-hacking: pdf/kernel-hacking.pdf PASSED: hwmon: pdf/hwmon.pdf Summary ======= PASSED - Mageia 9 (7 tests) Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/bd6e03c79b890ad0168493cdb4cdaf610bbc8c45.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 224db3af17db..758a84ae6347 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -979,6 +979,8 @@ class SphinxDependencyChecker(MissingCheckers): tex_pkgs = [ "texlive-fontsextra", + "texlive-fonts-asian", + "fonts-ttf-dejavu", ] if re.search(r"OpenMandriva", self.system_release): -- cgit From 4509d36ceea288a78bb256416d9ca38d3298c7cb Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:49 +0200 Subject: scripts: sphinx-pre-install: fix PDF dependencies for gentoo Package fonts are wrong. Fix it. With that, most PDF files now builds. PDF docs: --------- PASSED: dev-tools: pdf/dev-tools.pdf PASSED: tools: pdf/tools.pdf PASSED: filesystems: pdf/filesystems.pdf PASSED: w1: pdf/w1.pdf PASSED: maintainer: pdf/maintainer.pdf PASSED: process: pdf/process.pdf PASSED: isdn: pdf/isdn.pdf PASSED: fault-injection: pdf/fault-injection.pdf PASSED: iio: pdf/iio.pdf PASSED: scheduler: pdf/scheduler.pdf PASSED: staging: pdf/staging.pdf PASSED: fpga: pdf/fpga.pdf PASSED: power: pdf/power.pdf PASSED: leds: pdf/leds.pdf PASSED: edac: pdf/edac.pdf PASSED: PCI: pdf/PCI.pdf PASSED: firmware-guide: pdf/firmware-guide.pdf PASSED: cpu-freq: pdf/cpu-freq.pdf PASSED: mhi: pdf/mhi.pdf PASSED: wmi: pdf/wmi.pdf PASSED: timers: pdf/timers.pdf PASSED: accel: pdf/accel.pdf PASSED: hid: pdf/hid.pdf FAILED: userspace-api: Build failed (FAILED) PASSED: spi: pdf/spi.pdf PASSED: networking: pdf/networking.pdf PASSED: virt: pdf/virt.pdf PASSED: nvme: pdf/nvme.pdf FAILED: translations: Build failed (FAILED) PASSED: input: pdf/input.pdf PASSED: tee: pdf/tee.pdf PASSED: doc-guide: pdf/doc-guide.pdf PASSED: cdrom: pdf/cdrom.pdf FAILED: gpu: Build failed (FAILED) FAILED: i2c: Build failed (FAILED) FAILED: RCU: Build failed (FAILED) PASSED: watchdog: pdf/watchdog.pdf PASSED: usb: pdf/usb.pdf PASSED: rust: pdf/rust.pdf PASSED: crypto: pdf/crypto.pdf PASSED: kbuild: pdf/kbuild.pdf PASSED: livepatch: pdf/livepatch.pdf PASSED: mm: pdf/mm.pdf PASSED: locking: pdf/locking.pdf PASSED: infiniband: pdf/infiniband.pdf PASSED: driver-api: pdf/driver-api.pdf PASSED: bpf: pdf/bpf.pdf PASSED: devicetree: pdf/devicetree.pdf PASSED: block: pdf/block.pdf PASSED: target: pdf/target.pdf FAILED: arch: Build failed (FAILED) PASSED: pcmcia: pdf/pcmcia.pdf PASSED: scsi: pdf/scsi.pdf PASSED: netlabel: pdf/netlabel.pdf PASSED: sound: pdf/sound.pdf PASSED: security: pdf/security.pdf PASSED: accounting: pdf/accounting.pdf PASSED: admin-guide: pdf/admin-guide.pdf FAILED: core-api: Build failed (FAILED) PASSED: fb: pdf/fb.pdf PASSED: peci: pdf/peci.pdf PASSED: trace: pdf/trace.pdf PASSED: misc-devices: pdf/misc-devices.pdf PASSED: kernel-hacking: pdf/kernel-hacking.pdf PASSED: hwmon: pdf/hwmon.pdf Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/1ccbac9fd1f4e598dda82e775b64768ec3696248.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 758a84ae6347..c46d7b76f93c 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -1069,10 +1069,10 @@ class SphinxDependencyChecker(MissingCheckers): Provide package installation hints for Gentoo. """ texlive_deps = [ + "dev-texlive/texlive-fontsrecommended", "dev-texlive/texlive-latexextra", "dev-texlive/texlive-xetex", "media-fonts/dejavu", - "media-fonts/lm", ] progs = { -- cgit From c6e23912855d4848883080200e09551b6dcbc7df Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:50 +0200 Subject: scripts/sphinx-pre-install: fix Archlinux PDF dependencies There are some missing packages causing PDF build to fail on Archlinux and add latexmk (from texlive-binextra package). Yet, at least today, PDF builds are failing on a very late stage, when trying to run xdvipdfmx: $ xdvipdfmx -E -o "peci.pdf" "peci.xdv" xdvipdfmx:fatal: Unrecognized paper format: # Simply write the paper name. See man 1 paper and "paper --no-size --all" for possible values Despite its message, even using a very simple document like: \def\sphinxdocclass{report} \documentclass[a4paper,11pt,english]{sphinxmanual} \begin{document} Test \end{document} or even: \def\sphinxdocclass{report} \documentclass{sphinxmanual} \begin{document} Test \end{document} Is causing xdvipdfmx to complain about geometry. As Archlinux is a rolling release distro, maybe I got it on a bad day. So, let's fix it in the hope that soon enough someone would fix the issues there. Such broken scenario happens with those packages installed: texlive-basic 2025.2-1 texlive-bin 2025.2-1 texlive-binextra 2025.2-1 texlive-fontsrecommended 2025.2-1 texlive-langchinese 2025.2-1 texlive-langcjk 2025.2-1 texlive-latex 2025.2-1 texlive-latexextra 2025.2-1 texlive-latexrecommended 2025.2-1 texlive-pictures 2025.2-1 texlive-xetex 2025.2-1 python-docutils 1:0.21.2-3 python-sphinx 8.2.3-1 python-sphinx-alabaster-theme 1.0.0-4 python-sphinxcontrib-applehelp 2.0.0-3 python-sphinxcontrib-devhelp 2.0.0-4 python-sphinxcontrib-htmlhelp 2.1.0-3 python-sphinxcontrib-jsmath 1.0.1-19 python-sphinxcontrib-qthelp 2.0.0-3 python-sphinxcontrib-serializinghtml 2.0.0-3 Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/574d902f7691861e18339217f42409850ee58791.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index c46d7b76f93c..954ed3dc0645 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -1048,7 +1048,12 @@ class SphinxDependencyChecker(MissingCheckers): } archlinux_tex_pkgs = [ + "texlive-basic", + "texlive-binextra", "texlive-core", + "texlive-fontsrecommended", + "texlive-langchinese", + "texlive-langcjk", "texlive-latexextra", "ttf-dejavu", ] -- cgit From 8a298579cdfcfdbcb70bb7af5e30769387494f37 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:35 +0200 Subject: scripts: sphinx-build-wrapper: get rid of uapi/media Makefile Now that kernel-include directive supports parsing data structs directly, we can finally get rid of the horrible hack we added to support parsing media uAPI symbols. As a side effect, Documentation/output doesn't have anymore media auto-generated .rst files on it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/5dbb257a4b283697271c9c7b8f4713857e8191c8.1755872208.git.mchehab+huawei@kernel.org --- scripts/sphinx-build-wrapper | 719 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 719 insertions(+) create mode 100755 scripts/sphinx-build-wrapper (limited to 'scripts') diff --git a/scripts/sphinx-build-wrapper b/scripts/sphinx-build-wrapper new file mode 100755 index 000000000000..abe8c26ae137 --- /dev/null +++ b/scripts/sphinx-build-wrapper @@ -0,0 +1,719 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2025 Mauro Carvalho Chehab +# +# pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103 +# +# Converted from docs Makefile and parallel-wrapper.sh, both under +# GPLv2, copyrighted since 2008 by the following authors: +# +# Akira Yokosawa +# Arnd Bergmann +# Breno Leitao +# Carlos Bilbao +# Dave Young +# Donald Hunter +# Geert Uytterhoeven +# Jani Nikula +# Jan Stancek +# Jonathan Corbet +# Joshua Clayton +# Kees Cook +# Linus Torvalds +# Magnus Damm +# Masahiro Yamada +# Mauro Carvalho Chehab +# Maxim Cournoyer +# Peter Foley +# Randy Dunlap +# Rob Herring +# Shuah Khan +# Thorsten Blum +# Tomas Winkler + + +""" +Sphinx build wrapper that handles Kernel-specific business rules: + +- it gets the Kernel build environment vars; +- it determines what's the best parallelism; +- it handles SPHINXDIRS + +This tool ensures that MIN_PYTHON_VERSION is satisfied. If version is +below that, it seeks for a new Python version. If found, it re-runs using +the newer version. +""" + +import argparse +import locale +import os +import re +import shlex +import shutil +import subprocess +import sys + +from concurrent import futures +from glob import glob + +LIB_DIR = "lib" +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) + +sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) + +from jobserver import JobserverExec # pylint: disable=C0413 + + +def parse_version(version): + """Convert a major.minor.patch version into a tuple""" + return tuple(int(x) for x in version.split(".")) + +def ver_str(version): + """Returns a version tuple as major.minor.patch""" + + return ".".join([str(x) for x in version]) + +# Minimal supported Python version needed by Sphinx and its extensions +MIN_PYTHON_VERSION = parse_version("3.7") + +# Default value for --venv parameter +VENV_DEFAULT = "sphinx_latest" + +# List of make targets and its corresponding builder and output directory +TARGETS = { + "cleandocs": { + "builder": "clean", + }, + "htmldocs": { + "builder": "html", + }, + "epubdocs": { + "builder": "epub", + "out_dir": "epub", + }, + "texinfodocs": { + "builder": "texinfo", + "out_dir": "texinfo", + }, + "infodocs": { + "builder": "texinfo", + "out_dir": "texinfo", + }, + "latexdocs": { + "builder": "latex", + "out_dir": "latex", + }, + "pdfdocs": { + "builder": "latex", + "out_dir": "latex", + }, + "xmldocs": { + "builder": "xml", + "out_dir": "xml", + }, + "linkcheckdocs": { + "builder": "linkcheck" + }, +} + +# Paper sizes. An empty value will pick the default +PAPER = ["", "a4", "letter"] + +class SphinxBuilder: + """ + Handles a sphinx-build target, adding needed arguments to build + with the Kernel. + """ + + def is_rust_enabled(self): + """Check if rust is enabled at .config""" + config_path = os.path.join(self.srctree, ".config") + if os.path.isfile(config_path): + with open(config_path, "r", encoding="utf-8") as f: + return "CONFIG_RUST=y" in f.read() + return False + + def get_path(self, path, abs_path=False): + """ + Ancillary routine to handle patches the right way, as shell does. + + It first expands "~" and "~user". Then, if patch is not absolute, + join self.srctree. Finally, if requested, convert to abspath. + """ + + path = os.path.expanduser(path) + if not path.startswith("/"): + path = os.path.join(self.srctree, path) + + if abs_path: + return os.path.abspath(path) + + return path + + def __init__(self, venv=None, verbose=False, n_jobs=None, interactive=None): + """Initialize internal variables""" + self.venv = venv + self.verbose = None + + # Normal variables passed from Kernel's makefile + self.kernelversion = os.environ.get("KERNELVERSION", "unknown") + self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown") + self.pdflatex = os.environ.get("PDFLATEX", "xelatex") + + if not interactive: + self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape") + else: + self.latexopts = os.environ.get("LATEXOPTS", "") + + if not verbose: + verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "") + + # Handle SPHINXOPTS evironment + sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) + + # As we handle number of jobs and quiet in separate, we need to pick + # it the same way as sphinx-build would pick, so let's use argparse + # do to the right argument expansion + parser = argparse.ArgumentParser() + parser.add_argument('-j', '--jobs', type=int) + parser.add_argument('-q', '--quiet', type=int) + + # Other sphinx-build arguments go as-is, so place them + # at self.sphinxopts + sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) + if sphinx_args.quiet == True: + self.verbose = False + + if sphinx_args.jobs: + self.n_jobs = sphinx_args.jobs + + # Command line arguments was passed, override SPHINXOPTS + if verbose is not None: + self.verbose = verbose + + self.n_jobs = n_jobs + + # Source tree directory. This needs to be at os.environ, as + # Sphinx extensions and media uAPI makefile needs it + self.srctree = os.environ.get("srctree") + if not self.srctree: + self.srctree = "." + os.environ["srctree"] = self.srctree + + # Now that we can expand srctree, get other directories as well + self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build") + self.kerneldoc = self.get_path(os.environ.get("KERNELDOC", + "scripts/kernel-doc.py")) + self.obj = os.environ.get("obj", "Documentation") + self.builddir = self.get_path(os.path.join(self.obj, "output"), + abs_path=True) + + # Media uAPI needs it + os.environ["BUILDDIR"] = self.builddir + + # Detect if rust is enabled + self.config_rust = self.is_rust_enabled() + + # Get directory locations for LaTeX build toolchain + self.pdflatex_cmd = shutil.which(self.pdflatex) + self.latexmk_cmd = shutil.which("latexmk") + + self.env = os.environ.copy() + + # If venv parameter is specified, run Sphinx from venv + if venv: + bin_dir = os.path.join(venv, "bin") + if os.path.isfile(os.path.join(bin_dir, "activate")): + # "activate" virtual env + self.env["PATH"] = bin_dir + ":" + self.env["PATH"] + self.env["VIRTUAL_ENV"] = venv + if "PYTHONHOME" in self.env: + del self.env["PYTHONHOME"] + print(f"Setting venv to {venv}") + else: + sys.exit(f"Venv {venv} not found.") + + def run_sphinx(self, sphinx_build, build_args, *args, **pwargs): + """ + Executes sphinx-build using current python3 command and setting + -j parameter if possible to run the build in parallel. + """ + + with JobserverExec() as jobserver: + if jobserver.claim: + n_jobs = str(jobserver.claim) + else: + n_jobs = "auto" # Supported since Sphinx 1.7 + + cmd = [] + + if self.venv: + cmd.append("python") + else: + cmd.append(sys.executable) + + cmd.append(sphinx_build) + + # if present, SPHINXOPTS or command line --jobs overrides default + if self.n_jobs: + n_jobs = str(self.n_jobs) + + if n_jobs: + cmd += [f"-j{n_jobs}"] + + if not self.verbose: + cmd.append("-q") + + cmd += self.sphinxopts + + cmd += build_args + + if self.verbose: + print(" ".join(cmd)) + + rc = subprocess.call(cmd, *args, **pwargs) + + def handle_html(self, css, output_dir): + """ + Extra steps for HTML and epub output. + + For such targets, we need to ensure that CSS will be properly + copied to the output _static directory + """ + + if not css: + return + + css = os.path.expanduser(css) + if not css.startswith("/"): + css = os.path.join(self.srctree, css) + + static_dir = os.path.join(output_dir, "_static") + os.makedirs(static_dir, exist_ok=True) + + try: + shutil.copy2(css, static_dir) + except (OSError, IOError) as e: + print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) + + def build_pdf_file(self, latex_cmd, from_dir, path): + """Builds a single pdf file using latex_cmd""" + try: + subprocess.run(latex_cmd + [path], + cwd=from_dir, check=True) + + return True + except subprocess.CalledProcessError: + # LaTeX PDF error code is almost useless: it returns + # error codes even when build succeeds but has warnings. + # So, we'll ignore the results + return False + + def pdf_parallel_build(self, tex_suffix, latex_cmd, tex_files, n_jobs): + """Build PDF files in parallel if possible""" + builds = {} + build_failed = False + max_len = 0 + has_tex = False + + # Process files in parallel + with futures.ThreadPoolExecutor(max_workers=n_jobs) as executor: + jobs = {} + + for from_dir, pdf_dir, entry in tex_files: + name = entry.name + + if not name.endswith(tex_suffix): + continue + + name = name[:-len(tex_suffix)] + + max_len = max(max_len, len(name)) + + has_tex = True + + future = executor.submit(self.build_pdf_file, latex_cmd, + from_dir, entry.path) + jobs[future] = (from_dir, name, entry.path) + + for future in futures.as_completed(jobs): + from_dir, name, path = jobs[future] + + pdf_name = name + ".pdf" + pdf_from = os.path.join(from_dir, pdf_name) + + try: + success = future.result() + + if success and os.path.exists(pdf_from): + pdf_to = os.path.join(pdf_dir, pdf_name) + + os.rename(pdf_from, pdf_to) + builds[name] = os.path.relpath(pdf_to, self.builddir) + else: + builds[name] = "FAILED" + build_failed = True + except Exception as e: + builds[name] = f"FAILED ({str(e)})" + build_failed = True + + # Handle case where no .tex files were found + if not has_tex: + name = "Sphinx LaTeX builder" + max_len = max(max_len, len(name)) + builds[name] = "FAILED (no .tex file was generated)" + build_failed = True + + return builds, build_failed, max_len + + def handle_pdf(self, output_dirs): + """ + Extra steps for PDF output. + + As PDF is handled via a LaTeX output, after building the .tex file, + a new build is needed to create the PDF output from the latex + directory. + """ + builds = {} + max_len = 0 + tex_suffix = ".tex" + + # Get all tex files that will be used for PDF build + tex_files = [] + for from_dir in output_dirs: + pdf_dir = os.path.join(from_dir, "../pdf") + os.makedirs(pdf_dir, exist_ok=True) + + if self.latexmk_cmd: + latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"] + else: + latex_cmd = [self.pdflatex] + + latex_cmd.extend(shlex.split(self.latexopts)) + + # Get a list of tex files to process + with os.scandir(from_dir) as it: + for entry in it: + if entry.name.endswith(tex_suffix): + tex_files.append((from_dir, pdf_dir, entry)) + + # When using make, this won't be used, as the number of jobs comes + # from POSIX jobserver. So, this covers the case where build comes + # from command line. On such case, serialize by default, except if + # the user explicitly sets the number of jobs. + n_jobs = 1 + + # n_jobs is either an integer or "auto". Only use it if it is a number + if self.n_jobs: + try: + n_jobs = int(self.n_jobs) + except ValueError: + pass + + # When using make, jobserver.claim is the number of jobs that were + # used with "-j" and that aren't used by other make targets + with JobserverExec() as jobserver: + n_jobs = 1 + + # Handle the case when a parameter is passed via command line, + # using it as default, if jobserver doesn't claim anything + if self.n_jobs: + try: + n_jobs = int(self.n_jobs) + except ValueError: + pass + + if jobserver.claim: + n_jobs = jobserver.claim + + # Build files in parallel + builds, build_failed, max_len = self.pdf_parallel_build(tex_suffix, + latex_cmd, + tex_files, + n_jobs) + + msg = "Summary" + msg += "\n" + "=" * len(msg) + print() + print(msg) + + for pdf_name, pdf_file in builds.items(): + print(f"{pdf_name:<{max_len}}: {pdf_file}") + + print() + + # return an error if a PDF file is missing + + if build_failed: + sys.exit(f"PDF build failed: not all PDF files were created.") + else: + print("All PDF files were built.") + + def handle_info(self, output_dirs): + """ + Extra steps for Info output. + + For texinfo generation, an additional make is needed from the + texinfo directory. + """ + + for output_dir in output_dirs: + try: + subprocess.run(["make", "info"], cwd=output_dir, check=True) + except subprocess.CalledProcessError as e: + sys.exit(f"Error generating info docs: {e}") + + def cleandocs(self, builder): + + shutil.rmtree(self.builddir, ignore_errors=True) + + def build(self, target, sphinxdirs=None, conf="conf.py", + theme=None, css=None, paper=None): + """ + Build documentation using Sphinx. This is the core function of this + module. It prepares all arguments required by sphinx-build. + """ + + builder = TARGETS[target]["builder"] + out_dir = TARGETS[target].get("out_dir", "") + + # Cleandocs doesn't require sphinx-build + if target == "cleandocs": + self.cleandocs(builder) + return + + # Other targets require sphinx-build + sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) + if not sphinxbuild: + sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") + + if builder == "latex": + if not self.pdflatex_cmd and not self.latexmk_cmd: + sys.exit("Error: pdflatex or latexmk required for PDF generation") + + docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation")) + + # Prepare base arguments for Sphinx build + kerneldoc = self.kerneldoc + if kerneldoc.startswith(self.srctree): + kerneldoc = os.path.relpath(kerneldoc, self.srctree) + + # Prepare common Sphinx options + args = [ + "-b", builder, + "-c", docs_dir, + ] + + if builder == "latex": + if not paper: + paper = PAPER[1] + + args.extend(["-D", f"latex_elements.papersize={paper}paper"]) + + if self.config_rust: + args.extend(["-t", "rustdoc"]) + + if conf: + self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True) + + if not sphinxdirs: + sphinxdirs = os.environ.get("SPHINXDIRS", ".") + + # The sphinx-build tool has a bug: internally, it tries to set + # locale with locale.setlocale(locale.LC_ALL, ''). This causes a + # crash if language is not set. Detect and fix it. + try: + locale.setlocale(locale.LC_ALL, '') + except Exception: + self.env["LC_ALL"] = "C" + self.env["LANG"] = "C" + + # sphinxdirs can be a list or a whitespace-separated string + sphinxdirs_list = [] + for sphinxdir in sphinxdirs: + if isinstance(sphinxdir, list): + sphinxdirs_list += sphinxdir + else: + for name in sphinxdir.split(" "): + sphinxdirs_list.append(name) + + # Build each directory + output_dirs = [] + for sphinxdir in sphinxdirs_list: + src_dir = os.path.join(docs_dir, sphinxdir) + doctree_dir = os.path.join(self.builddir, ".doctrees") + output_dir = os.path.join(self.builddir, sphinxdir, out_dir) + + # Make directory names canonical + src_dir = os.path.normpath(src_dir) + doctree_dir = os.path.normpath(doctree_dir) + output_dir = os.path.normpath(output_dir) + + os.makedirs(doctree_dir, exist_ok=True) + os.makedirs(output_dir, exist_ok=True) + + output_dirs.append(output_dir) + + build_args = args + [ + "-d", doctree_dir, + "-D", f"kerneldoc_bin={kerneldoc}", + "-D", f"version={self.kernelversion}", + "-D", f"release={self.kernelrelease}", + "-D", f"kerneldoc_srctree={self.srctree}", + src_dir, + output_dir, + ] + + # Execute sphinx-build + try: + self.run_sphinx(sphinxbuild, build_args, env=self.env) + except Exception as e: + sys.exit(f"Build failed: {e}") + + # Ensure that html/epub will have needed static files + if target in ["htmldocs", "epubdocs"]: + self.handle_html(css, output_dir) + + # PDF and Info require a second build step + if target == "pdfdocs": + self.handle_pdf(output_dirs) + elif target == "infodocs": + self.handle_info(output_dirs) + + @staticmethod + def get_python_version(cmd): + """ + Get python version from a Python binary. As we need to detect if + are out there newer python binaries, we can't rely on sys.release here. + """ + + result = subprocess.run([cmd, "--version"], check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + version = result.stdout.strip() + + match = re.search(r"(\d+\.\d+\.\d+)", version) + if match: + return parse_version(match.group(1)) + + print(f"Can't parse version {version}") + return (0, 0, 0) + + @staticmethod + def find_python(): + """ + Detect if are out there any python 3.xy version newer than the + current one. + + Note: this routine is limited to up to 2 digits for python3. We + may need to update it one day, hopefully on a distant future. + """ + patterns = [ + "python3.[0-9]", + "python3.[0-9][0-9]", + ] + + # Seek for a python binary newer than MIN_PYTHON_VERSION + for path in os.getenv("PATH", "").split(":"): + for pattern in patterns: + for cmd in glob(os.path.join(path, pattern)): + if os.path.isfile(cmd) and os.access(cmd, os.X_OK): + version = SphinxBuilder.get_python_version(cmd) + if version >= MIN_PYTHON_VERSION: + return cmd + + return None + + @staticmethod + def check_python(): + """ + Check if the current python binary satisfies our minimal requirement + for Sphinx build. If not, re-run with a newer version if found. + """ + cur_ver = sys.version_info[:3] + if cur_ver >= MIN_PYTHON_VERSION: + return + + python_ver = ver_str(cur_ver) + + new_python_cmd = SphinxBuilder.find_python() + if not new_python_cmd: + sys.exit(f"Python version {python_ver} is not supported anymore.") + + # Restart script using the newer version + script_path = os.path.abspath(sys.argv[0]) + args = [new_python_cmd, script_path] + sys.argv[1:] + + print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") + + try: + os.execv(new_python_cmd, args) + except OSError as e: + sys.exit(f"Failed to restart with {new_python_cmd}: {e}") + +def jobs_type(value): + """ + Handle valid values for -j. Accepts Sphinx "-jauto", plus a number + equal or bigger than one. + """ + if value is None: + return None + + if value.lower() == 'auto': + return value.lower() + + try: + if int(value) >= 1: + return value + + raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}") + except ValueError: + raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") + +def main(): + """ + Main function. The only mandatory argument is the target. If not + specified, the other arguments will use default values if not + specified at os.environ. + """ + parser = argparse.ArgumentParser(description="Kernel documentation builder") + + parser.add_argument("target", choices=list(TARGETS.keys()), + help="Documentation target to build") + parser.add_argument("--sphinxdirs", nargs="+", + help="Specific directories to build") + parser.add_argument("--conf", default="conf.py", + help="Sphinx configuration file") + + parser.add_argument("--theme", help="Sphinx theme to use") + + parser.add_argument("--css", help="Custom CSS file for HTML/EPUB") + + parser.add_argument("--paper", choices=PAPER, default=PAPER[0], + help="Paper size for LaTeX/PDF output") + + parser.add_argument("-v", "--verbose", action='store_true', + help="place build in verbose mode") + + parser.add_argument('-j', '--jobs', type=jobs_type, + help="Sets number of jobs to use with sphinx-build") + + parser.add_argument('-i', '--interactive', action='store_true', + help="Change latex default to run in interactive mode") + + parser.add_argument("-V", "--venv", nargs='?', const=f'{VENV_DEFAULT}', + default=None, + help=f'If used, run Sphinx from a venv dir (default dir: {VENV_DEFAULT})') + + args = parser.parse_args() + + SphinxBuilder.check_python() + + builder = SphinxBuilder(venv=args.venv, verbose=args.verbose, + n_jobs=args.jobs, interactive=args.interactive) + + builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf, + theme=args.theme, css=args.css, paper=args.paper) + +if __name__ == "__main__": + main() -- cgit From 8b00d6fe96960aaba1b923d4a8c1ddb173c9c1ff Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 4 Sep 2025 13:33:56 -0600 Subject: docs: kdoc: trim __cacheline_group_* with the other annotations The special case for __cacheline_group_begin/end() can be handled by just adding another pattern to the struct_prefixes, eliminating the need for a special case in push_parameter(). One change is that these annotations no longer appear in the rendered output, just like all the other annotations that we clean out. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index a560546c1867..a90f77d6b669 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -90,6 +90,7 @@ struct_prefixes = [ (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), (KernRe(r'\s*____cacheline_aligned', re.S), ' '), + (KernRe(r'\s*__cacheline_group_(begin|end)\([^\)]+\);'), ''), # # Unwrap struct_group macros based on this definition: # __struct_group(TAG, NAME, ATTRS, MEMBERS...) @@ -447,12 +448,6 @@ class KernelDoc: self.entry.parameterdescs[param] = "anonymous\n" self.entry.anon_struct_union = True - # Handle cache group enforcing variables: they do not need - # to be described in header files - elif "__cacheline_group" in param: - # Ignore __cacheline_group_begin and __cacheline_group_end - return - # Warn if parameter has no description # (but ignore ones starting with # as these are not parameters # but inline preprocessor statements) -- cgit From e214cca38f1f35d42e63e990c610c96f993343c4 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 4 Sep 2025 14:01:20 -0600 Subject: docs: kdoc: tighten up the push_parameter() no-type case The handling of untyped parameters involved a number of redundant tests; restructure the code to remove them and be more compact. No output changes. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 44 +++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 24 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index a90f77d6b669..2118c20b3056 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -423,30 +423,26 @@ class KernelDoc: param = KernRe(r'[\[\)].*').sub('', param, count=1) - if dtype == "" and param.endswith("..."): - if KernRe(r'\w\.\.\.$').search(param): - # For named variable parameters of the form `x...`, - # remove the dots - param = param[:-3] - else: - # Handles unnamed variable parameters - param = "..." - - if param not in self.entry.parameterdescs or \ - not self.entry.parameterdescs[param]: - - self.entry.parameterdescs[param] = "variable arguments" - - elif dtype == "" and (not param or param == "void"): - param = "void" - self.entry.parameterdescs[param] = "no arguments" - - elif dtype == "" and param in ["struct", "union"]: - # Handle unnamed (anonymous) union or struct - dtype = param - param = "{unnamed_" + param + "}" - self.entry.parameterdescs[param] = "anonymous\n" - self.entry.anon_struct_union = True + # + # Look at various "anonymous type" cases. + # + if dtype == '': + if param.endswith("..."): + if len(param) > 3: # there is a name provided, use that + param = param[:-3] + if not self.entry.parameterdescs.get(param): + self.entry.parameterdescs[param] = "variable arguments" + + elif (not param) or param == "void": + param = "void" + self.entry.parameterdescs[param] = "no arguments" + + elif param in ["struct", "union"]: + # Handle unnamed (anonymous) union or struct + dtype = param + param = "{unnamed_" + param + "}" + self.entry.parameterdescs[param] = "anonymous\n" + self.entry.anon_struct_union = True # Warn if parameter has no description # (but ignore ones starting with # as these are not parameters -- cgit From f853e83006ab39c3dafe085a488c14bb46906601 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 4 Sep 2025 14:27:00 -0600 Subject: docs: kdoc: remove a single-use variable struct_attribute is only used once, so just put its value there directly and drop the name. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 2118c20b3056..b25c8d80b965 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -76,13 +76,11 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' # Here begins a long set of transformations to turn structure member prefixes # and macro invocations into something we can parse and generate kdoc for. # -struct_attribute = KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", - flags=re.I | re.S, cache=False) struct_args_pattern = r'([^,)]+)' struct_prefixes = [ # Strip attributes - (struct_attribute, ' '), + (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), -- cgit From 4c232a81b0831e7bfa7518968e431d5db29b2cac Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 4 Sep 2025 16:40:49 -0600 Subject: docs: kdoc: move the function transform patterns out of dump_function() Move these definitions to file level, where they are executed once, and don't clutter the function itself. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 78 ++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 43 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index b25c8d80b965..37811cddd55c 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -161,6 +161,37 @@ struct_nested_prefixes = [ (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), ] +# +# Transforms for function prototypes +# +function_xforms = [ + (r"^static +", "", 0), + (r"^extern +", "", 0), + (r"^asmlinkage +", "", 0), + (r"^inline +", "", 0), + (r"^__inline__ +", "", 0), + (r"^__inline +", "", 0), + (r"^__always_inline +", "", 0), + (r"^noinline +", "", 0), + (r"^__FORTIFY_INLINE +", "", 0), + (r"__init +", "", 0), + (r"__init_or_module +", "", 0), + (r"__deprecated +", "", 0), + (r"__flatten +", "", 0), + (r"__meminit +", "", 0), + (r"__must_check +", "", 0), + (r"__weak +", "", 0), + (r"__sched +", "", 0), + (r"_noprof", "", 0), + (r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +", "", 0), + (r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +", "", 0), + (r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +", "", 0), + (r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)", r"\1, \2", 0), + (r"__attribute_const__ +", "", 0), + (r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+", "", 0), +] + + # # A little helper to get rid of excess white space @@ -894,49 +925,10 @@ class KernelDoc: return_type = '' decl_type = 'function' - # Prefixes that would be removed - sub_prefixes = [ - (r"^static +", "", 0), - (r"^extern +", "", 0), - (r"^asmlinkage +", "", 0), - (r"^inline +", "", 0), - (r"^__inline__ +", "", 0), - (r"^__inline +", "", 0), - (r"^__always_inline +", "", 0), - (r"^noinline +", "", 0), - (r"^__FORTIFY_INLINE +", "", 0), - (r"__init +", "", 0), - (r"__init_or_module +", "", 0), - (r"__deprecated +", "", 0), - (r"__flatten +", "", 0), - (r"__meminit +", "", 0), - (r"__must_check +", "", 0), - (r"__weak +", "", 0), - (r"__sched +", "", 0), - (r"_noprof", "", 0), - (r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +", "", 0), - (r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +", "", 0), - (r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +", "", 0), - (r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)", r"\1, \2", 0), - (r"__attribute_const__ +", "", 0), - - # It seems that Python support for re.X is broken: - # At least for me (Python 3.13), this didn't work -# (r""" -# __attribute__\s*\(\( -# (?: -# [\w\s]+ # attribute name -# (?:\([^)]*\))? # attribute arguments -# \s*,? # optional comma at the end -# )+ -# \)\)\s+ -# """, "", re.X), - - # So, remove whitespaces and comments from it - (r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+", "", 0), - ] - - for search, sub, flags in sub_prefixes: + # + # Apply the initial transformations. + # + for search, sub, flags in function_xforms: prototype = KernRe(search, flags).sub(sub, prototype) # Macros are a special case, as they change the prototype format -- cgit From a2752f8c631201e189f501fc4d320354efa3e72e Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 4 Sep 2025 16:49:52 -0600 Subject: doc: kdoc: unify transform handling Both functions and structs are passed through a set of regex-based transforms, but the two were structured differently, despite being the same thing. Create a utility function to apply transformations and use it in both cases. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 65 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 31 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 37811cddd55c..1a1558211acd 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -78,7 +78,7 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' # struct_args_pattern = r'([^,)]+)' -struct_prefixes = [ +struct_xforms = [ # Strip attributes (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), @@ -165,33 +165,39 @@ struct_nested_prefixes = [ # Transforms for function prototypes # function_xforms = [ - (r"^static +", "", 0), - (r"^extern +", "", 0), - (r"^asmlinkage +", "", 0), - (r"^inline +", "", 0), - (r"^__inline__ +", "", 0), - (r"^__inline +", "", 0), - (r"^__always_inline +", "", 0), - (r"^noinline +", "", 0), - (r"^__FORTIFY_INLINE +", "", 0), - (r"__init +", "", 0), - (r"__init_or_module +", "", 0), - (r"__deprecated +", "", 0), - (r"__flatten +", "", 0), - (r"__meminit +", "", 0), - (r"__must_check +", "", 0), - (r"__weak +", "", 0), - (r"__sched +", "", 0), - (r"_noprof", "", 0), - (r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +", "", 0), - (r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +", "", 0), - (r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +", "", 0), - (r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)", r"\1, \2", 0), - (r"__attribute_const__ +", "", 0), - (r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+", "", 0), + (KernRe(r"^static +"), ""), + (KernRe(r"^extern +"), ""), + (KernRe(r"^asmlinkage +"), ""), + (KernRe(r"^inline +"), ""), + (KernRe(r"^__inline__ +"), ""), + (KernRe(r"^__inline +"), ""), + (KernRe(r"^__always_inline +"), ""), + (KernRe(r"^noinline +"), ""), + (KernRe(r"^__FORTIFY_INLINE +"), ""), + (KernRe(r"__init +"), ""), + (KernRe(r"__init_or_module +"), ""), + (KernRe(r"__deprecated +"), ""), + (KernRe(r"__flatten +"), ""), + (KernRe(r"__meminit +"), ""), + (KernRe(r"__must_check +"), ""), + (KernRe(r"__weak +"), ""), + (KernRe(r"__sched +"), ""), + (KernRe(r"_noprof"), ""), + (KernRe(r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +"), ""), + (KernRe(r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +"), ""), + (KernRe(r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +"), ""), + (KernRe(r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)"), r"\1, \2"), + (KernRe(r"__attribute_const__ +"), ""), + (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""), ] - +# +# Apply a set of transforms to a block of text. +# +def apply_transforms(xforms, text): + for search, subst in xforms: + text = search.sub(subst, text) + return text # # A little helper to get rid of excess white space @@ -807,8 +813,7 @@ class KernelDoc: # Go through the list of members applying all of our transformations. # members = trim_private_members(members) - for search, sub in struct_prefixes: - members = search.sub(sub, members) + members = apply_transforms(struct_xforms, members) nested = NestedMatch() for search, sub in struct_nested_prefixes: @@ -924,12 +929,10 @@ class KernelDoc: func_macro = False return_type = '' decl_type = 'function' - # # Apply the initial transformations. # - for search, sub, flags in function_xforms: - prototype = KernRe(search, flags).sub(sub, prototype) + prototype = apply_transforms(function_xforms, prototype) # Macros are a special case, as they change the prototype format new_proto = KernRe(r"^#\s*define\s+").sub("", prototype) -- cgit From fee63c8f10c2fe77f618f9955c2f5521ff9cc622 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Fri, 5 Sep 2025 15:53:05 -0600 Subject: docs: kdoc: remove a couple of spurious regex characters The "name" regex in dump_function() includes both the tilde and colon characters, but neither has any place in function prototypes. Remove the characters, after which the regex simplifies to "\w+" No output changes. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 1a1558211acd..decd127df82e 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -958,7 +958,7 @@ class KernelDoc: # - atomic_set (macro) # - pci_match_device, __copy_to_user (long return type) - name = r'[a-zA-Z0-9_~:]+' + name = r'\w+' prototype_end1 = r'[^\(]*' prototype_end2 = r'[^\{]*' prototype_end = fr'\(({prototype_end1}|{prototype_end2})\)' -- cgit From 08b5228cf455d46a23bfb341766563d1a48e3c8f Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Fri, 5 Sep 2025 16:30:48 -0600 Subject: docs: kdoc: remove a useless empty capture group The is_define_proto case in dump_function() uses a regex with an empty capture group - () - that has no use; just take it out. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index decd127df82e..f9be5414244d 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -972,11 +972,11 @@ class KernelDoc: found = False if is_define_proto: - r = KernRe(r'^()(' + name + r')\s+') + r = KernRe(r'^(' + name + r')\s+') if r.search(prototype): return_type = '' - declaration_name = r.group(2) + declaration_name = r.group(1) func_macro = True found = True -- cgit From ff1f2af341b72bd5b6b5d432da55faf2f6d24cfe Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 13:22:42 -0600 Subject: docs: kdoc: Simplify the dump_function() prototype regexes The regexes for the parsing of function prototypes were more complicated than they needed to be and difficult to understand -- at least, I spent a fair amount of time bashing my head against them. Simplify them, and add some documentation comments as well. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index f9be5414244d..ec2e6e83df05 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -959,15 +959,15 @@ class KernelDoc: # - pci_match_device, __copy_to_user (long return type) name = r'\w+' - prototype_end1 = r'[^\(]*' - prototype_end2 = r'[^\{]*' - prototype_end = fr'\(({prototype_end1}|{prototype_end2})\)' - - # Besides compiling, Perl qr{[\w\s]+} works as a non-capturing group. - # So, this needs to be mapped in Python with (?:...)? or (?:...)+ - type1 = r'(?:[\w\s]+)?' type2 = r'(?:[\w\s]+\*+)+' + # + # Attempt to match first on (args) with no internal parentheses; this + # lets us easily filter out __acquires() and other post-args stuff. If + # that fails, just grab the rest of the line to the last closing + # parenthesis. + # + proto_args = r'\(([^\(]*|.*)\)' found = False @@ -983,9 +983,9 @@ class KernelDoc: if not found: patterns = [ - rf'^()({name})\s*{prototype_end}', - rf'^({type1})\s+({name})\s*{prototype_end}', - rf'^({type2})\s*({name})\s*{prototype_end}', + rf'^()({name})\s*{proto_args}', + rf'^({type1})\s+({name})\s*{proto_args}', + rf'^({type2})\s*({name})\s*{proto_args}', ] for p in patterns: -- cgit From 370f430527ecd35938ad94167e45fc784f6e4d95 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 16:01:15 -0600 Subject: docs: kdoc: consolidate some of the macro-processing logic The logic to handle macros is split in dump_function(); bring it all together into a single place and add a comment saying what's going on. Remove the unneeded is_define_proto variable, and tighten up the code a bit. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 43 +++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 23 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index ec2e6e83df05..27329ce9b5e9 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -926,21 +926,31 @@ class KernelDoc: Stores a function of function macro inside self.entries array. """ - func_macro = False + found = func_macro = False return_type = '' decl_type = 'function' # # Apply the initial transformations. # prototype = apply_transforms(function_xforms, prototype) - - # Macros are a special case, as they change the prototype format + # + # If we have a macro, remove the "#define" at the front. + # new_proto = KernRe(r"^#\s*define\s+").sub("", prototype) if new_proto != prototype: - is_define_proto = True prototype = new_proto - else: - is_define_proto = False + # + # Dispense with the simple "#define A B" case here; the key + # is the space after the name of the symbol being defined. + # NOTE that the seemingly misnamed "func_macro" indicates a + # macro *without* arguments. + # + r = KernRe(r'^(\w+)\s+') + if r.search(prototype): + return_type = '' + declaration_name = r.group(1) + func_macro = True + found = True # Yes, this truly is vile. We are looking for: # 1. Return type (may be nothing if we're looking at a macro) @@ -968,19 +978,10 @@ class KernelDoc: # parenthesis. # proto_args = r'\(([^\(]*|.*)\)' - - found = False - - if is_define_proto: - r = KernRe(r'^(' + name + r')\s+') - - if r.search(prototype): - return_type = '' - declaration_name = r.group(1) - func_macro = True - - found = True - + # + # (Except for the simple macro case) attempt to split up the prototype + # in the various ways we understand. + # if not found: patterns = [ rf'^()({name})\s*{proto_args}', @@ -990,16 +991,12 @@ class KernelDoc: for p in patterns: r = KernRe(p) - if r.match(prototype): - return_type = r.group(1) declaration_name = r.group(2) args = r.group(3) - self.create_parameter_list(ln, decl_type, args, ',', declaration_name) - found = True break if not found: -- cgit From 3dff54410e56ddee2dad8824ea77e60cd5b16d5b Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 16:21:28 -0600 Subject: docs: kdoc: final dump_function() cleanups Add some more comments to dump_function(), add some comments, and trim out an unneeded duplicate output_declaration() call. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 27329ce9b5e9..5e41acfef7b8 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -999,32 +999,28 @@ class KernelDoc: declaration_name) found = True break + # + # Parsing done; make sure that things are as we expect. + # if not found: self.emit_msg(ln, f"cannot understand function prototype: '{prototype}'") return - if self.entry.identifier != declaration_name: - self.emit_msg(ln, - f"expecting prototype for {self.entry.identifier}(). Prototype was for {declaration_name}() instead") + self.emit_msg(ln, f"expecting prototype for {self.entry.identifier}(). " + f"Prototype was for {declaration_name}() instead") return - self.check_sections(ln, declaration_name, "function") - self.check_return_section(ln, declaration_name, return_type) + # + # Store the result. + # + self.output_declaration(decl_type, declaration_name, + typedef=('typedef' in return_type), + functiontype=return_type, + purpose=self.entry.declaration_purpose, + func_macro=func_macro) - if 'typedef' in return_type: - self.output_declaration(decl_type, declaration_name, - typedef=True, - functiontype=return_type, - purpose=self.entry.declaration_purpose, - func_macro=func_macro) - else: - self.output_declaration(decl_type, declaration_name, - typedef=False, - functiontype=return_type, - purpose=self.entry.declaration_purpose, - func_macro=func_macro) def dump_typedef(self, ln, proto): """ -- cgit From 999a642d7e7d4241cc7dba942a13c67d0685284b Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 16:32:10 -0600 Subject: docs: kdoc: remove some dead code in dump_typedef() The regex in this block of code makes no sense, and a quick test shows that it never matches anything; simply delete the code. No output changes. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 5e41acfef7b8..7c739b495d58 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -1060,11 +1060,6 @@ class KernelDoc: purpose=self.entry.declaration_purpose) return - # Handle nested parentheses or brackets - r = KernRe(r'(\(*.\)\s*|\[*.\]\s*);$') - while r.search(proto): - proto = r.sub('', proto) - # Parse simple typedefs r = KernRe(r'typedef.*\s+(\w+)\s*;') if r.match(proto): -- cgit From 00fa9bc4e93cb336575a5f2c1da90d2443ff14c8 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 16:49:58 -0600 Subject: docs: kdoc: remove redundant comment stripping in dump_typedef() By the time we get here, comments have long since been stripped out; there is no need to do it again. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 7c739b495d58..ad9df0536bbf 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -1034,9 +1034,6 @@ class KernelDoc: typedef1 = KernRe(r'typedef' + typedef_type + r'\(' + typedef_ident + r'\)' + typedef_args) typedef2 = KernRe(r'typedef' + typedef_type + typedef_ident + typedef_args) - # Strip comments - proto = KernRe(r'/\*.*?\*/', flags=re.S).sub('', proto) - # Parse function typedef prototypes for r in [typedef1, typedef2]: if not r.match(proto): -- cgit From c01878437739f86851da76235f394346c6bd8ce3 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 17:12:14 -0600 Subject: docs: kdoc: a few more dump_typedef() tweaks Merge "typedef" into the typedef_type pattern rather than repeating it later, and add some comments. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index ad9df0536bbf..2376f180b1fa 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -1026,13 +1026,15 @@ class KernelDoc: """ Stores a typedef inside self.entries array. """ - - typedef_type = r'((?:\s+[\w*]+\b){0,7}\s+(?:\w+\b|\*+))\s*' + # + # We start by looking for function typedefs. + # + typedef_type = r'typedef((?:\s+[\w*]+\b){0,7}\s+(?:\w+\b|\*+))\s*' typedef_ident = r'\*?\s*(\w\S+)\s*' typedef_args = r'\s*\((.*)\);' - typedef1 = KernRe(r'typedef' + typedef_type + r'\(' + typedef_ident + r'\)' + typedef_args) - typedef2 = KernRe(r'typedef' + typedef_type + typedef_ident + typedef_args) + typedef1 = KernRe(typedef_type + r'\(' + typedef_ident + r'\)' + typedef_args) + typedef2 = KernRe(typedef_type + typedef_ident + typedef_args) # Parse function typedef prototypes for r in [typedef1, typedef2]: @@ -1048,16 +1050,16 @@ class KernelDoc: f"expecting prototype for typedef {self.entry.identifier}. Prototype was for typedef {declaration_name} instead\n") return - decl_type = 'function' - self.create_parameter_list(ln, decl_type, args, ',', declaration_name) + self.create_parameter_list(ln, 'function', args, ',', declaration_name) - self.output_declaration(decl_type, declaration_name, + self.output_declaration('function', declaration_name, typedef=True, functiontype=return_type, purpose=self.entry.declaration_purpose) return - - # Parse simple typedefs + # + # Not a function, try to parse a simple typedef. + # r = KernRe(r'typedef.*\s+(\w+)\s*;') if r.match(proto): declaration_name = r.group(1) -- cgit