From c072d2b49507420edd57ba3d87690ab81ef5273a Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Fri, 25 Jul 2025 15:06:26 -0500 Subject: scripts/dtc: Update to upstream version v1.7.2-35-g52f07dcca47c This adds the following commits from upstream: 52f07dcca47c dtc: Add informative error for stray identifier 9cabae6b0351 checks: Fix detection of 'i2c-bus' node 605dc044c3fe New helper to add markers 7da5d106c740 fdtput: Fix documentation about existing nodes 53c63dd421d7 dtdiff: Use input format dtb for dtbo files 84d9dd2fcbc8 dtc: Add data_insert_data function 97011d1f4e98 meson: use override_find_program/override_dependency b841391bbd08 srcpos: Define srcpos_free e0b7749c26a9 Add alloc_marker ecb21febfdd3 meson: port python bindings to build natively via meson and meson-python 7ebfcac8520e Makefile: deprecate in favor of Meson f4c53f4ebf78 Use __ASSEMBLER__ instead of __ASSEMBLY__ 205fbef17b7b Fix some typos da85f91931e5 Remove duplicated words in documentation and comments dd1b3e532d22 meson: support building libfdt without static library 1ccd232709d4 meson: don't build test programs by default ce1d8588880a tests: When building .so from -O asm output mark as non-executable stack 915daadbb62d Start with empty __local_fixups__ and __fixups__ nodes 4ea851f5a44d Let get_subnode() not return deleted nodes 175d2a564c47 Use build_root_node() instead of open-coding it 18f4f305fdd7 build: fix -Dtools=false build 267efc7d4694 checks: Warn about missing #address-cells for interrupt parents 755db115355b libfdt: Add fdt_setprop_namelen_string() bdca8612009e libfdt: Add fdt_setprop_namelen() 0f69cedc08fc libfdt_internal: fdt_find_string_len_() 56b2b30c5bd0 libfdt: add fdt_get_property_namelen_w() 1e8c5f60e127 Add clang-format config 6f183c7d9246 checks: Relax avoid_unnecessary_addr_size check to allow child ranges properties 66c7d0e6f4f3 tests/sw_tree1.c: fix unitialized saveptr 9a969f3b70b0 pylibfdt/libfdt.i: fix backwards compatibility of return values 4292b072a23a .github/workflows: update ubuntu runner to supported version 1c745a9bd169 libfdt: Remove fdt parameter from overlay_fixup_one_phandle b3bbee6b1242 libfdt: Move the SBOM authors section d1656730abfb Add a SBOM file in CycloneDX format b75515af4576 libfdt: Remove extra semi-colons outside functions 2d10aa2afe35 Bump version to v1.7.2 48795c82bdb6 pylibfdt: Don't emit warnings from swig generate C code 838f11e830e3 fdtoverlay: provide better error message for missing `/__symbols__` d1e2384185c5 pylibfdt/libfdt.i: Use SWIG_AppendOutput 18aa49a9f68d Escape spaces in depfile with backslashes. f9968fa06921 libfdt.h: whitespace consistency fixups 9b5f65fb3d8d libfdt.h: typo and consistency fixes 99031e3a4a6e Bump version to v1.7.1 3d5e376925fd setup: Move setting of srcdir down to the bottom e277553b9880 setup: Collect top-level code together 7e5a88984081 setup: Move version and full_description into a function 78b6a85c113b Tidy up some pylint warnings 3501d373f0a2 Require Python 3 The added include of string.h in libfdt_internal.h breaks the kernel overriding libfdt_env.h with its own string functions, so it is dropped. An upstream fix is pending. Signed-off-by: Rob Herring (Arm) --- scripts/Makefile.dtbs | 1 + scripts/dtc/checks.c | 23 +++-- scripts/dtc/data.c | 47 ++++++++- scripts/dtc/dtc-lexer.l | 15 +++ scripts/dtc/dtc.c | 6 +- scripts/dtc/dtc.h | 5 +- scripts/dtc/fdtoverlay.c | 8 ++ scripts/dtc/flattree.c | 2 +- scripts/dtc/libfdt/fdt.c | 8 +- scripts/dtc/libfdt/fdt.h | 4 +- scripts/dtc/libfdt/fdt_overlay.c | 8 +- scripts/dtc/libfdt/fdt_rw.c | 41 ++++---- scripts/dtc/libfdt/libfdt.h | 179 ++++++++++++++++++++++++++++------- scripts/dtc/libfdt/libfdt_internal.h | 14 ++- scripts/dtc/livetree.c | 25 +++-- scripts/dtc/srcpos.c | 17 +++- scripts/dtc/srcpos.h | 1 + scripts/dtc/treesource.c | 52 +++++++--- scripts/dtc/util.c | 16 ++++ scripts/dtc/util.h | 5 + scripts/dtc/version_gen.h | 2 +- 21 files changed, 372 insertions(+), 107 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.dtbs b/scripts/Makefile.dtbs index 8d56c0815f33..2d321b813600 100644 --- a/scripts/Makefile.dtbs +++ b/scripts/Makefile.dtbs @@ -97,6 +97,7 @@ DTC_FLAGS += -Wno-unit_address_vs_reg \ -Wno-avoid_unnecessary_addr_size \ -Wno-alias_paths \ -Wno-graph_child_address \ + -Wno-interrupt_map \ -Wno-simple_bus_reg else DTC_FLAGS += -Wunique_unit_address_if_enabled diff --git a/scripts/dtc/checks.c b/scripts/dtc/checks.c index 6e06aeab5503..7e3fed5005b3 100644 --- a/scripts/dtc/checks.c +++ b/scripts/dtc/checks.c @@ -1024,7 +1024,7 @@ static void check_i2c_bus_bridge(struct check *c, struct dt_info *dti, struct no } else if (strprefixeq(node->name, node->basenamelen, "i2c")) { struct node *child; for_each_child(node, child) { - if (strprefixeq(child->name, node->basenamelen, "i2c-bus")) + if (strprefixeq(child->name, child->basenamelen, "i2c-bus")) return; } node->bus = &i2c_bus; @@ -1217,9 +1217,7 @@ WARNING(avoid_default_addr_size, check_avoid_default_addr_size, NULL, static void check_avoid_unnecessary_addr_size(struct check *c, struct dt_info *dti, struct node *node) { - struct property *prop; struct node *child; - bool has_reg = false; if (!node->parent || node->addr_cells < 0 || node->size_cells < 0) return; @@ -1228,13 +1226,18 @@ static void check_avoid_unnecessary_addr_size(struct check *c, struct dt_info *d return; for_each_child(node, child) { - prop = get_property(child, "reg"); - if (prop) - has_reg = true; + /* + * Even if the child devices' address space is not mapped into + * the parent bus (no 'ranges' property on node), children can + * still have registers on a local bus, or map local addresses + * to another subordinate address space. The properties on the + * child nodes then make #address-cells/#size-cells necessary: + */ + if (get_property(child, "reg") || get_property(child, "ranges")) + return; } - if (!has_reg) - FAIL(c, dti, node, "unnecessary #address-cells/#size-cells without \"ranges\", \"dma-ranges\" or child \"reg\" property"); + FAIL(c, dti, node, "unnecessary #address-cells/#size-cells without \"ranges\", \"dma-ranges\" or child \"reg\" or \"ranges\" property"); } WARNING(avoid_unnecessary_addr_size, check_avoid_unnecessary_addr_size, NULL, &avoid_default_addr_size); @@ -1673,6 +1676,10 @@ static void check_interrupt_map(struct check *c, cellprop = get_property(provider_node, "#address-cells"); if (cellprop) parent_cellsize += propval_cell(cellprop); + else + FAIL_PROP(c, dti, node, irq_map_prop, + "Missing property '#address-cells' in node %s, using 0 as fallback", + provider_node->fullpath); cell += 1 + parent_cellsize; if (cell > map_cells) diff --git a/scripts/dtc/data.c b/scripts/dtc/data.c index 14734233ad8b..5b25aa060416 100644 --- a/scripts/dtc/data.c +++ b/scripts/dtc/data.c @@ -228,11 +228,7 @@ struct data data_add_marker(struct data d, enum markertype type, char *ref) { struct marker *m; - m = xmalloc(sizeof(*m)); - m->offset = d.len; - m->type = type; - m->ref = ref; - m->next = NULL; + m = alloc_marker(d.len, type, ref); return data_append_markers(d, m); } @@ -254,3 +250,44 @@ bool data_is_one_string(struct data d) return true; } + +struct data data_insert_data(struct data d, struct marker *m, struct data old) +{ + unsigned int offset = m->offset; + struct marker *next = m->next; + struct marker *marker; + struct data new_data; + char *ref; + + new_data = data_insert_at_marker(d, m, old.val, old.len); + + /* Copy all markers from old value */ + marker = old.markers; + for_each_marker(marker) { + ref = NULL; + + if (marker->ref) + ref = xstrdup(marker->ref); + + m->next = alloc_marker(marker->offset + offset, marker->type, + ref); + m = m->next; + } + m->next = next; + + return new_data; +} + +struct marker *alloc_marker(unsigned int offset, enum markertype type, + char *ref) +{ + struct marker *m; + + m = xmalloc(sizeof(*m)); + m->offset = offset; + m->type = type; + m->ref = ref; + m->next = NULL; + + return m; +} diff --git a/scripts/dtc/dtc-lexer.l b/scripts/dtc/dtc-lexer.l index de60a70b6bdb..15d585c80798 100644 --- a/scripts/dtc/dtc-lexer.l +++ b/scripts/dtc/dtc-lexer.l @@ -151,6 +151,21 @@ static void PRINTF(1, 2) lexical_error(const char *fmt, ...); return DT_LABEL; } +{LABEL} { + /* Missed includes or macro definitions while + * preprocessing can lead to unexpected identifiers in + * the input. Report a slightly more informative error + * in this case */ + + lexical_error("Unexpected '%s'", yytext); + + /* Treat it as a literal which often generates further + * useful error messages */ + + yylval.integer = 0; + return DT_LITERAL; + } + ([0-9]+|0[xX][0-9a-fA-F]+)(U|L|UL|LL|ULL)? { char *e; DPRINT("Integer Literal: '%s'\n", yytext); diff --git a/scripts/dtc/dtc.c b/scripts/dtc/dtc.c index 0655c2e2c362..b3445b7d6473 100644 --- a/scripts/dtc/dtc.c +++ b/scripts/dtc/dtc.c @@ -15,7 +15,7 @@ int quiet; /* Level of quietness */ unsigned int reservenum;/* Number of memory reservation slots */ int minsize; /* Minimum blob size */ int padsize; /* Additional padding to blob */ -int alignsize; /* Additional padding to blob accroding to the alignsize */ +int alignsize; /* Additional padding to blob according to the alignsize */ int phandle_format = PHANDLE_EPAPR; /* Use linux,phandle or phandle properties */ int generate_symbols; /* enable symbols & fixup support */ int generate_fixups; /* suppress generation of fixups on symbol support */ @@ -289,7 +289,9 @@ int main(int argc, char *argv[]) if (!depfile) die("Couldn't open dependency file %s: %s\n", depname, strerror(errno)); - fprintf(depfile, "%s:", outname); + + fprint_path_escaped(depfile, outname); + fputc(':', depfile); } if (inform == NULL) diff --git a/scripts/dtc/dtc.h b/scripts/dtc/dtc.h index 4c4aaca1fc41..3a220b9afc99 100644 --- a/scripts/dtc/dtc.h +++ b/scripts/dtc/dtc.h @@ -38,7 +38,7 @@ extern int quiet; /* Level of quietness */ extern unsigned int reservenum; /* Number of memory reservation slots */ extern int minsize; /* Minimum blob size */ extern int padsize; /* Additional padding to blob */ -extern int alignsize; /* Additional padding to blob accroding to the alignsize */ +extern int alignsize; /* Additional padding to blob according to the alignsize */ extern int phandle_format; /* Use linux,phandle or phandle properties */ extern int generate_symbols; /* generate symbols for nodes with labels */ extern int generate_fixups; /* generate fixups */ @@ -182,7 +182,10 @@ struct data data_append_addr(struct data d, uint64_t addr); struct data data_append_byte(struct data d, uint8_t byte); struct data data_append_zeroes(struct data d, int len); struct data data_append_align(struct data d, int align); +struct data data_insert_data(struct data d, struct marker *m, struct data old); +struct marker *alloc_marker(unsigned int offset, enum markertype type, + char *ref); struct data data_add_marker(struct data d, enum markertype type, char *ref); bool data_is_one_string(struct data d); diff --git a/scripts/dtc/fdtoverlay.c b/scripts/dtc/fdtoverlay.c index 699b4f616502..ee1eb8f3ad28 100644 --- a/scripts/dtc/fdtoverlay.c +++ b/scripts/dtc/fdtoverlay.c @@ -46,6 +46,7 @@ static void *apply_one(char *base, const char *overlay, size_t *buf_len, char *tmp = NULL; char *tmpo; int ret; + bool has_symbols; /* * We take copies first, because a failed apply can trash @@ -62,6 +63,8 @@ static void *apply_one(char *base, const char *overlay, size_t *buf_len, fdt_strerror(ret)); goto fail; } + ret = fdt_path_offset(tmp, "/__symbols__"); + has_symbols = ret >= 0; memcpy(tmpo, overlay, fdt_totalsize(overlay)); @@ -74,6 +77,11 @@ static void *apply_one(char *base, const char *overlay, size_t *buf_len, if (ret) { fprintf(stderr, "\nFailed to apply '%s': %s\n", name, fdt_strerror(ret)); + if (!has_symbols) { + fprintf(stderr, + "base blob does not have a '/__symbols__' node, " + "make sure you have compiled the base blob with '-@' option\n"); + } goto fail; } diff --git a/scripts/dtc/flattree.c b/scripts/dtc/flattree.c index 1bcd8089c5b9..30e6de2044b2 100644 --- a/scripts/dtc/flattree.c +++ b/scripts/dtc/flattree.c @@ -503,7 +503,7 @@ void dt_to_asm(FILE *f, struct dt_info *dti, int version) * Reserve map entries. * Align the reserve map to a doubleword boundary. * Each entry is an (address, size) pair of u64 values. - * Always supply a zero-sized temination entry. + * Always supply a zero-sized termination entry. */ asm_emit_align(f, 8); emit_label(f, symprefix, "reserve_map"); diff --git a/scripts/dtc/libfdt/fdt.c b/scripts/dtc/libfdt/fdt.c index 20c6415b9ced..95f644c31f94 100644 --- a/scripts/dtc/libfdt/fdt.c +++ b/scripts/dtc/libfdt/fdt.c @@ -312,14 +312,14 @@ int fdt_next_subnode(const void *fdt, int offset) return offset; } -const char *fdt_find_string_(const char *strtab, int tabsize, const char *s) +const char *fdt_find_string_len_(const char *strtab, int tabsize, const char *s, + int slen) { - int len = strlen(s) + 1; - const char *last = strtab + tabsize - len; + const char *last = strtab + tabsize - (slen + 1); const char *p; for (p = strtab; p <= last; p++) - if (memcmp(p, s, len) == 0) + if (memcmp(p, s, slen) == 0 && p[slen] == '\0') return p; return NULL; } diff --git a/scripts/dtc/libfdt/fdt.h b/scripts/dtc/libfdt/fdt.h index 0c91aa7f67b5..a07abfcc7108 100644 --- a/scripts/dtc/libfdt/fdt.h +++ b/scripts/dtc/libfdt/fdt.h @@ -7,7 +7,7 @@ * Copyright 2012 Kim Phillips, Freescale Semiconductor. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct fdt_header { fdt32_t magic; /* magic word FDT_MAGIC */ @@ -45,7 +45,7 @@ struct fdt_property { char data[]; }; -#endif /* !__ASSEMBLY */ +#endif /* !__ASSEMBLER__ */ #define FDT_MAGIC 0xd00dfeed /* 4: version, 4: total size */ #define FDT_TAGSIZE sizeof(fdt32_t) diff --git a/scripts/dtc/libfdt/fdt_overlay.c b/scripts/dtc/libfdt/fdt_overlay.c index 28b667ffc490..e6b9eb643958 100644 --- a/scripts/dtc/libfdt/fdt_overlay.c +++ b/scripts/dtc/libfdt/fdt_overlay.c @@ -307,7 +307,6 @@ static int overlay_update_local_references(void *fdto, uint32_t delta) /** * overlay_fixup_one_phandle - Set an overlay phandle to the base one - * @fdt: Base Device Tree blob * @fdto: Device tree overlay blob * @symbols_off: Node offset of the symbols node in the base device tree * @path: Path to a node holding a phandle in the overlay @@ -328,8 +327,7 @@ static int overlay_update_local_references(void *fdto, uint32_t delta) * 0 on success * Negative error code on failure */ -static int overlay_fixup_one_phandle(void *fdt, void *fdto, - int symbols_off, +static int overlay_fixup_one_phandle(void *fdto, int symbols_off, const char *path, uint32_t path_len, const char *name, uint32_t name_len, int poffset, uint32_t phandle) @@ -351,7 +349,7 @@ static int overlay_fixup_one_phandle(void *fdt, void *fdto, name, name_len, poffset, &phandle_prop, sizeof(phandle_prop)); -}; +} /** * overlay_fixup_phandle - Set an overlay phandle to the base one @@ -443,7 +441,7 @@ static int overlay_fixup_phandle(void *fdt, void *fdto, int symbols_off, if ((*endptr != '\0') || (endptr <= (sep + 1))) return -FDT_ERR_BADOVERLAY; - ret = overlay_fixup_one_phandle(fdt, fdto, symbols_off, + ret = overlay_fixup_one_phandle(fdto, symbols_off, path, path_len, name, name_len, poffset, phandle); if (ret) diff --git a/scripts/dtc/libfdt/fdt_rw.c b/scripts/dtc/libfdt/fdt_rw.c index 3621d3651d3f..7475cafce071 100644 --- a/scripts/dtc/libfdt/fdt_rw.c +++ b/scripts/dtc/libfdt/fdt_rw.c @@ -124,31 +124,33 @@ static int fdt_splice_string_(void *fdt, int newlen) * allocated. Ignored if can_assume(NO_ROLLBACK) * @return offset of string in the string table (whether found or added) */ -static int fdt_find_add_string_(void *fdt, const char *s, int *allocated) +static int fdt_find_add_string_(void *fdt, const char *s, int slen, + int *allocated) { char *strtab = (char *)fdt + fdt_off_dt_strings(fdt); const char *p; char *new; - int len = strlen(s) + 1; int err; if (!can_assume(NO_ROLLBACK)) *allocated = 0; - p = fdt_find_string_(strtab, fdt_size_dt_strings(fdt), s); + p = fdt_find_string_len_(strtab, fdt_size_dt_strings(fdt), s, slen); if (p) /* found it */ return (p - strtab); new = strtab + fdt_size_dt_strings(fdt); - err = fdt_splice_string_(fdt, len); + err = fdt_splice_string_(fdt, slen + 1); if (err) return err; if (!can_assume(NO_ROLLBACK)) *allocated = 1; - memcpy(new, s, len); + memcpy(new, s, slen); + new[slen] = '\0'; + return (new - strtab); } @@ -181,13 +183,15 @@ int fdt_del_mem_rsv(void *fdt, int n) return fdt_splice_mem_rsv_(fdt, re, 1, 0); } -static int fdt_resize_property_(void *fdt, int nodeoffset, const char *name, +static int fdt_resize_property_(void *fdt, int nodeoffset, + const char *name, int namelen, int len, struct fdt_property **prop) { int oldlen; int err; - *prop = fdt_get_property_w(fdt, nodeoffset, name, &oldlen); + *prop = fdt_get_property_namelen_w(fdt, nodeoffset, name, namelen, + &oldlen); if (!*prop) return oldlen; @@ -200,7 +204,7 @@ static int fdt_resize_property_(void *fdt, int nodeoffset, const char *name, } static int fdt_add_property_(void *fdt, int nodeoffset, const char *name, - int len, struct fdt_property **prop) + int namelen, int len, struct fdt_property **prop) { int proplen; int nextoffset; @@ -211,7 +215,7 @@ static int fdt_add_property_(void *fdt, int nodeoffset, const char *name, if ((nextoffset = fdt_check_node_offset_(fdt, nodeoffset)) < 0) return nextoffset; - namestroff = fdt_find_add_string_(fdt, name, &allocated); + namestroff = fdt_find_add_string_(fdt, name, namelen, &allocated); if (namestroff < 0) return namestroff; @@ -255,17 +259,18 @@ int fdt_set_name(void *fdt, int nodeoffset, const char *name) return 0; } -int fdt_setprop_placeholder(void *fdt, int nodeoffset, const char *name, - int len, void **prop_data) +int fdt_setprop_placeholder_namelen(void *fdt, int nodeoffset, const char *name, + int namelen, int len, void **prop_data) { struct fdt_property *prop; int err; FDT_RW_PROBE(fdt); - err = fdt_resize_property_(fdt, nodeoffset, name, len, &prop); + err = fdt_resize_property_(fdt, nodeoffset, name, namelen, len, &prop); if (err == -FDT_ERR_NOTFOUND) - err = fdt_add_property_(fdt, nodeoffset, name, len, &prop); + err = fdt_add_property_(fdt, nodeoffset, name, namelen, len, + &prop); if (err) return err; @@ -273,13 +278,14 @@ int fdt_setprop_placeholder(void *fdt, int nodeoffset, const char *name, return 0; } -int fdt_setprop(void *fdt, int nodeoffset, const char *name, - const void *val, int len) +int fdt_setprop_namelen(void *fdt, int nodeoffset, const char *name, + int namelen, const void *val, int len) { void *prop_data; int err; - err = fdt_setprop_placeholder(fdt, nodeoffset, name, len, &prop_data); + err = fdt_setprop_placeholder_namelen(fdt, nodeoffset, name, namelen, + len, &prop_data); if (err) return err; @@ -307,7 +313,8 @@ int fdt_appendprop(void *fdt, int nodeoffset, const char *name, prop->len = cpu_to_fdt32(newlen); memcpy(prop->data + oldlen, val, len); } else { - err = fdt_add_property_(fdt, nodeoffset, name, len, &prop); + err = fdt_add_property_(fdt, nodeoffset, name, strlen(name), + len, &prop); if (err) return err; memcpy(prop->data, val, len); diff --git a/scripts/dtc/libfdt/libfdt.h b/scripts/dtc/libfdt/libfdt.h index 2d409d8e829b..914bf90785ab 100644 --- a/scripts/dtc/libfdt/libfdt.h +++ b/scripts/dtc/libfdt/libfdt.h @@ -14,7 +14,7 @@ extern "C" { #endif #define FDT_FIRST_SUPPORTED_VERSION 0x02 -#define FDT_LAST_COMPATIBLE_VERSION 0x10 +#define FDT_LAST_COMPATIBLE_VERSION 0x10 #define FDT_LAST_SUPPORTED_VERSION 0x11 /* Error codes: informative error codes */ @@ -263,16 +263,16 @@ int fdt_next_subnode(const void *fdt, int offset); struct fdt_header *fdth = (struct fdt_header *)fdt; \ fdth->name = cpu_to_fdt32(val); \ } -fdt_set_hdr_(magic); -fdt_set_hdr_(totalsize); -fdt_set_hdr_(off_dt_struct); -fdt_set_hdr_(off_dt_strings); -fdt_set_hdr_(off_mem_rsvmap); -fdt_set_hdr_(version); -fdt_set_hdr_(last_comp_version); -fdt_set_hdr_(boot_cpuid_phys); -fdt_set_hdr_(size_dt_strings); -fdt_set_hdr_(size_dt_struct); +fdt_set_hdr_(magic) +fdt_set_hdr_(totalsize) +fdt_set_hdr_(off_dt_struct) +fdt_set_hdr_(off_dt_strings) +fdt_set_hdr_(off_mem_rsvmap) +fdt_set_hdr_(version) +fdt_set_hdr_(last_comp_version) +fdt_set_hdr_(boot_cpuid_phys) +fdt_set_hdr_(size_dt_strings) +fdt_set_hdr_(size_dt_struct) #undef fdt_set_hdr_ /** @@ -285,7 +285,7 @@ size_t fdt_header_size(const void *fdt); /** * fdt_header_size_ - internal function to get header size from a version number - * @version: devicetree version number + * @version: device tree version number * * Return: size of DTB header in bytes */ @@ -554,7 +554,7 @@ int fdt_path_offset_namelen(const void *fdt, const char *path, int namelen); * -FDT_ERR_BADPATH, given path does not begin with '/' and the first * component is not a valid alias * -FDT_ERR_NOTFOUND, if the requested node does not exist - * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADMAGIC, * -FDT_ERR_BADVERSION, * -FDT_ERR_BADSTATE, * -FDT_ERR_BADSTRUCTURE, @@ -599,7 +599,7 @@ const char *fdt_get_name(const void *fdt, int nodeoffset, int *lenp); * structure block offset of the property (>=0), on success * -FDT_ERR_NOTFOUND, if the requested node has no properties * -FDT_ERR_BADOFFSET, if nodeoffset did not point to an FDT_BEGIN_NODE tag - * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADMAGIC, * -FDT_ERR_BADVERSION, * -FDT_ERR_BADSTATE, * -FDT_ERR_BADSTRUCTURE, @@ -620,7 +620,7 @@ int fdt_first_property_offset(const void *fdt, int nodeoffset); * structure block offset of the next property (>=0), on success * -FDT_ERR_NOTFOUND, if the given property is the last in its node * -FDT_ERR_BADOFFSET, if nodeoffset did not point to an FDT_PROP tag - * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADMAGIC, * -FDT_ERR_BADVERSION, * -FDT_ERR_BADSTATE, * -FDT_ERR_BADSTRUCTURE, @@ -712,6 +712,13 @@ const struct fdt_property *fdt_get_property_namelen(const void *fdt, int nodeoffset, const char *name, int namelen, int *lenp); +static inline struct fdt_property * +fdt_get_property_namelen_w(void *fdt, int nodeoffset, const char *name, + int namelen, int *lenp) +{ + return (struct fdt_property *)(uintptr_t)fdt_get_property_namelen( + fdt, nodeoffset, name, namelen, lenp); +} #endif /** @@ -764,7 +771,7 @@ static inline struct fdt_property *fdt_get_property_w(void *fdt, int nodeoffset, * to within the device blob itself, not a copy of the value). If * lenp is non-NULL, the length of the property value is also * returned, in the integer pointed to by lenp. If namep is non-NULL, - * the property's namne will also be returned in the char * pointed to + * the property's name will also be returned in the char * pointed to * by namep (this will be a pointer to within the device tree's string * block, not a new copy of the name). * @@ -772,7 +779,7 @@ static inline struct fdt_property *fdt_get_property_w(void *fdt, int nodeoffset, * pointer to the property's value * if lenp is non-NULL, *lenp contains the length of the property * value (>=0) - * if namep is non-NULL *namep contiains a pointer to the property + * if namep is non-NULL *namep contains a pointer to the property * name. * NULL, on error * if lenp is non-NULL, *lenp contains an error code (<0): @@ -866,7 +873,7 @@ uint32_t fdt_get_phandle(const void *fdt, int nodeoffset); /** * fdt_get_alias_namelen - get alias based on substring * @fdt: pointer to the device tree blob - * @name: name of the alias th look up + * @name: name of the alias to look up * @namelen: number of characters of name to consider * * Identical to fdt_get_alias(), but only examine the first @namelen @@ -883,7 +890,7 @@ const char *fdt_get_alias_namelen(const void *fdt, /** * fdt_get_alias - retrieve the path referenced by a given alias * @fdt: pointer to the device tree blob - * @name: name of the alias th look up + * @name: name of the alias to look up * * fdt_get_alias() retrieves the value of a given alias. That is, the * value of the property named @name in the node /aliases. @@ -1259,8 +1266,8 @@ const char *fdt_stringlist_get(const void *fdt, int nodeoffset, * * returns: * 0 <= n < FDT_MAX_NCELLS, on success - * 2, if the node has no #address-cells property - * -FDT_ERR_BADNCELLS, if the node has a badly formatted or invalid + * 2, if the node has no #address-cells property + * -FDT_ERR_BADNCELLS, if the node has a badly formatted or invalid * #address-cells property * -FDT_ERR_BADMAGIC, * -FDT_ERR_BADVERSION, @@ -1280,8 +1287,8 @@ int fdt_address_cells(const void *fdt, int nodeoffset); * * returns: * 0 <= n < FDT_MAX_NCELLS, on success - * 1, if the node has no #size-cells property - * -FDT_ERR_BADNCELLS, if the node has a badly formatted or invalid + * 1, if the node has no #size-cells property + * -FDT_ERR_BADNCELLS, if the node has a badly formatted or invalid * #size-cells property * -FDT_ERR_BADMAGIC, * -FDT_ERR_BADVERSION, @@ -1562,7 +1569,7 @@ static inline int fdt_property_cell(void *fdt, const char *name, uint32_t val) * @fdt: pointer to the device tree blob * @name: name of property to add * @len: length of property value in bytes - * @valp: returns a pointer to where where the value should be placed + * @valp: returns a pointer to where the value should be placed * * returns: * 0, on success @@ -1659,6 +1666,38 @@ int fdt_del_mem_rsv(void *fdt, int n); */ int fdt_set_name(void *fdt, int nodeoffset, const char *name); +/** + * fdt_setprop_namelen - create or change a property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @namelen: length of the name + * @val: pointer to data to set the property value to + * @len: length of the property value + * + * fdt_setprop_namelen() sets the value of the named property in the given + * node to the given value and length, creating the property if it + * does not already exist. + * + * This function may insert or delete data from the blob, and will + * therefore change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_setprop_namelen(void *fdt, int nodeoffset, const char *name, + int namelen, const void *val, int len); + /** * fdt_setprop - create or change a property * @fdt: pointer to the device tree blob @@ -1687,8 +1726,44 @@ int fdt_set_name(void *fdt, int nodeoffset, const char *name); * -FDT_ERR_BADLAYOUT, * -FDT_ERR_TRUNCATED, standard meanings */ -int fdt_setprop(void *fdt, int nodeoffset, const char *name, - const void *val, int len); +static inline int fdt_setprop(void *fdt, int nodeoffset, const char *name, + const void *val, int len) +{ + return fdt_setprop_namelen(fdt, nodeoffset, name, strlen(name), val, + len); +} + +/** + * fdt_setprop_placeholder_namelen - allocate space for a property + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @namelen: length of the name + * @len: length of the property value + * @prop_data: return pointer to property data + * + * fdt_setprop_placeholder_namelen() allocates the named property in the given node. + * If the property exists it is resized. In either case a pointer to the + * property data is returned. + * + * This function may insert or delete data from the blob, and will + * therefore change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +int fdt_setprop_placeholder_namelen(void *fdt, int nodeoffset, const char *name, + int namelen, int len, void **prop_data); /** * fdt_setprop_placeholder - allocate space for a property @@ -1698,7 +1773,7 @@ int fdt_setprop(void *fdt, int nodeoffset, const char *name, * @len: length of the property value * @prop_data: return pointer to property data * - * fdt_setprop_placeholer() allocates the named property in the given node. + * fdt_setprop_placeholder() allocates the named property in the given node. * If the property exists it is resized. In either case a pointer to the * property data is returned. * @@ -1718,8 +1793,13 @@ int fdt_setprop(void *fdt, int nodeoffset, const char *name, * -FDT_ERR_BADLAYOUT, * -FDT_ERR_TRUNCATED, standard meanings */ -int fdt_setprop_placeholder(void *fdt, int nodeoffset, const char *name, - int len, void **prop_data); +static inline int fdt_setprop_placeholder(void *fdt, int nodeoffset, + const char *name, int len, + void **prop_data) +{ + return fdt_setprop_placeholder_namelen(fdt, nodeoffset, name, + strlen(name), len, prop_data); +} /** * fdt_setprop_u32 - set a property to a 32-bit integer @@ -1839,6 +1919,38 @@ static inline int fdt_setprop_cell(void *fdt, int nodeoffset, const char *name, #define fdt_setprop_string(fdt, nodeoffset, name, str) \ fdt_setprop((fdt), (nodeoffset), (name), (str), strlen(str)+1) +/** + * fdt_setprop_namelen_string - set a property to a string value + * @fdt: pointer to the device tree blob + * @nodeoffset: offset of the node whose property to change + * @name: name of the property to change + * @namelen: number of characters of name to consider + * @str: string value for the property + * + * fdt_setprop_namelen_string() sets the value of the named property in the + * given node to the given string value (using the length of the + * string to determine the new length of the property), or creates a + * new property with that value if it does not already exist. + * + * This function may insert or delete data from the blob, and will + * therefore change the offsets of some existing nodes. + * + * returns: + * 0, on success + * -FDT_ERR_NOSPACE, there is insufficient free space in the blob to + * contain the new property value + * -FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADVERSION, + * -FDT_ERR_BADSTATE, + * -FDT_ERR_BADSTRUCTURE, + * -FDT_ERR_BADLAYOUT, + * -FDT_ERR_TRUNCATED, standard meanings + */ +#define fdt_setprop_namelen_string(fdt, nodeoffset, name, namelen, str) \ + fdt_setprop_namelen((fdt), (nodeoffset), (name), (namelen), (str), \ + strlen(str) + 1) /** * fdt_setprop_empty - set a property to an empty value @@ -2059,7 +2171,7 @@ int fdt_appendprop_addrrange(void *fdt, int parent, int nodeoffset, * @nodeoffset: offset of the node whose property to nop * @name: name of the property to nop * - * fdt_del_property() will delete the given property. + * fdt_delprop() will delete the given property. * * This function will delete data from the blob, and will therefore * change the offsets of some existing nodes. @@ -2111,8 +2223,7 @@ int fdt_add_subnode_namelen(void *fdt, int parentoffset, * change the offsets of some existing nodes. * * returns: - * structure block offset of the created nodeequested subnode (>=0), on - * success + * structure block offset of the created subnode (>=0), on success * -FDT_ERR_NOTFOUND, if the requested subnode does not exist * -FDT_ERR_BADOFFSET, if parentoffset did not point to an FDT_BEGIN_NODE * tag @@ -2122,7 +2233,7 @@ int fdt_add_subnode_namelen(void *fdt, int parentoffset, * blob to contain the new node * -FDT_ERR_NOSPACE * -FDT_ERR_BADLAYOUT - * -FDT_ERR_BADMAGIC, + * -FDT_ERR_BADMAGIC, * -FDT_ERR_BADVERSION, * -FDT_ERR_BADSTATE, * -FDT_ERR_BADSTRUCTURE, @@ -2167,7 +2278,7 @@ int fdt_del_node(void *fdt, int nodeoffset); * returns: * 0, on success * -FDT_ERR_NOSPACE, there's not enough space in the base device tree - * -FDT_ERR_NOTFOUND, the overlay points to some inexistant nodes or + * -FDT_ERR_NOTFOUND, the overlay points to some nonexistent nodes or * properties in the base DT * -FDT_ERR_BADPHANDLE, * -FDT_ERR_BADOVERLAY, diff --git a/scripts/dtc/libfdt/libfdt_internal.h b/scripts/dtc/libfdt/libfdt_internal.h index 16bda1906a7b..b60b5456f596 100644 --- a/scripts/dtc/libfdt/libfdt_internal.h +++ b/scripts/dtc/libfdt/libfdt_internal.h @@ -20,7 +20,15 @@ int32_t fdt_ro_probe_(const void *fdt); int fdt_check_node_offset_(const void *fdt, int offset); int fdt_check_prop_offset_(const void *fdt, int offset); -const char *fdt_find_string_(const char *strtab, int tabsize, const char *s); + +const char *fdt_find_string_len_(const char *strtab, int tabsize, const char *s, + int s_len); +static inline const char *fdt_find_string_(const char *strtab, int tabsize, + const char *s) +{ + return fdt_find_string_len_(strtab, tabsize, s, strlen(s)); +} + int fdt_node_end_offset_(void *fdt, int nodeoffset); static inline const void *fdt_offset_ptr_(const void *fdt, int offset) @@ -47,8 +55,8 @@ static inline struct fdt_reserve_entry *fdt_mem_rsv_w_(void *fdt, int n) } /* - * Internal helpers to access tructural elements of the device tree - * blob (rather than for exaple reading integers from within property + * Internal helpers to access structural elements of the device tree + * blob (rather than for example reading integers from within property * values). We assume that we are either given a naturally aligned * address for the platform or if we are not, we are on a platform * where unaligned memory reads will be handled in a graceful manner. diff --git a/scripts/dtc/livetree.c b/scripts/dtc/livetree.c index 49f723002f85..d51d05830b18 100644 --- a/scripts/dtc/livetree.c +++ b/scripts/dtc/livetree.c @@ -174,7 +174,7 @@ struct node *merge_nodes(struct node *old_node, struct node *new_node) old_prop->val = new_prop->val; old_prop->deleted = 0; - free(old_prop->srcpos); + srcpos_free(old_prop->srcpos); old_prop->srcpos = new_prop->srcpos; free(new_prop); new_prop = NULL; @@ -504,7 +504,7 @@ struct node *get_subnode(struct node *node, const char *nodename) struct node *child; for_each_child(node, child) - if (streq(child->name, nodename)) + if (streq(child->name, nodename) && !child->deleted) return child; return NULL; @@ -1014,9 +1014,7 @@ static void add_local_fixup_entry(struct dt_info *dti, /* walk the path components creating nodes if they don't exist */ for (wn = lfn, i = 1; i < depth; i++, wn = nwn) { /* if no node exists, create it */ - nwn = get_subnode(wn, compp[i]); - if (!nwn) - nwn = build_and_name_child_node(wn, compp[i]); + nwn = build_root_node(wn, compp[i]); } free(compp); @@ -1058,16 +1056,29 @@ void generate_label_tree(struct dt_info *dti, const char *name, bool allocph) void generate_fixups_tree(struct dt_info *dti, const char *name) { + struct node *n = get_subnode(dti->dt, name); + + /* Start with an empty __fixups__ node to not get duplicates */ + if (n) + n->deleted = true; + if (!any_fixup_tree(dti, dti->dt)) return; - generate_fixups_tree_internal(dti, build_root_node(dti->dt, name), + generate_fixups_tree_internal(dti, + build_and_name_child_node(dti->dt, name), dti->dt); } void generate_local_fixups_tree(struct dt_info *dti, const char *name) { + struct node *n = get_subnode(dti->dt, name); + + /* Start with an empty __local_fixups__ node to not get duplicates */ + if (n) + n->deleted = true; if (!any_local_fixup_tree(dti, dti->dt)) return; - generate_local_fixups_tree_internal(dti, build_root_node(dti->dt, name), + generate_local_fixups_tree_internal(dti, + build_and_name_child_node(dti->dt, name), dti->dt); } diff --git a/scripts/dtc/srcpos.c b/scripts/dtc/srcpos.c index 8e4d18a90b47..5bb57bf6856c 100644 --- a/scripts/dtc/srcpos.c +++ b/scripts/dtc/srcpos.c @@ -160,8 +160,10 @@ FILE *srcfile_relative_open(const char *fname, char **fullnamep) strerror(errno)); } - if (depfile) - fprintf(depfile, " %s", fullname); + if (depfile) { + fputc(' ', depfile); + fprint_path_escaped(depfile, fullname); + } if (fullnamep) *fullnamep = fullname; @@ -285,6 +287,17 @@ struct srcpos *srcpos_extend(struct srcpos *pos, struct srcpos *newtail) return pos; } +void srcpos_free(struct srcpos *pos) +{ + struct srcpos *p_next; + + while (pos) { + p_next = pos->next; + free(pos); + pos = p_next; + } +} + char * srcpos_string(struct srcpos *pos) { diff --git a/scripts/dtc/srcpos.h b/scripts/dtc/srcpos.h index 4318d7ad34d9..4d60b50e3119 100644 --- a/scripts/dtc/srcpos.h +++ b/scripts/dtc/srcpos.h @@ -88,6 +88,7 @@ extern void srcpos_update(struct srcpos *pos, const char *text, int len); extern struct srcpos *srcpos_copy(struct srcpos *pos); extern struct srcpos *srcpos_extend(struct srcpos *new_srcpos, struct srcpos *old_srcpos); +extern void srcpos_free(struct srcpos *pos); extern char *srcpos_string(struct srcpos *pos); extern char *srcpos_string_first(struct srcpos *pos, int level); extern char *srcpos_string_last(struct srcpos *pos, int level); diff --git a/scripts/dtc/treesource.c b/scripts/dtc/treesource.c index ae15839ba6a5..d25f01fc6937 100644 --- a/scripts/dtc/treesource.c +++ b/scripts/dtc/treesource.c @@ -139,26 +139,48 @@ static const char *delim_end[] = { [TYPE_STRING] = "", }; +/* + * The invariants in the marker list are: + * - offsets are non-strictly monotonically increasing + * - for a single offset there is at most one type marker + * - for a single offset that has both a type marker and non-type markers, the + * type marker appears before the others. + */ +static struct marker **add_marker(struct marker **mi, + enum markertype type, unsigned int offset, char *ref) +{ + struct marker *nm; + + while (*mi && (*mi)->offset < offset) + mi = &(*mi)->next; + + if (*mi && (*mi)->offset == offset && is_type_marker((*mi)->type)) { + if (is_type_marker(type)) + return mi; + mi = &(*mi)->next; + } + + if (*mi && (*mi)->offset == offset && type == (*mi)->type) + return mi; + + nm = xmalloc(sizeof(*nm)); + nm->type = type; + nm->offset = offset; + nm->ref = ref; + nm->next = *mi; + *mi = nm; + + return &nm->next; +} + static void add_string_markers(struct property *prop) { int l, len = prop->val.len; const char *p = prop->val.val; + struct marker **mi = &prop->val.markers; - for (l = strlen(p) + 1; l < len; l += strlen(p + l) + 1) { - struct marker *m, **nextp; - - m = xmalloc(sizeof(*m)); - m->offset = l; - m->type = TYPE_STRING; - m->ref = NULL; - m->next = NULL; - - /* Find the end of the markerlist */ - nextp = &prop->val.markers; - while (*nextp) - nextp = &((*nextp)->next); - *nextp = m; - } + for (l = strlen(p) + 1; l < len; l += strlen(p + l) + 1) + mi = add_marker(mi, TYPE_STRING, l, NULL); } static enum markertype guess_value_type(struct property *prop) diff --git a/scripts/dtc/util.c b/scripts/dtc/util.c index 507f0120cd13..412592320265 100644 --- a/scripts/dtc/util.c +++ b/scripts/dtc/util.c @@ -23,6 +23,22 @@ #include "util.h" #include "version_gen.h" +void fprint_path_escaped(FILE *fp, const char *path) +{ + const char *p = path; + + while (*p) { + if (*p == ' ') { + fputc('\\', fp); + fputc(' ', fp); + } else { + fputc(*p, fp); + } + + p++; + } +} + char *xstrdup(const char *s) { int len = strlen(s) + 1; diff --git a/scripts/dtc/util.h b/scripts/dtc/util.h index b448cd79efd3..800f2e2c55b1 100644 --- a/scripts/dtc/util.h +++ b/scripts/dtc/util.h @@ -42,6 +42,11 @@ static inline void NORETURN PRINTF(1, 2) die(const char *str, ...) exit(1); } +/** + * Writes path to fp, escaping spaces with a backslash. + */ +void fprint_path_escaped(FILE *fp, const char *path); + static inline void *xmalloc(size_t len) { void *new = malloc(len); diff --git a/scripts/dtc/version_gen.h b/scripts/dtc/version_gen.h index bf81ce593685..226c48bf75dc 100644 --- a/scripts/dtc/version_gen.h +++ b/scripts/dtc/version_gen.h @@ -1 +1 @@ -#define DTC_VERSION "DTC 1.7.0-gbcd02b52" +#define DTC_VERSION "DTC 1.7.2-g52f07dcc" -- cgit From 6656ae4df1a6ad6dfb5c4ce4c76136a42abb9bf4 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:28 -0600 Subject: docs: kdoc: consolidate the stripping of private struct/union members There were two locations duplicating the logic of stripping private members and associated comments; coalesce them into one, and add some comments describing what's going on. Output change: we now no longer add extraneous white space around macro definitions. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-2-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index c3fe4bd5eab4..93fcd8807aa8 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -81,6 +81,21 @@ multi_space = KernRe(r'\s\s+') def trim_whitespace(s): return multi_space.sub(' ', s.strip()) +# +# Remove struct/enum members that have been marked "private". +# +def trim_private_members(text): + # + # First look for a "public:" block that ends a private region, then + # handle the "private until the end" case. + # + text = KernRe(r'/\*\s*private:.*?/\*\s*public:.*?\*/', flags=re.S).sub('', text) + text = KernRe(r'/\*\s*private:.*', flags=re.S).sub('', text) + # + # We needed the comments to do the above, but now we can take them out. + # + return KernRe(r'\s*/\*.*?\*/\s*', flags=re.S).sub('', text).strip() + class state: """ State machine enums @@ -568,12 +583,6 @@ class KernelDoc: args_pattern = r'([^,)]+)' sub_prefixes = [ - (KernRe(r'\/\*\s*private:.*?\/\*\s*public:.*?\*\/', re.S | re.I), ''), - (KernRe(r'\/\*\s*private:.*', re.S | re.I), ''), - - # Strip comments - (KernRe(r'\/\*.*?\*\/', re.S), ''), - # Strip attributes (attribute, ' '), (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), @@ -648,6 +657,7 @@ class KernelDoc: (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), ] + members = trim_private_members(members) for search, sub in sub_prefixes: members = search.sub(sub, members) @@ -797,24 +807,18 @@ class KernelDoc: """ Stores an enum inside self.entries array. """ - - # Ignore members marked private - proto = KernRe(r'\/\*\s*private:.*?\/\*\s*public:.*?\*\/', flags=re.S).sub('', proto) - proto = KernRe(r'\/\*\s*private:.*}', flags=re.S).sub('}', proto) - - # Strip comments - proto = KernRe(r'\/\*.*?\*\/', flags=re.S).sub('', proto) - - # Strip #define macros inside enums + # + # Strip preprocessor directives. Note that this depends on the + # trailing semicolon we added in process_proto_type(). + # proto = KernRe(r'#\s*((define|ifdef|if)\s+|endif)[^;]*;', flags=re.S).sub('', proto) - # # Parse out the name and members of the enum. Typedef form first. # r = KernRe(r'typedef\s+enum\s*\{(.*)\}\s*(\w*)\s*;') if r.search(proto): declaration_name = r.group(2) - members = r.group(1).rstrip() + members = trim_private_members(r.group(1)) # # Failing that, look for a straight enum # @@ -822,7 +826,7 @@ class KernelDoc: r = KernRe(r'enum\s+(\w*)\s*\{(.*)\}') if r.match(proto): declaration_name = r.group(1) - members = r.group(2).rstrip() + members = trim_private_members(r.group(2)) # # OK, this isn't going to work. # -- cgit From 259feba4dde78f165b03e231ea9985dfe600c202 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:29 -0600 Subject: docs: kdoc: Move a regex line in dump_struct() The complex struct_members regex was defined far from its use; bring the two together. Remove some extraneous backslashes while making the move. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-3-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 93fcd8807aa8..aa6d11bf29b1 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -551,7 +551,6 @@ class KernelDoc: ] definition_body = r'\{(.*)\}\s*' + "(?:" + '|'.join(qualifiers) + ")?" - struct_members = KernRe(type_pattern + r'([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\}\;]*)(\;)') # Extract struct/union definition members = None @@ -683,6 +682,7 @@ class KernelDoc: # So, we need to have an extra loop on Python to override such # re limitation. + struct_members = KernRe(type_pattern + r'([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\};]*)(;)') while True: tuples = struct_members.findall(members) if not tuples: -- cgit From 5fd513f01169ae93d202b8c30f0837096664e7d7 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:30 -0600 Subject: docs: kdoc: backslashectomy in kdoc_parser A lot of the regular expressions in this file have extraneous backslashes that may have been needed in Perl, but aren't helpful here. Take them out to reduce slightly the visual noise. Escaping of (){}[] has been left in place, even when unnecessary, for visual clarity. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-4-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index aa6d11bf29b1..14ded23f11e0 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -46,7 +46,7 @@ doc_decl = doc_com + KernRe(r'(\w+)', cache=False) known_section_names = 'description|context|returns?|notes?|examples?' known_sections = KernRe(known_section_names, flags = re.I) doc_sect = doc_com + \ - KernRe(r'\s*(\@[.\w]+|\@\.\.\.|' + known_section_names + r')\s*:([^:].*)?$', + KernRe(r'\s*(@[.\w]+|@\.\.\.|' + known_section_names + r')\s*:([^:].*)?$', flags=re.I, cache=False) doc_content = doc_com_body + KernRe(r'(.*)', cache=False) @@ -60,7 +60,7 @@ attribute = KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False) export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) -type_param = KernRe(r"\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) +type_param = KernRe(r"@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) # # Tests for the beginning of a kerneldoc block in its various forms. @@ -405,7 +405,7 @@ class KernelDoc: for arg in args.split(splitter): # Strip comments - arg = KernRe(r'\/\*.*\*\/').sub('', arg) + arg = KernRe(r'/\*.*\*/').sub('', arg) # Ignore argument attributes arg = KernRe(r'\sPOS0?\s').sub(' ', arg) @@ -428,7 +428,7 @@ class KernelDoc: arg = arg.replace('#', ',') - r = KernRe(r'[^\(]+\(\*?\s*([\w\[\]\.]*)\s*\)') + r = KernRe(r'[^\(]+\(\*?\s*([\w\[\].]*)\s*\)') if r.match(arg): param = r.group(1) else: @@ -443,7 +443,7 @@ class KernelDoc: # Array-of-pointers arg = arg.replace('#', ',') - r = KernRe(r'[^\(]+\(\s*\*\s*([\w\[\]\.]*?)\s*(\s*\[\s*[\w]+\s*\]\s*)*\)') + r = KernRe(r'[^\(]+\(\s*\*\s*([\w\[\].]*?)\s*(\s*\[\s*[\w]+\s*\]\s*)*\)') if r.match(arg): param = r.group(1) else: @@ -709,7 +709,7 @@ class KernelDoc: if not arg: continue - r = KernRe(r'^([^\(]+\(\*?\s*)([\w\.]*)(\s*\).*)') + r = KernRe(r'^([^\(]+\(\*?\s*)([\w.]*)(\s*\).*)') if r.match(arg): # Pointer-to-function dtype = r.group(1) @@ -1044,7 +1044,7 @@ class KernelDoc: Stores a typedef inside self.entries array. """ - typedef_type = r'((?:\s+[\w\*]+\b){0,7}\s+(?:\w+\b|\*+))\s*' + typedef_type = r'((?:\s+[\w*]+\b){0,7}\s+(?:\w+\b|\*+))\s*' typedef_ident = r'\*?\s*(\w\S+)\s*' typedef_args = r'\s*\((.*)\);' @@ -1265,7 +1265,7 @@ class KernelDoc: self.dump_section() # Look for doc_com + + doc_end: - r = KernRe(r'\s*\*\s*[a-zA-Z_0-9:\.]+\*/') + r = KernRe(r'\s*\*\s*[a-zA-Z_0-9:.]+\*/') if r.match(line): self.emit_msg(ln, f"suspicious ending line: {line}") @@ -1476,7 +1476,7 @@ class KernelDoc: """Ancillary routine to process a function prototype""" # strip C99-style comments to end of line - line = KernRe(r"\/\/.*$", re.S).sub('', line) + line = KernRe(r"//.*$", re.S).sub('', line) # # Soak up the line's worth of prototype text, stopping at { or ; if present. # -- cgit From 64cf83bcd3217a9583caeb404ff136366a46705c Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:31 -0600 Subject: docs: kdoc: move the prefix transforms out of dump_struct() dump_struct is one of the longest functions in the kdoc_parser class, making it hard to read and reason about. Move the definition of the prefix transformations out of the function, join them with the definition of "attribute" (which was defined at the top of the file but only used here), and reformat the code slightly for shorter line widths. Just code movement in the end. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-5-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 179 +++++++++++++++++++++------------------- 1 file changed, 96 insertions(+), 83 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 14ded23f11e0..3d007d200da6 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -54,8 +54,6 @@ doc_inline_start = KernRe(r'^\s*/\*\*\s*$', cache=False) doc_inline_sect = KernRe(r'\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)', cache=False) doc_inline_end = KernRe(r'^\s*\*/\s*$', cache=False) doc_inline_oneline = KernRe(r'^\s*/\*\*\s*(@[\w\s]+):\s*(.*)\s*\*/\s*$', cache=False) -attribute = KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", - flags=re.I | re.S, cache=False) export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False) export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) @@ -74,6 +72,97 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' r'(?:[-:].*)?$', # description (not captured) cache = False) +# +# Here begins a long set of transformations to turn structure member prefixes +# and macro invocations into something we can parse and generate kdoc for. +# +struct_attribute = KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", + flags=re.I | re.S, cache=False) +struct_args_pattern = r'([^,)]+)' + +struct_prefixes = [ + # Strip attributes + (struct_attribute, ' '), + (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__packed\s*', re.S), ' '), + (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), + (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), + (KernRe(r'\s*____cacheline_aligned', re.S), ' '), + # + # Unwrap struct_group macros based on this definition: + # __struct_group(TAG, NAME, ATTRS, MEMBERS...) + # which has variants like: struct_group(NAME, MEMBERS...) + # Only MEMBERS arguments require documentation. + # + # Parsing them happens on two steps: + # + # 1. drop struct group arguments that aren't at MEMBERS, + # storing them as STRUCT_GROUP(MEMBERS) + # + # 2. remove STRUCT_GROUP() ancillary macro. + # + # The original logic used to remove STRUCT_GROUP() using an + # advanced regex: + # + # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; + # + # with two patterns that are incompatible with + # Python re module, as it has: + # + # - a recursive pattern: (?1) + # - an atomic grouping: (?>...) + # + # I tried a simpler version: but it didn't work either: + # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; + # + # As it doesn't properly match the end parenthesis on some cases. + # + # So, a better solution was crafted: there's now a NestedMatch + # class that ensures that delimiters after a search are properly + # matched. So, the implementation to drop STRUCT_GROUP() will be + # handled in separate. + # + (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), + (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), + (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), + (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), + # + # Replace macros + # + # TODO: use NestedMatch for FOO($1, $2, ...) matches + # + # it is better to also move those to the NestedMatch logic, + # to ensure that parenthesis will be properly matched. + # + (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), + r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), + (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), + r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), + (KernRe(r'DECLARE_BITMAP\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', + re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), + (KernRe(r'DECLARE_HASHTABLE\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', + re.S), r'unsigned long \1[1 << ((\2) - 1)]'), + (KernRe(r'DECLARE_KFIFO\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + + r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), + (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + struct_args_pattern + r',\s*' + + struct_args_pattern + r'\)', re.S), r'\2 *\1'), + (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + struct_args_pattern + r',\s*' + + struct_args_pattern + r'\)', re.S), r'\1 \2[]'), + (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), + (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'), +] +# +# Regexes here are guaranteed to have the end limiter matching +# the start delimiter. Yet, right now, only one replace group +# is allowed. +# +struct_nested_prefixes = [ + (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), +] + + # # A little helper to get rid of excess white space # @@ -578,91 +667,15 @@ class KernelDoc: self.emit_msg(ln, f"expecting prototype for {decl_type} {self.entry.identifier}. Prototype was for {decl_type} {declaration_name} instead\n") return - - args_pattern = r'([^,)]+)' - - sub_prefixes = [ - # Strip attributes - (attribute, ' '), - (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__packed\s*', re.S), ' '), - (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), - (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), - (KernRe(r'\s*____cacheline_aligned', re.S), ' '), - - # Unwrap struct_group macros based on this definition: - # __struct_group(TAG, NAME, ATTRS, MEMBERS...) - # which has variants like: struct_group(NAME, MEMBERS...) - # Only MEMBERS arguments require documentation. - # - # Parsing them happens on two steps: - # - # 1. drop struct group arguments that aren't at MEMBERS, - # storing them as STRUCT_GROUP(MEMBERS) - # - # 2. remove STRUCT_GROUP() ancillary macro. - # - # The original logic used to remove STRUCT_GROUP() using an - # advanced regex: - # - # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; - # - # with two patterns that are incompatible with - # Python re module, as it has: - # - # - a recursive pattern: (?1) - # - an atomic grouping: (?>...) - # - # I tried a simpler version: but it didn't work either: - # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; - # - # As it doesn't properly match the end parenthesis on some cases. - # - # So, a better solution was crafted: there's now a NestedMatch - # class that ensures that delimiters after a search are properly - # matched. So, the implementation to drop STRUCT_GROUP() will be - # handled in separate. - - (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), - (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), - (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), - (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), - - # Replace macros - # - # TODO: use NestedMatch for FOO($1, $2, ...) matches - # - # it is better to also move those to the NestedMatch logic, - # to ensure that parenthesis will be properly matched. - - (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), - (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), - (KernRe(r'DECLARE_BITMAP\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), - (KernRe(r'DECLARE_HASHTABLE\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'unsigned long \1[1 << ((\2) - 1)]'), - (KernRe(r'DECLARE_KFIFO\s*\(' + args_pattern + r',\s*' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'\2 *\1'), - (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'\2 *\1'), - (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'\1 \2[]'), - (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + args_pattern + r'\)', re.S), r'dma_addr_t \1'), - (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + args_pattern + r'\)', re.S), r'__u32 \1'), - ] - - # Regexes here are guaranteed to have the end limiter matching - # the start delimiter. Yet, right now, only one replace group - # is allowed. - - sub_nested_prefixes = [ - (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), - ] - + # + # Go through the list of members applying all of our transformations. + # members = trim_private_members(members) - for search, sub in sub_prefixes: + for search, sub in struct_prefixes: members = search.sub(sub, members) nested = NestedMatch() - - for search, sub in sub_nested_prefixes: + for search, sub in struct_nested_prefixes: members = nested.sub(search, sub, members) # Keeps the original declaration as-is -- cgit From 0f7344129434a6b44d4abb5080a9d67dd734ee07 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:32 -0600 Subject: docs: kdoc: split top-level prototype parsing out of dump_struct() Move the initial split of the prototype into its own function in the ongoing effort to cut dump_struct() down to size. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-6-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 43 +++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 23 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 3d007d200da6..ab896dcd9572 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -624,13 +624,11 @@ class KernelDoc: self.emit_msg(ln, f"No description found for return value of '{declaration_name}'") - def dump_struct(self, ln, proto): - """ - Store an entry for an struct or union - """ - + # + # Split apart a structure prototype; returns (struct|union, name, members) or None + # + def split_struct_proto(self, proto): type_pattern = r'(struct|union)' - qualifiers = [ "__attribute__", "__packed", @@ -638,34 +636,33 @@ class KernelDoc: "____cacheline_aligned_in_smp", "____cacheline_aligned", ] - definition_body = r'\{(.*)\}\s*' + "(?:" + '|'.join(qualifiers) + ")?" - # Extract struct/union definition - members = None - declaration_name = None - decl_type = None - r = KernRe(type_pattern + r'\s+(\w+)\s*' + definition_body) if r.search(proto): - decl_type = r.group(1) - declaration_name = r.group(2) - members = r.group(3) + return (r.group(1), r.group(2), r.group(3)) else: r = KernRe(r'typedef\s+' + type_pattern + r'\s*' + definition_body + r'\s*(\w+)\s*;') - if r.search(proto): - decl_type = r.group(1) - declaration_name = r.group(3) - members = r.group(2) + return (r.group(1), r.group(3), r.group(2)) + return None - if not members: + def dump_struct(self, ln, proto): + """ + Store an entry for an struct or union + """ + # + # Do the basic parse to get the pieces of the declaration. + # + struct_parts = self.split_struct_proto(proto) + if not struct_parts: self.emit_msg(ln, f"{proto} error: Cannot parse struct or union!") return + decl_type, declaration_name, members = struct_parts if self.entry.identifier != declaration_name: - self.emit_msg(ln, - f"expecting prototype for {decl_type} {self.entry.identifier}. Prototype was for {decl_type} {declaration_name} instead\n") + self.emit_msg(ln, f"expecting prototype for {decl_type} {self.entry.identifier}. " + f"Prototype was for {decl_type} {declaration_name} instead\n") return # # Go through the list of members applying all of our transformations. @@ -695,7 +692,7 @@ class KernelDoc: # So, we need to have an extra loop on Python to override such # re limitation. - struct_members = KernRe(type_pattern + r'([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\};]*)(;)') + struct_members = KernRe(r'(struct|union)([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\};]*)(;)') while True: tuples = struct_members.findall(members) if not tuples: -- cgit From 77e3c875f0a83d9192079e88d8569ac36c6b6bea Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:33 -0600 Subject: docs: kdoc: split struct-member rewriting out of dump_struct() The massive loop that massages struct members shares no data with the rest of dump_struct(); split it out into its own function. Code movement only, no other changes. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-7-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 65 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 31 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index ab896dcd9572..fbd7f6ce3360 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -647,37 +647,7 @@ class KernelDoc: return (r.group(1), r.group(3), r.group(2)) return None - def dump_struct(self, ln, proto): - """ - Store an entry for an struct or union - """ - # - # Do the basic parse to get the pieces of the declaration. - # - struct_parts = self.split_struct_proto(proto) - if not struct_parts: - self.emit_msg(ln, f"{proto} error: Cannot parse struct or union!") - return - decl_type, declaration_name, members = struct_parts - - if self.entry.identifier != declaration_name: - self.emit_msg(ln, f"expecting prototype for {decl_type} {self.entry.identifier}. " - f"Prototype was for {decl_type} {declaration_name} instead\n") - return - # - # Go through the list of members applying all of our transformations. - # - members = trim_private_members(members) - for search, sub in struct_prefixes: - members = search.sub(sub, members) - - nested = NestedMatch() - for search, sub in struct_nested_prefixes: - members = nested.sub(search, sub, members) - - # Keeps the original declaration as-is - declaration = members - + def rewrite_struct_members(self, members): # Split nested struct/union elements # # This loop was simpler at the original kernel-doc perl version, as @@ -768,6 +738,39 @@ class KernelDoc: newmember += f"{dtype} {s_id}.{name}; " members = members.replace(oldmember, newmember) + return members + + def dump_struct(self, ln, proto): + """ + Store an entry for an struct or union + """ + # + # Do the basic parse to get the pieces of the declaration. + # + struct_parts = self.split_struct_proto(proto) + if not struct_parts: + self.emit_msg(ln, f"{proto} error: Cannot parse struct or union!") + return + decl_type, declaration_name, members = struct_parts + + if self.entry.identifier != declaration_name: + self.emit_msg(ln, f"expecting prototype for {decl_type} {self.entry.identifier}. " + f"Prototype was for {decl_type} {declaration_name} instead\n") + return + # + # Go through the list of members applying all of our transformations. + # + members = trim_private_members(members) + for search, sub in struct_prefixes: + members = search.sub(sub, members) + + nested = NestedMatch() + for search, sub in struct_nested_prefixes: + members = nested.sub(search, sub, members) + + # Keeps the original declaration as-is + declaration = members + members = self.rewrite_struct_members(members) # Ignore other nested elements, like enums members = re.sub(r'(\{[^\{\}]*\})', '', members) -- cgit From f8208676c1c85c0b91e726954c05f5859b890ccb Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:34 -0600 Subject: docs: kdoc: rework the rewrite_struct_members() main loop Adopt a more Pythonic form for the main loop of this function, getting rid of the "while True:" construction and making the actual loop invariant explicit. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-8-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index fbd7f6ce3360..e11f3d6e9469 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -663,11 +663,8 @@ class KernelDoc: # re limitation. struct_members = KernRe(r'(struct|union)([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\};]*)(;)') - while True: - tuples = struct_members.findall(members) - if not tuples: - break - + tuples = struct_members.findall(members) + while tuples: for t in tuples: newmember = "" maintype = t[0] @@ -738,6 +735,7 @@ class KernelDoc: newmember += f"{dtype} {s_id}.{name}; " members = members.replace(oldmember, newmember) + tuples = struct_members.findall(members) return members def dump_struct(self, ln, proto): -- cgit From fb20e610393b02a832a0bf4964e12c20a7ffa2f8 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:35 -0600 Subject: docs: kdoc: remove an extraneous strip() call ...the variable in question was already strip()ed at the top of the loop. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-9-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 1 - 1 file changed, 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index e11f3d6e9469..0c279aa802a0 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -703,7 +703,6 @@ class KernelDoc: newmember += f"{dtype}{s_id}.{name}{extra}; " else: - arg = arg.strip() # Handle bitmaps arg = KernRe(r':\s*\d+\s*').sub('', arg) -- cgit From a8c4b0a8f1969e5ee0e8abb62cde8f0d7bcc2009 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:36 -0600 Subject: docs: kdoc: Some rewrite_struct_members() commenting Add comments to rewrite_struct_members() describing what it is actually doing, and reformat/comment the main struct_members regex so that it is (more) comprehensible to humans. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-10-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 0c279aa802a0..e3d0270b1a19 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -647,22 +647,28 @@ class KernelDoc: return (r.group(1), r.group(3), r.group(2)) return None + # + # Rewrite the members of a structure or union for easier formatting later on. + # Among other things, this function will turn a member like: + # + # struct { inner_members; } foo; + # + # into: + # + # struct foo; inner_members; + # def rewrite_struct_members(self, members): - # Split nested struct/union elements - # - # This loop was simpler at the original kernel-doc perl version, as - # while ($members =~ m/$struct_members/) { ... } - # reads 'members' string on each interaction. # - # Python behavior is different: it parses 'members' only once, - # creating a list of tuples from the first interaction. + # Process struct/union members from the most deeply nested outward. The + # trick is in the ^{ below - it prevents a match of an outer struct/union + # until the inner one has been munged (removing the "{" in the process). # - # On other words, this won't get nested structs. - # - # So, we need to have an extra loop on Python to override such - # re limitation. - - struct_members = KernRe(r'(struct|union)([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\};]*)(;)') + struct_members = KernRe(r'(struct|union)' # 0: declaration type + r'([^\{\};]+)' # 1: possible name + r'(\{)' + r'([^\{\}]*)' # 3: Contents of declaration + r'(\})' + r'([^\{\};]*)(;)') # 5: Remaining stuff after declaration tuples = struct_members.findall(members) while tuples: for t in tuples: -- cgit From e6dd4e2a5ca1c1e2fb168249532da1d95b5b24af Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:37 -0600 Subject: docs: kdoc: further rewrite_struct_members() cleanup Get rid of some redundant checks, and generally tighten up the code; no logical change. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-11-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 86 ++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 45 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index e3d0270b1a19..b3f937901037 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -673,73 +673,69 @@ class KernelDoc: while tuples: for t in tuples: newmember = "" - maintype = t[0] - s_ids = t[5] - content = t[3] - - oldmember = "".join(t) - - for s_id in s_ids.split(','): + oldmember = "".join(t) # Reconstruct the original formatting + dtype, name, lbr, content, rbr, rest, semi = t + # + # Pass through each field name, normalizing the form and formatting. + # + for s_id in rest.split(','): s_id = s_id.strip() - - newmember += f"{maintype} {s_id}; " + newmember += f"{dtype} {s_id}; " + # + # Remove bitfield/array/pointer info, getting the bare name. + # s_id = KernRe(r'[:\[].*').sub('', s_id) s_id = KernRe(r'^\s*\**(\S+)\s*').sub(r'\1', s_id) - + # + # Pass through the members of this inner structure/union. + # for arg in content.split(';'): arg = arg.strip() - - if not arg: - continue - + # + # Look for (type)(*name)(args) - pointer to function + # r = KernRe(r'^([^\(]+\(\*?\s*)([\w.]*)(\s*\).*)') if r.match(arg): + dtype, name, extra = r.group(1), r.group(2), r.group(3) # Pointer-to-function - dtype = r.group(1) - name = r.group(2) - extra = r.group(3) - - if not name: - continue - if not s_id: # Anonymous struct/union newmember += f"{dtype}{name}{extra}; " else: newmember += f"{dtype}{s_id}.{name}{extra}; " - + # + # Otherwise a non-function member. + # else: - # Handle bitmaps + # + # Remove bitmap and array portions and spaces around commas + # arg = KernRe(r':\s*\d+\s*').sub('', arg) - - # Handle arrays arg = KernRe(r'\[.*\]').sub('', arg) - - # Handle multiple IDs arg = KernRe(r'\s*,\s*').sub(',', arg) - + # + # Look for a normal decl - "type name[,name...]" + # r = KernRe(r'(.*)\s+([\S+,]+)') - if r.search(arg): - dtype = r.group(1) - names = r.group(2) + for name in r.group(2).split(','): + name = KernRe(r'^\s*\**(\S+)\s*').sub(r'\1', name) + if not s_id: + # Anonymous struct/union + newmember += f"{r.group(1)} {name}; " + else: + newmember += f"{r.group(1)} {s_id}.{name}; " else: newmember += f"{arg}; " - continue - - for name in names.split(','): - name = KernRe(r'^\s*\**(\S+)\s*').sub(r'\1', name).strip() - - if not name: - continue - - if not s_id: - # Anonymous struct/union - newmember += f"{dtype} {name}; " - else: - newmember += f"{dtype} {s_id}.{name}; " - + # + # At the end of the s_id loop, replace the original declaration with + # the munged version. + # members = members.replace(oldmember, newmember) + # + # End of the tuple loop - search again and see if there are outer members + # that now turn up. + # tuples = struct_members.findall(members) return members -- cgit From 23c47b09315935df140ca5ce2ddddb85453ed64d Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:38 -0600 Subject: docs: kdoc: extract output formatting from dump_struct() The last thing done in dump_struct() is to format the structure for printing. That, too, is a separate activity; split it out into its own function. dump_struct() now fits in a single, full-hight editor screen. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-12-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 72 +++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 35 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index b3f937901037..878fbfab4ac7 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -739,6 +739,42 @@ class KernelDoc: tuples = struct_members.findall(members) return members + # + # Format the struct declaration into a standard form for inclusion in the + # resulting docs. + # + def format_struct_decl(self, declaration): + # + # Insert newlines, get rid of extra spaces. + # + declaration = KernRe(r'([\{;])').sub(r'\1\n', declaration) + declaration = KernRe(r'\}\s+;').sub('};', declaration) + # + # Format inline enums with each member on its own line. + # + r = KernRe(r'(enum\s+\{[^\}]+),([^\n])') + while r.search(declaration): + declaration = r.sub(r'\1,\n\2', declaration) + # + # Now go through and supply the right number of tabs + # for each line. + # + def_args = declaration.split('\n') + level = 1 + declaration = "" + for clause in def_args: + clause = KernRe(r'\s+').sub(' ', clause.strip(), count=1) + if clause: + if '}' in clause and level > 1: + level -= 1 + if not clause.startswith('#'): + declaration += "\t" * level + declaration += "\t" + clause + "\n" + if "{" in clause and "}" not in clause: + level += 1 + return declaration + + def dump_struct(self, ln, proto): """ Store an entry for an struct or union @@ -777,42 +813,8 @@ class KernelDoc: self.create_parameter_list(ln, decl_type, members, ';', declaration_name) self.check_sections(ln, declaration_name, decl_type) - - # Adjust declaration for better display - declaration = KernRe(r'([\{;])').sub(r'\1\n', declaration) - declaration = KernRe(r'\}\s+;').sub('};', declaration) - - # Better handle inlined enums - while True: - r = KernRe(r'(enum\s+\{[^\}]+),([^\n])') - if not r.search(declaration): - break - - declaration = r.sub(r'\1,\n\2', declaration) - - def_args = declaration.split('\n') - level = 1 - declaration = "" - for clause in def_args: - - clause = clause.strip() - clause = KernRe(r'\s+').sub(' ', clause, count=1) - - if not clause: - continue - - if '}' in clause and level > 1: - level -= 1 - - if not KernRe(r'^\s*#').match(clause): - declaration += "\t" * level - - declaration += "\t" + clause + "\n" - if "{" in clause and "}" not in clause: - level += 1 - self.output_declaration(decl_type, declaration_name, - definition=declaration, + definition=self.format_struct_decl(declaration), purpose=self.entry.declaration_purpose) def dump_enum(self, ln, proto): -- cgit From e282303e718b2007b3db77c6db75ecaf4419a1af Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 7 Aug 2025 15:16:39 -0600 Subject: docs: kdoc: a few final dump_struct() touches Add a couple more comments so that each phase of the process is now clearly marked. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250807211639.47286-13-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 878fbfab4ac7..9b21fb86709a 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -802,14 +802,15 @@ class KernelDoc: nested = NestedMatch() for search, sub in struct_nested_prefixes: members = nested.sub(search, sub, members) - - # Keeps the original declaration as-is + # + # Deal with embedded struct and union members, and drop enums entirely. + # declaration = members members = self.rewrite_struct_members(members) - - # Ignore other nested elements, like enums members = re.sub(r'(\{[^\{\}]*\})', '', members) - + # + # Output the result and we are done. + # self.create_parameter_list(ln, decl_type, members, ';', declaration_name) self.check_sections(ln, declaration_name, decl_type) -- cgit From fc973dcd73f242480c61eccb1aa7306adafd2907 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 29 Jul 2025 18:43:03 +0200 Subject: docs: kernel-doc: avoid script crash on ancient Python While we do need at least 3.6 for kernel-doc to work, and at least 3.7 for it to output functions and structs with parameters at the right order, let the python binary be compatible with legacy versions. The rationale is that the Kernel build nowadays calls kernel-doc with -none on some places. Better not to bail out when older versions are found. With that, potentially this will run with python 2.7 and 3.2+, according with vermin: $ vermin --no-tips -v ./scripts/kernel-doc Detecting python files.. Analyzing using 24 processes.. 2.7, 3.2 /new_devel/v4l/docs/scripts/kernel-doc Minimum required versions: 2.7, 3.2 3.2 minimal requirement is due to argparse. The minimal version I could check was version 3.4 (using anaconda). Anaconda doesn't support 3.2 or 3.3 anymore, and 3.2 doesn't even compile (I tested compiling Python 3.2 on Fedora 42 and on Fedora 32 - no show). With 3.4, the script didn't crash and emitted the right warning: $ conda create -n py34 python=3.4 $ conda activate py34 python --version Python 3.4.5 $ python ./scripts/kernel-doc --none include/media Error: Python 3.6 or later is required by kernel-doc $ conda deactivate $ python --version Python 3.13.5 $ python ./scripts/kernel-doc --none include/media (no warnings and script ran properly) Supporting 2.7 is out of scope, as it is EOL for 5 years, and changing shebang to point to "python" instead of "python3" would have a wider impact. I did some extra checks about the differences from 3.2 and 3.4, and didn't find anything that would cause troubles: grep -rE "yield from|asyncio|pathlib|async|await|enum" scripts/kernel-doc Also, it doesn't use "@" operator. So, I'm confident that it should run (producing the exit warning) since Python 3.2. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/87d55e76b0b1391cb7a83e3e965dbddb83fa9786.1753806485.git.mchehab+huawei@kernel.org --- scripts/kernel-doc.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index fc3d46ef519f..d9fe2bcbd39c 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -2,8 +2,17 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright(c) 2025: Mauro Carvalho Chehab . # -# pylint: disable=C0103,R0915 -# +# pylint: disable=C0103,R0912,R0914,R0915 + +# NOTE: While kernel-doc requires at least version 3.6 to run, the +# command line should work with Python 3.2+ (tested with 3.4). +# The rationale is that it shall fail gracefully during Kernel +# compilation with older Kernel versions. Due to that: +# - encoding line is needed here; +# - no f-strings can be used on this file. +# - the libraries that require newer versions can only be included +# after Python version is checked. + # Converted from the kernel-doc script originally written in Perl # under GPLv2, copyrighted since 1998 by the following authors: # @@ -107,9 +116,6 @@ SRC_DIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) -from kdoc_files import KernelFiles # pylint: disable=C0413 -from kdoc_output import RestFormat, ManFormat # pylint: disable=C0413 - DESC = """ Read C language source or header FILEs, extract embedded documentation comments, and print formatted documentation to standard output. @@ -273,14 +279,22 @@ def main(): python_ver = sys.version_info[:2] if python_ver < (3,6): - logger.warning("Python 3.6 or later is required by kernel-doc") + # Depending on Kernel configuration, kernel-doc --none is called at + # build time. As we don't want to break compilation due to the + # usage of an old Python version, return 0 here. + if args.none: + logger.error("Python 3.6 or later is required by kernel-doc. skipping checks") + sys.exit(0) - # Return 0 here to avoid breaking compilation - sys.exit(0) + sys.exit("Python 3.6 or later is required by kernel-doc. Aborting.") if python_ver < (3,7): logger.warning("Python 3.7 or later is required for correct results") + # Import kernel-doc libraries only after checking Python version + from kdoc_files import KernelFiles # pylint: disable=C0415 + from kdoc_output import RestFormat, ManFormat # pylint: disable=C0415 + if args.man: out_style = ManFormat(modulename=args.modulename) elif args.none: @@ -308,11 +322,11 @@ def main(): sys.exit(0) if args.werror: - print(f"{error_count} warnings as errors") + print("%s warnings as errors" % error_count) # pylint: disable=C0209 sys.exit(error_count) if args.verbose: - print(f"{error_count} errors") + print("%s errors" % error_count) # pylint: disable=C0209 if args.none: sys.exit(0) -- cgit From 7bb184222e9fa8b839247c3a1cb1bcf628e519a5 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:18 +0200 Subject: scripts: sphinx-pre-install: fix version check for Fedora The script is now picking the wrong version. Fix it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/9d1e5c9906534e2bae586f891770066346463146.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 3f8d6925e896..07234d482fa8 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -413,7 +413,7 @@ sub give_redhat_hints() my $old = 0; my $rel; my $noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts"; - $rel = $1 if ($system_release =~ /(release|Linux)\s+(\d+)/); + $rel = $2 if ($system_release =~ /(release|Linux)\s+(\d+)/); if (!($system_release =~ /Fedora/)) { $map{"virtualenv"} = "python-virtualenv"; -- cgit From a9c50a51e0129057dbaf58fbaf7ba1a8a69e9b21 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:19 +0200 Subject: scripts: sphinx-pre-install: rename it to scripts/sphinx-pre-install.pl That helps us to later replace the scripts. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/08d9a32a5aaf3784fef016594efe505d7c5a2697.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 1056 ----------------------------------------- scripts/sphinx-pre-install.pl | 1056 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1056 insertions(+), 1056 deletions(-) delete mode 100755 scripts/sphinx-pre-install create mode 100755 scripts/sphinx-pre-install.pl (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install deleted file mode 100755 index 07234d482fa8..000000000000 --- a/scripts/sphinx-pre-install +++ /dev/null @@ -1,1056 +0,0 @@ -#!/usr/bin/env perl -# SPDX-License-Identifier: GPL-2.0-or-later -use strict; - -# Copyright (c) 2017-2020 Mauro Carvalho Chehab -# - -my $prefix = "./"; -$prefix = "$ENV{'srctree'}/" if ($ENV{'srctree'}); - -my $conf = $prefix . "Documentation/conf.py"; -my $requirement_file = $prefix . "Documentation/sphinx/requirements.txt"; -my $virtenv_prefix = "sphinx_"; - -# -# Static vars -# - -my %missing; -my $system_release; -my $need = 0; -my $optional = 0; -my $need_symlink = 0; -my $need_sphinx = 0; -my $need_pip = 0; -my $need_virtualenv = 0; -my $rec_sphinx_upgrade = 0; -my $verbose_warn_install = 1; -my $install = ""; -my $virtenv_dir = ""; -my $python_cmd = ""; -my $activate_cmd; -my $min_version; -my $cur_version; -my $rec_version = "3.4.3"; -my $latest_avail_ver; - -# -# Command line arguments -# - -my $pdf = 1; -my $virtualenv = 1; -my $version_check = 0; - -# -# List of required texlive packages on Fedora and OpenSuse -# - -my %texlive = ( - 'amsfonts.sty' => 'texlive-amsfonts', - 'amsmath.sty' => 'texlive-amsmath', - 'amssymb.sty' => 'texlive-amsfonts', - 'amsthm.sty' => 'texlive-amscls', - 'anyfontsize.sty' => 'texlive-anyfontsize', - 'atbegshi.sty' => 'texlive-oberdiek', - 'bm.sty' => 'texlive-tools', - 'capt-of.sty' => 'texlive-capt-of', - 'cmap.sty' => 'texlive-cmap', - 'ecrm1000.tfm' => 'texlive-ec', - 'eqparbox.sty' => 'texlive-eqparbox', - 'eu1enc.def' => 'texlive-euenc', - 'fancybox.sty' => 'texlive-fancybox', - 'fancyvrb.sty' => 'texlive-fancyvrb', - 'float.sty' => 'texlive-float', - 'fncychap.sty' => 'texlive-fncychap', - 'footnote.sty' => 'texlive-mdwtools', - 'framed.sty' => 'texlive-framed', - 'luatex85.sty' => 'texlive-luatex85', - 'multirow.sty' => 'texlive-multirow', - 'needspace.sty' => 'texlive-needspace', - 'palatino.sty' => 'texlive-psnfss', - 'parskip.sty' => 'texlive-parskip', - 'polyglossia.sty' => 'texlive-polyglossia', - 'tabulary.sty' => 'texlive-tabulary', - 'threeparttable.sty' => 'texlive-threeparttable', - 'titlesec.sty' => 'texlive-titlesec', - 'ucs.sty' => 'texlive-ucs', - 'upquote.sty' => 'texlive-upquote', - 'wrapfig.sty' => 'texlive-wrapfig', - 'ctexhook.sty' => 'texlive-ctex', -); - -# -# Subroutines that checks if a feature exists -# - -sub check_missing(%) -{ - my %map = %{$_[0]}; - - foreach my $prog (sort keys %missing) { - my $is_optional = $missing{$prog}; - - # At least on some LTS distros like CentOS 7, texlive doesn't - # provide all packages we need. When such distros are - # detected, we have to disable PDF output. - # - # So, we need to ignore the packages that distros would - # need for LaTeX to work - if ($is_optional == 2 && !$pdf) { - $optional--; - next; - } - - if ($verbose_warn_install) { - if ($is_optional) { - print "Warning: better to also install \"$prog\".\n"; - } else { - print "ERROR: please install \"$prog\", otherwise, build won't work.\n"; - } - } - if (defined($map{$prog})) { - $install .= " " . $map{$prog}; - } else { - $install .= " " . $prog; - } - } - - $install =~ s/^\s//; -} - -sub add_package($$) -{ - my $package = shift; - my $is_optional = shift; - - $missing{$package} = $is_optional; - if ($is_optional) { - $optional++; - } else { - $need++; - } -} - -sub check_missing_file($$$) -{ - my $files = shift; - my $package = shift; - my $is_optional = shift; - - for (@$files) { - return if(-e $_); - } - - add_package($package, $is_optional); -} - -sub findprog($) -{ - foreach(split(/:/, $ENV{PATH})) { - return "$_/$_[0]" if(-x "$_/$_[0]"); - } -} - -sub find_python_no_venv() -{ - my $prog = shift; - - my $cur_dir = qx(pwd); - $cur_dir =~ s/\s+$//; - - foreach my $dir (split(/:/, $ENV{PATH})) { - next if ($dir =~ m,($cur_dir)/sphinx,); - return "$dir/python3" if(-x "$dir/python3"); - } - foreach my $dir (split(/:/, $ENV{PATH})) { - next if ($dir =~ m,($cur_dir)/sphinx,); - return "$dir/python" if(-x "$dir/python"); - } - return "python"; -} - -sub check_program($$) -{ - my $prog = shift; - my $is_optional = shift; - - return $prog if findprog($prog); - - add_package($prog, $is_optional); -} - -sub check_perl_module($$) -{ - my $prog = shift; - my $is_optional = shift; - - my $err = system("perl -M$prog -e 1 2>/dev/null /dev/null"); - return if ($err == 0); - - add_package($prog, $is_optional); -} - -sub check_python_module($$) -{ - my $prog = shift; - my $is_optional = shift; - - return if (!$python_cmd); - - my $err = system("$python_cmd -c 'import $prog' 2>/dev/null /dev/null"); - return if ($err == 0); - - add_package($prog, $is_optional); -} - -sub check_rpm_missing($$) -{ - my @pkgs = @{$_[0]}; - my $is_optional = $_[1]; - - foreach my $prog(@pkgs) { - my $err = system("rpm -q '$prog' 2>/dev/null >/dev/null"); - add_package($prog, $is_optional) if ($err); - } -} - -sub check_pacman_missing($$) -{ - my @pkgs = @{$_[0]}; - my $is_optional = $_[1]; - - foreach my $prog(@pkgs) { - my $err = system("pacman -Q '$prog' 2>/dev/null >/dev/null"); - add_package($prog, $is_optional) if ($err); - } -} - -sub check_missing_tex($) -{ - my $is_optional = shift; - my $kpsewhich = findprog("kpsewhich"); - - foreach my $prog(keys %texlive) { - my $package = $texlive{$prog}; - if (!$kpsewhich) { - add_package($package, $is_optional); - next; - } - my $file = qx($kpsewhich $prog); - add_package($package, $is_optional) if ($file =~ /^\s*$/); - } -} - -sub get_sphinx_fname() -{ - if ($ENV{'SPHINXBUILD'}) { - return $ENV{'SPHINXBUILD'}; - } - - my $fname = "sphinx-build"; - return $fname if findprog($fname); - - $fname = "sphinx-build-3"; - if (findprog($fname)) { - $need_symlink = 1; - return $fname; - } - - return ""; -} - -sub get_sphinx_version($) -{ - my $cmd = shift; - my $ver; - - open IN, "$cmd --version 2>&1 |"; - while () { - if (m/^\s*sphinx-build\s+([\d\.]+)((\+\/[\da-f]+)|(b\d+))?$/) { - $ver=$1; - last; - } - # Sphinx 1.2.x uses a different format - if (m/^\s*Sphinx.*\s+([\d\.]+)$/) { - $ver=$1; - last; - } - } - close IN; - return $ver; -} - -sub check_sphinx() -{ - open IN, $conf or die "Can't open $conf"; - while () { - if (m/^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]/) { - $min_version=$1; - last; - } - } - close IN; - - die "Can't get needs_sphinx version from $conf" if (!$min_version); - - $virtenv_dir = $virtenv_prefix . "latest"; - - my $sphinx = get_sphinx_fname(); - if ($sphinx eq "") { - $need_sphinx = 1; - return; - } - - $cur_version = get_sphinx_version($sphinx); - die "$sphinx didn't return its version" if (!$cur_version); - - if ($cur_version lt $min_version) { - printf "ERROR: Sphinx version is %s. It should be >= %s\n", - $cur_version, $min_version; - $need_sphinx = 1; - return; - } - - return if ($cur_version lt $rec_version); - - # On version check mode, just assume Sphinx has all mandatory deps - exit (0) if ($version_check); -} - -# -# Ancillary subroutines -# - -sub catcheck($) -{ - my $res = ""; - $res = qx(cat $_[0]) if (-r $_[0]); - return $res; -} - -sub which($) -{ - my $file = shift; - my @path = split ":", $ENV{PATH}; - - foreach my $dir(@path) { - my $name = $dir.'/'.$file; - return $name if (-x $name ); - } - return undef; -} - -# -# Subroutines that check distro-specific hints -# - -sub give_debian_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-yaml", - "ensurepip" => "python3-venv", - "virtualenv" => "virtualenv", - "dot" => "graphviz", - "convert" => "imagemagick", - "Pod::Usage" => "perl-modules", - "xelatex" => "texlive-xetex", - "rsvg-convert" => "librsvg2-bin", - ); - - if ($pdf) { - check_missing_file(["/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty"], - "texlive-lang-chinese", 2); - - check_missing_file(["/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"], - "fonts-dejavu", 2); - - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc"], - "fonts-noto-cjk", 2); - } - - check_program("dvipng", 2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo apt-get install $install\n"); -} - -sub give_redhat_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-pyyaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive-xetex-bin", - "rsvg-convert" => "librsvg2-tools", - ); - - my @fedora26_opt_pkgs = ( - "graphviz-gd", # Fedora 26: needed for PDF support - ); - - my @fedora_tex_pkgs = ( - "texlive-collection-fontsrecommended", - "texlive-collection-latex", - "texlive-xecjk", - "dejavu-sans-fonts", - "dejavu-serif-fonts", - "dejavu-sans-mono-fonts", - ); - - # - # Checks valid for RHEL/CentOS version 7.x. - # - my $old = 0; - my $rel; - my $noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts"; - $rel = $2 if ($system_release =~ /(release|Linux)\s+(\d+)/); - - if (!($system_release =~ /Fedora/)) { - $map{"virtualenv"} = "python-virtualenv"; - - if ($rel && $rel < 8) { - $old = 1; - $pdf = 0; - - printf("Note: texlive packages on RHEL/CENTOS <= 7 are incomplete. Can't support PDF output\n"); - printf("If you want to build PDF, please read:\n"); - printf("\thttps://www.systutorials.com/241660/how-to-install-tex-live-on-centos-7-linux/\n"); - } - } else { - if ($rel && $rel < 26) { - $old = 1; - } - if ($rel && $rel >= 38) { - $noto_sans_redhat = "google-noto-sans-cjk-fonts"; - } - } - if (!$rel) { - printf("Couldn't identify release number\n"); - $old = 1; - $pdf = 0; - } - - if ($pdf) { - check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc"], - $noto_sans_redhat, 2); - } - - check_rpm_missing(\@fedora26_opt_pkgs, 2) if ($pdf && !$old); - check_rpm_missing(\@fedora_tex_pkgs, 2) if ($pdf); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - - if (!$old) { - # dnf, for Fedora 18+ - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo dnf install -y $install\n"); - } else { - # yum, for RHEL (and clones) or Fedora version < 18 - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo yum install -y $install\n"); - } -} - -sub give_opensuse_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-pyyaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive-xetex-bin", - ); - - # On Tumbleweed, this package is also named rsvg-convert - $map{"rsvg-convert"} = "rsvg-view" if (!($system_release =~ /Tumbleweed/)); - - my @suse_tex_pkgs = ( - "texlive-babel-english", - "texlive-caption", - "texlive-colortbl", - "texlive-courier", - "texlive-dvips", - "texlive-helvetic", - "texlive-makeindex", - "texlive-metafont", - "texlive-metapost", - "texlive-palatino", - "texlive-preview", - "texlive-times", - "texlive-zapfchan", - "texlive-zapfding", - ); - - $map{"latexmk"} = "texlive-latexmk-bin"; - - # FIXME: add support for installing CJK fonts - # - # I tried hard, but was unable to find a way to install - # "Noto Sans CJK SC" on openSUSE - - check_rpm_missing(\@suse_tex_pkgs, 2) if ($pdf); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo zypper install --no-recommends $install\n"); -} - -sub give_mageia_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-yaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive", - "rsvg-convert" => "librsvg2", - ); - - my @tex_pkgs = ( - "texlive-fontsextra", - ); - - $map{"latexmk"} = "texlive-collection-basic"; - - my $packager_cmd; - my $noto_sans; - if ($system_release =~ /OpenMandriva/) { - $packager_cmd = "dnf install"; - $noto_sans = "noto-sans-cjk-fonts"; - @tex_pkgs = ( "texlive-collection-fontsextra" ); - } else { - $packager_cmd = "urpmi"; - $noto_sans = "google-noto-sans-cjk-ttc-fonts"; - } - - - if ($pdf) { - check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/TTF/NotoSans-Regular.ttf"], - $noto_sans, 2); - } - - check_rpm_missing(\@tex_pkgs, 2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo $packager_cmd $install\n"); -} - -sub give_arch_linux_hints() -{ - my %map = ( - "yaml" => "python-yaml", - "virtualenv" => "python-virtualenv", - "dot" => "graphviz", - "convert" => "imagemagick", - "xelatex" => "texlive-xetex", - "latexmk" => "texlive-core", - "rsvg-convert" => "extra/librsvg", - ); - - my @archlinux_tex_pkgs = ( - "texlive-core", - "texlive-latexextra", - "ttf-dejavu", - ); - check_pacman_missing(\@archlinux_tex_pkgs, 2) if ($pdf); - - if ($pdf) { - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], - "noto-fonts-cjk", 2); - } - - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo pacman -S $install\n"); -} - -sub give_gentoo_hints() -{ - my %map = ( - "yaml" => "dev-python/pyyaml", - "virtualenv" => "dev-python/virtualenv", - "dot" => "media-gfx/graphviz", - "convert" => "media-gfx/imagemagick", - "xelatex" => "dev-texlive/texlive-xetex media-fonts/dejavu", - "rsvg-convert" => "gnome-base/librsvg", - ); - - check_missing_file(["/usr/share/fonts/dejavu/DejaVuSans.ttf"], - "media-fonts/dejavu", 2) if ($pdf); - - if ($pdf) { - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", - "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc"], - "media-fonts/noto-cjk", 2); - } - - check_missing(\%map); - - return if (!$need && !$optional); - - printf("You should run:\n") if ($verbose_warn_install); - printf("\n"); - - my $imagemagick = "media-gfx/imagemagick svg png"; - my $cairo = "media-gfx/graphviz cairo pdf"; - my $portage_imagemagick = "/etc/portage/package.use/imagemagick"; - my $portage_cairo = "/etc/portage/package.use/graphviz"; - - if (qx(grep imagemagick $portage_imagemagick 2>/dev/null) eq "") { - printf("\tsudo su -c 'echo \"$imagemagick\" > $portage_imagemagick'\n") - } - if (qx(grep graphviz $portage_cairo 2>/dev/null) eq "") { - printf("\tsudo su -c 'echo \"$cairo\" > $portage_cairo'\n"); - } - - printf("\tsudo emerge --ask $install\n"); - -} - -sub check_distros() -{ - # Distro-specific hints - if ($system_release =~ /Red Hat Enterprise Linux/) { - give_redhat_hints; - return; - } - if ($system_release =~ /CentOS/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Scientific Linux/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Oracle Linux Server/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Fedora/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Ubuntu/) { - give_debian_hints; - return; - } - if ($system_release =~ /Debian/) { - give_debian_hints; - return; - } - if ($system_release =~ /openSUSE/) { - give_opensuse_hints; - return; - } - if ($system_release =~ /Mageia/) { - give_mageia_hints; - return; - } - if ($system_release =~ /OpenMandriva/) { - give_mageia_hints; - return; - } - if ($system_release =~ /Arch Linux/) { - give_arch_linux_hints; - return; - } - if ($system_release =~ /Gentoo/) { - give_gentoo_hints; - return; - } - - # - # Fall-back to generic hint code for other distros - # That's far from ideal, specially for LaTeX dependencies. - # - my %map = ( - "sphinx-build" => "sphinx" - ); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - print "I don't know distro $system_release.\n"; - print "So, I can't provide you a hint with the install procedure.\n"; - print "There are likely missing dependencies.\n"; -} - -# -# Common dependencies -# - -sub deactivate_help() -{ - printf "\n If you want to exit the virtualenv, you can use:\n"; - printf "\tdeactivate\n"; -} - -sub get_virtenv() -{ - my $ver; - my $min_activate = "$ENV{'PWD'}/${virtenv_prefix}${min_version}/bin/activate"; - my @activates = glob "$ENV{'PWD'}/${virtenv_prefix}*/bin/activate"; - - @activates = sort {$b cmp $a} @activates; - - foreach my $f (@activates) { - next if ($f lt $min_activate); - - my $sphinx_cmd = $f; - $sphinx_cmd =~ s/activate/sphinx-build/; - next if (! -f $sphinx_cmd); - - my $ver = get_sphinx_version($sphinx_cmd); - - if (!$ver) { - $f =~ s#/bin/activate##; - print("Warning: virtual environment $f is not working.\nPython version upgrade? Remove it with:\n\n\trm -rf $f\n\n"); - } - - if ($need_sphinx && ($ver ge $min_version)) { - return ($f, $ver); - } elsif ($ver gt $cur_version) { - return ($f, $ver); - } - } - return ("", ""); -} - -sub recommend_sphinx_upgrade() -{ - my $venv_ver; - - # Avoid running sphinx-builds from venv if $cur_version is good - if ($cur_version && ($cur_version ge $rec_version)) { - $latest_avail_ver = $cur_version; - return; - } - - # Get the highest version from sphinx_*/bin/sphinx-build and the - # corresponding command to activate the venv/virtenv - ($activate_cmd, $venv_ver) = get_virtenv(); - - # Store the highest version from Sphinx existing virtualenvs - if (($activate_cmd ne "") && ($venv_ver gt $cur_version)) { - $latest_avail_ver = $venv_ver; - } else { - $latest_avail_ver = $cur_version if ($cur_version); - } - - # As we don't know package version of Sphinx, and there's no - # virtual environments, don't check if upgrades are needed - if (!$virtualenv) { - return if (!$latest_avail_ver); - } - - # Either there are already a virtual env or a new one should be created - $need_pip = 1; - - return if (!$latest_avail_ver); - - # Return if the reason is due to an upgrade or not - if ($latest_avail_ver lt $rec_version) { - $rec_sphinx_upgrade = 1; - } - - return $latest_avail_ver; -} - -# -# The logic here is complex, as it have to deal with different versions: -# - minimal supported version; -# - minimal PDF version; -# - recommended version. -# It also needs to work fine with both distro's package and venv/virtualenv -sub recommend_sphinx_version($) -{ - my $virtualenv_cmd = shift; - - # Version is OK. Nothing to do. - if ($cur_version && ($cur_version ge $rec_version)) { - return; - }; - - if (!$need_sphinx) { - # sphinx-build is present and its version is >= $min_version - - #only recommend enabling a newer virtenv version if makes sense. - if ($latest_avail_ver gt $cur_version) { - printf "\nYou may also use the newer Sphinx version $latest_avail_ver with:\n"; - printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - return if ($latest_avail_ver ge $rec_version); - } - - if (!$virtualenv) { - # No sphinx either via package or via virtenv. As we can't - # Compare the versions here, just return, recommending the - # user to install it from the package distro. - return if (!$latest_avail_ver); - - # User doesn't want a virtenv recommendation, but he already - # installed one via virtenv with a newer version. - # So, print commands to enable it - if ($latest_avail_ver gt $cur_version) { - printf "\nYou may also use the Sphinx virtualenv version $latest_avail_ver with:\n"; - printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - print "\n"; - } else { - $need++ if ($need_sphinx); - } - - # Suggest newer versions if current ones are too old - if ($latest_avail_ver && $latest_avail_ver ge $min_version) { - # If there's a good enough version, ask the user to enable it - if ($latest_avail_ver ge $rec_version) { - printf "\nNeed to activate Sphinx (version $latest_avail_ver) on virtualenv with:\n"; - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - - # Version is above the minimal required one, but may be - # below the recommended one. So, print warnings/notes - - if ($latest_avail_ver lt $rec_version) { - print "Warning: It is recommended at least Sphinx version $rec_version.\n"; - } - } - - # At this point, either it needs Sphinx or upgrade is recommended, - # both via pip - - if ($rec_sphinx_upgrade) { - if (!$virtualenv) { - print "Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n"; - } else { - print "To upgrade Sphinx, use:\n\n"; - } - } else { - print "\nSphinx needs to be installed either:\n1) via pip/pypi with:\n\n"; - } - - $python_cmd = find_python_no_venv(); - - printf "\t$virtualenv_cmd $virtenv_dir\n"; - - printf "\t. $virtenv_dir/bin/activate\n"; - printf "\tpip install -r $requirement_file\n"; - deactivate_help(); - - printf "\n2) As a package with:\n"; - - my $old_need = $need; - my $old_optional = $optional; - %missing = (); - $pdf = 0; - $optional = 0; - $install = ""; - $verbose_warn_install = 0; - - add_package("python-sphinx", 0); - - check_distros(); - - $need = $old_need; - $optional = $old_optional; - - printf "\n Please note that Sphinx >= 3.0 will currently produce false-positive\n"; - printf " warning when the same name is used for more than one type (functions,\n"; - printf " structs, enums,...). This is known Sphinx bug. For more details, see:\n"; - printf "\thttps://github.com/sphinx-doc/sphinx/pull/8313\n"; -} - -sub check_needs() -{ - # Check if Sphinx is already accessible from current environment - check_sphinx(); - - if ($system_release) { - print "Detected OS: $system_release.\n"; - } else { - print "Unknown OS\n"; - } - printf "Sphinx version: %s\n\n", $cur_version if ($cur_version); - - # Check python command line, trying first python3 - $python_cmd = findprog("python3"); - $python_cmd = check_program("python", 0) if (!$python_cmd); - - # Check the type of virtual env, depending on Python version - if ($python_cmd) { - if ($virtualenv) { - my $tmp = qx($python_cmd --version 2>&1); - if ($tmp =~ m/(\d+\.)(\d+\.)/) { - if ($1 < 3) { - # Fail if it finds python2 (or worse) - die "Python 3 is required to build the kernel docs\n"; - } - if ($1 == 3 && $2 < 3) { - # Need Python 3.3 or upper for venv - $need_virtualenv = 1; - } - } else { - die "Warning: couldn't identify $python_cmd version!"; - } - } else { - add_package("python-sphinx", 0); - } - } - - my $venv_ver = recommend_sphinx_upgrade(); - - my $virtualenv_cmd; - - if ($need_pip) { - # Set virtualenv command line, if python < 3.3 - if ($need_virtualenv) { - $virtualenv_cmd = findprog("virtualenv-3"); - $virtualenv_cmd = findprog("virtualenv-3.5") if (!$virtualenv_cmd); - if (!$virtualenv_cmd) { - check_program("virtualenv", 0); - $virtualenv_cmd = "virtualenv"; - } - } else { - $virtualenv_cmd = "$python_cmd -m venv"; - check_python_module("ensurepip", 0); - } - } - - # Check for needed programs/tools - check_perl_module("Pod::Usage", 0); - check_python_module("yaml", 0); - check_program("make", 0); - check_program("gcc", 0); - check_program("dot", 1); - check_program("convert", 1); - - # Extra PDF files - should use 2 for is_optional - check_program("xelatex", 2) if ($pdf); - check_program("rsvg-convert", 2) if ($pdf); - check_program("latexmk", 2) if ($pdf); - - # Do distro-specific checks and output distro-install commands - check_distros(); - - if (!$python_cmd) { - if ($need == 1) { - die "Can't build as $need mandatory dependency is missing"; - } elsif ($need) { - die "Can't build as $need mandatory dependencies are missing"; - } - } - - # Check if sphinx-build is called sphinx-build-3 - if ($need_symlink) { - printf "\tsudo ln -sf %s /usr/bin/sphinx-build\n\n", - which("sphinx-build-3"); - } - - recommend_sphinx_version($virtualenv_cmd); - printf "\n"; - - print "All optional dependencies are met.\n" if (!$optional); - - if ($need == 1) { - die "Can't build as $need mandatory dependency is missing"; - } elsif ($need) { - die "Can't build as $need mandatory dependencies are missing"; - } - - print "Needed package dependencies are met.\n"; -} - -# -# Main -# - -while (@ARGV) { - my $arg = shift(@ARGV); - - if ($arg eq "--no-virtualenv") { - $virtualenv = 0; - } elsif ($arg eq "--no-pdf"){ - $pdf = 0; - } elsif ($arg eq "--version-check"){ - $version_check = 1; - } else { - print "Usage:\n\t$0 <--no-virtualenv> <--no-pdf> <--version-check>\n\n"; - print "Where:\n"; - print "\t--no-virtualenv\t- Recommend installing Sphinx instead of using a virtualenv\n"; - print "\t--version-check\t- if version is compatible, don't check for missing dependencies\n"; - print "\t--no-pdf\t- don't check for dependencies required to build PDF docs\n\n"; - exit -1; - } -} - -# -# Determine the system type. There's no standard unique way that would -# work with all distros with a minimal package install. So, several -# methods are used here. -# -# By default, it will use lsb_release function. If not available, it will -# fail back to reading the known different places where the distro name -# is stored -# - -$system_release = qx(lsb_release -d) if which("lsb_release"); -$system_release =~ s/Description:\s*// if ($system_release); -$system_release = catcheck("/etc/system-release") if !$system_release; -$system_release = catcheck("/etc/redhat-release") if !$system_release; -$system_release = catcheck("/etc/lsb-release") if !$system_release; -$system_release = catcheck("/etc/gentoo-release") if !$system_release; - -# This seems more common than LSB these days -if (!$system_release) { - my %os_var; - if (open IN, "cat /etc/os-release|") { - while () { - if (m/^([\w\d\_]+)=\"?([^\"]*)\"?\n/) { - $os_var{$1}=$2; - } - } - $system_release = $os_var{"NAME"}; - if (defined($os_var{"VERSION_ID"})) { - $system_release .= " " . $os_var{"VERSION_ID"} if (defined($os_var{"VERSION_ID"})); - } else { - $system_release .= " " . $os_var{"VERSION"}; - } - } -} -$system_release = catcheck("/etc/issue") if !$system_release; -$system_release =~ s/\s+$//; - -check_needs; diff --git a/scripts/sphinx-pre-install.pl b/scripts/sphinx-pre-install.pl new file mode 100755 index 000000000000..07234d482fa8 --- /dev/null +++ b/scripts/sphinx-pre-install.pl @@ -0,0 +1,1056 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0-or-later +use strict; + +# Copyright (c) 2017-2020 Mauro Carvalho Chehab +# + +my $prefix = "./"; +$prefix = "$ENV{'srctree'}/" if ($ENV{'srctree'}); + +my $conf = $prefix . "Documentation/conf.py"; +my $requirement_file = $prefix . "Documentation/sphinx/requirements.txt"; +my $virtenv_prefix = "sphinx_"; + +# +# Static vars +# + +my %missing; +my $system_release; +my $need = 0; +my $optional = 0; +my $need_symlink = 0; +my $need_sphinx = 0; +my $need_pip = 0; +my $need_virtualenv = 0; +my $rec_sphinx_upgrade = 0; +my $verbose_warn_install = 1; +my $install = ""; +my $virtenv_dir = ""; +my $python_cmd = ""; +my $activate_cmd; +my $min_version; +my $cur_version; +my $rec_version = "3.4.3"; +my $latest_avail_ver; + +# +# Command line arguments +# + +my $pdf = 1; +my $virtualenv = 1; +my $version_check = 0; + +# +# List of required texlive packages on Fedora and OpenSuse +# + +my %texlive = ( + 'amsfonts.sty' => 'texlive-amsfonts', + 'amsmath.sty' => 'texlive-amsmath', + 'amssymb.sty' => 'texlive-amsfonts', + 'amsthm.sty' => 'texlive-amscls', + 'anyfontsize.sty' => 'texlive-anyfontsize', + 'atbegshi.sty' => 'texlive-oberdiek', + 'bm.sty' => 'texlive-tools', + 'capt-of.sty' => 'texlive-capt-of', + 'cmap.sty' => 'texlive-cmap', + 'ecrm1000.tfm' => 'texlive-ec', + 'eqparbox.sty' => 'texlive-eqparbox', + 'eu1enc.def' => 'texlive-euenc', + 'fancybox.sty' => 'texlive-fancybox', + 'fancyvrb.sty' => 'texlive-fancyvrb', + 'float.sty' => 'texlive-float', + 'fncychap.sty' => 'texlive-fncychap', + 'footnote.sty' => 'texlive-mdwtools', + 'framed.sty' => 'texlive-framed', + 'luatex85.sty' => 'texlive-luatex85', + 'multirow.sty' => 'texlive-multirow', + 'needspace.sty' => 'texlive-needspace', + 'palatino.sty' => 'texlive-psnfss', + 'parskip.sty' => 'texlive-parskip', + 'polyglossia.sty' => 'texlive-polyglossia', + 'tabulary.sty' => 'texlive-tabulary', + 'threeparttable.sty' => 'texlive-threeparttable', + 'titlesec.sty' => 'texlive-titlesec', + 'ucs.sty' => 'texlive-ucs', + 'upquote.sty' => 'texlive-upquote', + 'wrapfig.sty' => 'texlive-wrapfig', + 'ctexhook.sty' => 'texlive-ctex', +); + +# +# Subroutines that checks if a feature exists +# + +sub check_missing(%) +{ + my %map = %{$_[0]}; + + foreach my $prog (sort keys %missing) { + my $is_optional = $missing{$prog}; + + # At least on some LTS distros like CentOS 7, texlive doesn't + # provide all packages we need. When such distros are + # detected, we have to disable PDF output. + # + # So, we need to ignore the packages that distros would + # need for LaTeX to work + if ($is_optional == 2 && !$pdf) { + $optional--; + next; + } + + if ($verbose_warn_install) { + if ($is_optional) { + print "Warning: better to also install \"$prog\".\n"; + } else { + print "ERROR: please install \"$prog\", otherwise, build won't work.\n"; + } + } + if (defined($map{$prog})) { + $install .= " " . $map{$prog}; + } else { + $install .= " " . $prog; + } + } + + $install =~ s/^\s//; +} + +sub add_package($$) +{ + my $package = shift; + my $is_optional = shift; + + $missing{$package} = $is_optional; + if ($is_optional) { + $optional++; + } else { + $need++; + } +} + +sub check_missing_file($$$) +{ + my $files = shift; + my $package = shift; + my $is_optional = shift; + + for (@$files) { + return if(-e $_); + } + + add_package($package, $is_optional); +} + +sub findprog($) +{ + foreach(split(/:/, $ENV{PATH})) { + return "$_/$_[0]" if(-x "$_/$_[0]"); + } +} + +sub find_python_no_venv() +{ + my $prog = shift; + + my $cur_dir = qx(pwd); + $cur_dir =~ s/\s+$//; + + foreach my $dir (split(/:/, $ENV{PATH})) { + next if ($dir =~ m,($cur_dir)/sphinx,); + return "$dir/python3" if(-x "$dir/python3"); + } + foreach my $dir (split(/:/, $ENV{PATH})) { + next if ($dir =~ m,($cur_dir)/sphinx,); + return "$dir/python" if(-x "$dir/python"); + } + return "python"; +} + +sub check_program($$) +{ + my $prog = shift; + my $is_optional = shift; + + return $prog if findprog($prog); + + add_package($prog, $is_optional); +} + +sub check_perl_module($$) +{ + my $prog = shift; + my $is_optional = shift; + + my $err = system("perl -M$prog -e 1 2>/dev/null /dev/null"); + return if ($err == 0); + + add_package($prog, $is_optional); +} + +sub check_python_module($$) +{ + my $prog = shift; + my $is_optional = shift; + + return if (!$python_cmd); + + my $err = system("$python_cmd -c 'import $prog' 2>/dev/null /dev/null"); + return if ($err == 0); + + add_package($prog, $is_optional); +} + +sub check_rpm_missing($$) +{ + my @pkgs = @{$_[0]}; + my $is_optional = $_[1]; + + foreach my $prog(@pkgs) { + my $err = system("rpm -q '$prog' 2>/dev/null >/dev/null"); + add_package($prog, $is_optional) if ($err); + } +} + +sub check_pacman_missing($$) +{ + my @pkgs = @{$_[0]}; + my $is_optional = $_[1]; + + foreach my $prog(@pkgs) { + my $err = system("pacman -Q '$prog' 2>/dev/null >/dev/null"); + add_package($prog, $is_optional) if ($err); + } +} + +sub check_missing_tex($) +{ + my $is_optional = shift; + my $kpsewhich = findprog("kpsewhich"); + + foreach my $prog(keys %texlive) { + my $package = $texlive{$prog}; + if (!$kpsewhich) { + add_package($package, $is_optional); + next; + } + my $file = qx($kpsewhich $prog); + add_package($package, $is_optional) if ($file =~ /^\s*$/); + } +} + +sub get_sphinx_fname() +{ + if ($ENV{'SPHINXBUILD'}) { + return $ENV{'SPHINXBUILD'}; + } + + my $fname = "sphinx-build"; + return $fname if findprog($fname); + + $fname = "sphinx-build-3"; + if (findprog($fname)) { + $need_symlink = 1; + return $fname; + } + + return ""; +} + +sub get_sphinx_version($) +{ + my $cmd = shift; + my $ver; + + open IN, "$cmd --version 2>&1 |"; + while () { + if (m/^\s*sphinx-build\s+([\d\.]+)((\+\/[\da-f]+)|(b\d+))?$/) { + $ver=$1; + last; + } + # Sphinx 1.2.x uses a different format + if (m/^\s*Sphinx.*\s+([\d\.]+)$/) { + $ver=$1; + last; + } + } + close IN; + return $ver; +} + +sub check_sphinx() +{ + open IN, $conf or die "Can't open $conf"; + while () { + if (m/^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]/) { + $min_version=$1; + last; + } + } + close IN; + + die "Can't get needs_sphinx version from $conf" if (!$min_version); + + $virtenv_dir = $virtenv_prefix . "latest"; + + my $sphinx = get_sphinx_fname(); + if ($sphinx eq "") { + $need_sphinx = 1; + return; + } + + $cur_version = get_sphinx_version($sphinx); + die "$sphinx didn't return its version" if (!$cur_version); + + if ($cur_version lt $min_version) { + printf "ERROR: Sphinx version is %s. It should be >= %s\n", + $cur_version, $min_version; + $need_sphinx = 1; + return; + } + + return if ($cur_version lt $rec_version); + + # On version check mode, just assume Sphinx has all mandatory deps + exit (0) if ($version_check); +} + +# +# Ancillary subroutines +# + +sub catcheck($) +{ + my $res = ""; + $res = qx(cat $_[0]) if (-r $_[0]); + return $res; +} + +sub which($) +{ + my $file = shift; + my @path = split ":", $ENV{PATH}; + + foreach my $dir(@path) { + my $name = $dir.'/'.$file; + return $name if (-x $name ); + } + return undef; +} + +# +# Subroutines that check distro-specific hints +# + +sub give_debian_hints() +{ + my %map = ( + "python-sphinx" => "python3-sphinx", + "yaml" => "python3-yaml", + "ensurepip" => "python3-venv", + "virtualenv" => "virtualenv", + "dot" => "graphviz", + "convert" => "imagemagick", + "Pod::Usage" => "perl-modules", + "xelatex" => "texlive-xetex", + "rsvg-convert" => "librsvg2-bin", + ); + + if ($pdf) { + check_missing_file(["/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty"], + "texlive-lang-chinese", 2); + + check_missing_file(["/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"], + "fonts-dejavu", 2); + + check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc"], + "fonts-noto-cjk", 2); + } + + check_program("dvipng", 2) if ($pdf); + check_missing(\%map); + + return if (!$need && !$optional); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo apt-get install $install\n"); +} + +sub give_redhat_hints() +{ + my %map = ( + "python-sphinx" => "python3-sphinx", + "yaml" => "python3-pyyaml", + "virtualenv" => "python3-virtualenv", + "dot" => "graphviz", + "convert" => "ImageMagick", + "Pod::Usage" => "perl-Pod-Usage", + "xelatex" => "texlive-xetex-bin", + "rsvg-convert" => "librsvg2-tools", + ); + + my @fedora26_opt_pkgs = ( + "graphviz-gd", # Fedora 26: needed for PDF support + ); + + my @fedora_tex_pkgs = ( + "texlive-collection-fontsrecommended", + "texlive-collection-latex", + "texlive-xecjk", + "dejavu-sans-fonts", + "dejavu-serif-fonts", + "dejavu-sans-mono-fonts", + ); + + # + # Checks valid for RHEL/CentOS version 7.x. + # + my $old = 0; + my $rel; + my $noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts"; + $rel = $2 if ($system_release =~ /(release|Linux)\s+(\d+)/); + + if (!($system_release =~ /Fedora/)) { + $map{"virtualenv"} = "python-virtualenv"; + + if ($rel && $rel < 8) { + $old = 1; + $pdf = 0; + + printf("Note: texlive packages on RHEL/CENTOS <= 7 are incomplete. Can't support PDF output\n"); + printf("If you want to build PDF, please read:\n"); + printf("\thttps://www.systutorials.com/241660/how-to-install-tex-live-on-centos-7-linux/\n"); + } + } else { + if ($rel && $rel < 26) { + $old = 1; + } + if ($rel && $rel >= 38) { + $noto_sans_redhat = "google-noto-sans-cjk-fonts"; + } + } + if (!$rel) { + printf("Couldn't identify release number\n"); + $old = 1; + $pdf = 0; + } + + if ($pdf) { + check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc"], + $noto_sans_redhat, 2); + } + + check_rpm_missing(\@fedora26_opt_pkgs, 2) if ($pdf && !$old); + check_rpm_missing(\@fedora_tex_pkgs, 2) if ($pdf); + check_missing_tex(2) if ($pdf); + check_missing(\%map); + + return if (!$need && !$optional); + + if (!$old) { + # dnf, for Fedora 18+ + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo dnf install -y $install\n"); + } else { + # yum, for RHEL (and clones) or Fedora version < 18 + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo yum install -y $install\n"); + } +} + +sub give_opensuse_hints() +{ + my %map = ( + "python-sphinx" => "python3-sphinx", + "yaml" => "python3-pyyaml", + "virtualenv" => "python3-virtualenv", + "dot" => "graphviz", + "convert" => "ImageMagick", + "Pod::Usage" => "perl-Pod-Usage", + "xelatex" => "texlive-xetex-bin", + ); + + # On Tumbleweed, this package is also named rsvg-convert + $map{"rsvg-convert"} = "rsvg-view" if (!($system_release =~ /Tumbleweed/)); + + my @suse_tex_pkgs = ( + "texlive-babel-english", + "texlive-caption", + "texlive-colortbl", + "texlive-courier", + "texlive-dvips", + "texlive-helvetic", + "texlive-makeindex", + "texlive-metafont", + "texlive-metapost", + "texlive-palatino", + "texlive-preview", + "texlive-times", + "texlive-zapfchan", + "texlive-zapfding", + ); + + $map{"latexmk"} = "texlive-latexmk-bin"; + + # FIXME: add support for installing CJK fonts + # + # I tried hard, but was unable to find a way to install + # "Noto Sans CJK SC" on openSUSE + + check_rpm_missing(\@suse_tex_pkgs, 2) if ($pdf); + check_missing_tex(2) if ($pdf); + check_missing(\%map); + + return if (!$need && !$optional); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo zypper install --no-recommends $install\n"); +} + +sub give_mageia_hints() +{ + my %map = ( + "python-sphinx" => "python3-sphinx", + "yaml" => "python3-yaml", + "virtualenv" => "python3-virtualenv", + "dot" => "graphviz", + "convert" => "ImageMagick", + "Pod::Usage" => "perl-Pod-Usage", + "xelatex" => "texlive", + "rsvg-convert" => "librsvg2", + ); + + my @tex_pkgs = ( + "texlive-fontsextra", + ); + + $map{"latexmk"} = "texlive-collection-basic"; + + my $packager_cmd; + my $noto_sans; + if ($system_release =~ /OpenMandriva/) { + $packager_cmd = "dnf install"; + $noto_sans = "noto-sans-cjk-fonts"; + @tex_pkgs = ( "texlive-collection-fontsextra" ); + } else { + $packager_cmd = "urpmi"; + $noto_sans = "google-noto-sans-cjk-ttc-fonts"; + } + + + if ($pdf) { + check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/TTF/NotoSans-Regular.ttf"], + $noto_sans, 2); + } + + check_rpm_missing(\@tex_pkgs, 2) if ($pdf); + check_missing(\%map); + + return if (!$need && !$optional); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo $packager_cmd $install\n"); +} + +sub give_arch_linux_hints() +{ + my %map = ( + "yaml" => "python-yaml", + "virtualenv" => "python-virtualenv", + "dot" => "graphviz", + "convert" => "imagemagick", + "xelatex" => "texlive-xetex", + "latexmk" => "texlive-core", + "rsvg-convert" => "extra/librsvg", + ); + + my @archlinux_tex_pkgs = ( + "texlive-core", + "texlive-latexextra", + "ttf-dejavu", + ); + check_pacman_missing(\@archlinux_tex_pkgs, 2) if ($pdf); + + if ($pdf) { + check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], + "noto-fonts-cjk", 2); + } + + check_missing(\%map); + + return if (!$need && !$optional); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo pacman -S $install\n"); +} + +sub give_gentoo_hints() +{ + my %map = ( + "yaml" => "dev-python/pyyaml", + "virtualenv" => "dev-python/virtualenv", + "dot" => "media-gfx/graphviz", + "convert" => "media-gfx/imagemagick", + "xelatex" => "dev-texlive/texlive-xetex media-fonts/dejavu", + "rsvg-convert" => "gnome-base/librsvg", + ); + + check_missing_file(["/usr/share/fonts/dejavu/DejaVuSans.ttf"], + "media-fonts/dejavu", 2) if ($pdf); + + if ($pdf) { + check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", + "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc"], + "media-fonts/noto-cjk", 2); + } + + check_missing(\%map); + + return if (!$need && !$optional); + + printf("You should run:\n") if ($verbose_warn_install); + printf("\n"); + + my $imagemagick = "media-gfx/imagemagick svg png"; + my $cairo = "media-gfx/graphviz cairo pdf"; + my $portage_imagemagick = "/etc/portage/package.use/imagemagick"; + my $portage_cairo = "/etc/portage/package.use/graphviz"; + + if (qx(grep imagemagick $portage_imagemagick 2>/dev/null) eq "") { + printf("\tsudo su -c 'echo \"$imagemagick\" > $portage_imagemagick'\n") + } + if (qx(grep graphviz $portage_cairo 2>/dev/null) eq "") { + printf("\tsudo su -c 'echo \"$cairo\" > $portage_cairo'\n"); + } + + printf("\tsudo emerge --ask $install\n"); + +} + +sub check_distros() +{ + # Distro-specific hints + if ($system_release =~ /Red Hat Enterprise Linux/) { + give_redhat_hints; + return; + } + if ($system_release =~ /CentOS/) { + give_redhat_hints; + return; + } + if ($system_release =~ /Scientific Linux/) { + give_redhat_hints; + return; + } + if ($system_release =~ /Oracle Linux Server/) { + give_redhat_hints; + return; + } + if ($system_release =~ /Fedora/) { + give_redhat_hints; + return; + } + if ($system_release =~ /Ubuntu/) { + give_debian_hints; + return; + } + if ($system_release =~ /Debian/) { + give_debian_hints; + return; + } + if ($system_release =~ /openSUSE/) { + give_opensuse_hints; + return; + } + if ($system_release =~ /Mageia/) { + give_mageia_hints; + return; + } + if ($system_release =~ /OpenMandriva/) { + give_mageia_hints; + return; + } + if ($system_release =~ /Arch Linux/) { + give_arch_linux_hints; + return; + } + if ($system_release =~ /Gentoo/) { + give_gentoo_hints; + return; + } + + # + # Fall-back to generic hint code for other distros + # That's far from ideal, specially for LaTeX dependencies. + # + my %map = ( + "sphinx-build" => "sphinx" + ); + check_missing_tex(2) if ($pdf); + check_missing(\%map); + print "I don't know distro $system_release.\n"; + print "So, I can't provide you a hint with the install procedure.\n"; + print "There are likely missing dependencies.\n"; +} + +# +# Common dependencies +# + +sub deactivate_help() +{ + printf "\n If you want to exit the virtualenv, you can use:\n"; + printf "\tdeactivate\n"; +} + +sub get_virtenv() +{ + my $ver; + my $min_activate = "$ENV{'PWD'}/${virtenv_prefix}${min_version}/bin/activate"; + my @activates = glob "$ENV{'PWD'}/${virtenv_prefix}*/bin/activate"; + + @activates = sort {$b cmp $a} @activates; + + foreach my $f (@activates) { + next if ($f lt $min_activate); + + my $sphinx_cmd = $f; + $sphinx_cmd =~ s/activate/sphinx-build/; + next if (! -f $sphinx_cmd); + + my $ver = get_sphinx_version($sphinx_cmd); + + if (!$ver) { + $f =~ s#/bin/activate##; + print("Warning: virtual environment $f is not working.\nPython version upgrade? Remove it with:\n\n\trm -rf $f\n\n"); + } + + if ($need_sphinx && ($ver ge $min_version)) { + return ($f, $ver); + } elsif ($ver gt $cur_version) { + return ($f, $ver); + } + } + return ("", ""); +} + +sub recommend_sphinx_upgrade() +{ + my $venv_ver; + + # Avoid running sphinx-builds from venv if $cur_version is good + if ($cur_version && ($cur_version ge $rec_version)) { + $latest_avail_ver = $cur_version; + return; + } + + # Get the highest version from sphinx_*/bin/sphinx-build and the + # corresponding command to activate the venv/virtenv + ($activate_cmd, $venv_ver) = get_virtenv(); + + # Store the highest version from Sphinx existing virtualenvs + if (($activate_cmd ne "") && ($venv_ver gt $cur_version)) { + $latest_avail_ver = $venv_ver; + } else { + $latest_avail_ver = $cur_version if ($cur_version); + } + + # As we don't know package version of Sphinx, and there's no + # virtual environments, don't check if upgrades are needed + if (!$virtualenv) { + return if (!$latest_avail_ver); + } + + # Either there are already a virtual env or a new one should be created + $need_pip = 1; + + return if (!$latest_avail_ver); + + # Return if the reason is due to an upgrade or not + if ($latest_avail_ver lt $rec_version) { + $rec_sphinx_upgrade = 1; + } + + return $latest_avail_ver; +} + +# +# The logic here is complex, as it have to deal with different versions: +# - minimal supported version; +# - minimal PDF version; +# - recommended version. +# It also needs to work fine with both distro's package and venv/virtualenv +sub recommend_sphinx_version($) +{ + my $virtualenv_cmd = shift; + + # Version is OK. Nothing to do. + if ($cur_version && ($cur_version ge $rec_version)) { + return; + }; + + if (!$need_sphinx) { + # sphinx-build is present and its version is >= $min_version + + #only recommend enabling a newer virtenv version if makes sense. + if ($latest_avail_ver gt $cur_version) { + printf "\nYou may also use the newer Sphinx version $latest_avail_ver with:\n"; + printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); + printf "\t. $activate_cmd\n"; + deactivate_help(); + + return; + } + return if ($latest_avail_ver ge $rec_version); + } + + if (!$virtualenv) { + # No sphinx either via package or via virtenv. As we can't + # Compare the versions here, just return, recommending the + # user to install it from the package distro. + return if (!$latest_avail_ver); + + # User doesn't want a virtenv recommendation, but he already + # installed one via virtenv with a newer version. + # So, print commands to enable it + if ($latest_avail_ver gt $cur_version) { + printf "\nYou may also use the Sphinx virtualenv version $latest_avail_ver with:\n"; + printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); + printf "\t. $activate_cmd\n"; + deactivate_help(); + + return; + } + print "\n"; + } else { + $need++ if ($need_sphinx); + } + + # Suggest newer versions if current ones are too old + if ($latest_avail_ver && $latest_avail_ver ge $min_version) { + # If there's a good enough version, ask the user to enable it + if ($latest_avail_ver ge $rec_version) { + printf "\nNeed to activate Sphinx (version $latest_avail_ver) on virtualenv with:\n"; + printf "\t. $activate_cmd\n"; + deactivate_help(); + + return; + } + + # Version is above the minimal required one, but may be + # below the recommended one. So, print warnings/notes + + if ($latest_avail_ver lt $rec_version) { + print "Warning: It is recommended at least Sphinx version $rec_version.\n"; + } + } + + # At this point, either it needs Sphinx or upgrade is recommended, + # both via pip + + if ($rec_sphinx_upgrade) { + if (!$virtualenv) { + print "Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n"; + } else { + print "To upgrade Sphinx, use:\n\n"; + } + } else { + print "\nSphinx needs to be installed either:\n1) via pip/pypi with:\n\n"; + } + + $python_cmd = find_python_no_venv(); + + printf "\t$virtualenv_cmd $virtenv_dir\n"; + + printf "\t. $virtenv_dir/bin/activate\n"; + printf "\tpip install -r $requirement_file\n"; + deactivate_help(); + + printf "\n2) As a package with:\n"; + + my $old_need = $need; + my $old_optional = $optional; + %missing = (); + $pdf = 0; + $optional = 0; + $install = ""; + $verbose_warn_install = 0; + + add_package("python-sphinx", 0); + + check_distros(); + + $need = $old_need; + $optional = $old_optional; + + printf "\n Please note that Sphinx >= 3.0 will currently produce false-positive\n"; + printf " warning when the same name is used for more than one type (functions,\n"; + printf " structs, enums,...). This is known Sphinx bug. For more details, see:\n"; + printf "\thttps://github.com/sphinx-doc/sphinx/pull/8313\n"; +} + +sub check_needs() +{ + # Check if Sphinx is already accessible from current environment + check_sphinx(); + + if ($system_release) { + print "Detected OS: $system_release.\n"; + } else { + print "Unknown OS\n"; + } + printf "Sphinx version: %s\n\n", $cur_version if ($cur_version); + + # Check python command line, trying first python3 + $python_cmd = findprog("python3"); + $python_cmd = check_program("python", 0) if (!$python_cmd); + + # Check the type of virtual env, depending on Python version + if ($python_cmd) { + if ($virtualenv) { + my $tmp = qx($python_cmd --version 2>&1); + if ($tmp =~ m/(\d+\.)(\d+\.)/) { + if ($1 < 3) { + # Fail if it finds python2 (or worse) + die "Python 3 is required to build the kernel docs\n"; + } + if ($1 == 3 && $2 < 3) { + # Need Python 3.3 or upper for venv + $need_virtualenv = 1; + } + } else { + die "Warning: couldn't identify $python_cmd version!"; + } + } else { + add_package("python-sphinx", 0); + } + } + + my $venv_ver = recommend_sphinx_upgrade(); + + my $virtualenv_cmd; + + if ($need_pip) { + # Set virtualenv command line, if python < 3.3 + if ($need_virtualenv) { + $virtualenv_cmd = findprog("virtualenv-3"); + $virtualenv_cmd = findprog("virtualenv-3.5") if (!$virtualenv_cmd); + if (!$virtualenv_cmd) { + check_program("virtualenv", 0); + $virtualenv_cmd = "virtualenv"; + } + } else { + $virtualenv_cmd = "$python_cmd -m venv"; + check_python_module("ensurepip", 0); + } + } + + # Check for needed programs/tools + check_perl_module("Pod::Usage", 0); + check_python_module("yaml", 0); + check_program("make", 0); + check_program("gcc", 0); + check_program("dot", 1); + check_program("convert", 1); + + # Extra PDF files - should use 2 for is_optional + check_program("xelatex", 2) if ($pdf); + check_program("rsvg-convert", 2) if ($pdf); + check_program("latexmk", 2) if ($pdf); + + # Do distro-specific checks and output distro-install commands + check_distros(); + + if (!$python_cmd) { + if ($need == 1) { + die "Can't build as $need mandatory dependency is missing"; + } elsif ($need) { + die "Can't build as $need mandatory dependencies are missing"; + } + } + + # Check if sphinx-build is called sphinx-build-3 + if ($need_symlink) { + printf "\tsudo ln -sf %s /usr/bin/sphinx-build\n\n", + which("sphinx-build-3"); + } + + recommend_sphinx_version($virtualenv_cmd); + printf "\n"; + + print "All optional dependencies are met.\n" if (!$optional); + + if ($need == 1) { + die "Can't build as $need mandatory dependency is missing"; + } elsif ($need) { + die "Can't build as $need mandatory dependencies are missing"; + } + + print "Needed package dependencies are met.\n"; +} + +# +# Main +# + +while (@ARGV) { + my $arg = shift(@ARGV); + + if ($arg eq "--no-virtualenv") { + $virtualenv = 0; + } elsif ($arg eq "--no-pdf"){ + $pdf = 0; + } elsif ($arg eq "--version-check"){ + $version_check = 1; + } else { + print "Usage:\n\t$0 <--no-virtualenv> <--no-pdf> <--version-check>\n\n"; + print "Where:\n"; + print "\t--no-virtualenv\t- Recommend installing Sphinx instead of using a virtualenv\n"; + print "\t--version-check\t- if version is compatible, don't check for missing dependencies\n"; + print "\t--no-pdf\t- don't check for dependencies required to build PDF docs\n\n"; + exit -1; + } +} + +# +# Determine the system type. There's no standard unique way that would +# work with all distros with a minimal package install. So, several +# methods are used here. +# +# By default, it will use lsb_release function. If not available, it will +# fail back to reading the known different places where the distro name +# is stored +# + +$system_release = qx(lsb_release -d) if which("lsb_release"); +$system_release =~ s/Description:\s*// if ($system_release); +$system_release = catcheck("/etc/system-release") if !$system_release; +$system_release = catcheck("/etc/redhat-release") if !$system_release; +$system_release = catcheck("/etc/lsb-release") if !$system_release; +$system_release = catcheck("/etc/gentoo-release") if !$system_release; + +# This seems more common than LSB these days +if (!$system_release) { + my %os_var; + if (open IN, "cat /etc/os-release|") { + while () { + if (m/^([\w\d\_]+)=\"?([^\"]*)\"?\n/) { + $os_var{$1}=$2; + } + } + $system_release = $os_var{"NAME"}; + if (defined($os_var{"VERSION_ID"})) { + $system_release .= " " . $os_var{"VERSION_ID"} if (defined($os_var{"VERSION_ID"})); + } else { + $system_release .= " " . $os_var{"VERSION"}; + } + } +} +$system_release = catcheck("/etc/issue") if !$system_release; +$system_release =~ s/\s+$//; + +check_needs; -- cgit From ca9087f50772aa5794dd655ff5a0719dd4b83b1f Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:20 +0200 Subject: scripts: sphinx-pre-install: Convert script to Python Port scripts/sphinx-pre-install to Python. That allows a better maintainance of the file. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/8d236079545aff796900bde4cdaaf79411d04074.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 1102 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1102 insertions(+) create mode 100755 scripts/sphinx-pre-install.py (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py new file mode 100755 index 000000000000..dcee2181b72f --- /dev/null +++ b/scripts/sphinx-pre-install.py @@ -0,0 +1,1102 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (c) 2017-2025 Mauro Carvalho Chehab +# +# pylint: disable=C0103,C0114,C0115,C0116,C0301 +# pylint: disable=R0902,R0904,R0912,R0915,R1705,R1710,E1121 + + +import argparse +import os +import re +import subprocess +import sys +from glob import glob + + +def parse_version(version): + """Convert a major.minor.patch version into a tuple""" +# + return tuple(int(x) for x in version.split(".")) + + +def ver_str(version): + """Returns a version tuple as major.minor.patch""" + + return ".".join([str(x) for x in version]) + + +RECOMMENDED_VERSION = parse_version("3.4.3") + + +class SphinxDependencyChecker: + # List of required texlive packages on Fedora and OpenSuse + texlive = { + "amsfonts.sty": "texlive-amsfonts", + "amsmath.sty": "texlive-amsmath", + "amssymb.sty": "texlive-amsfonts", + "amsthm.sty": "texlive-amscls", + "anyfontsize.sty": "texlive-anyfontsize", + "atbegshi.sty": "texlive-oberdiek", + "bm.sty": "texlive-tools", + "capt-of.sty": "texlive-capt-of", + "cmap.sty": "texlive-cmap", + "ctexhook.sty": "texlive-ctex", + "ecrm1000.tfm": "texlive-ec", + "eqparbox.sty": "texlive-eqparbox", + "eu1enc.def": "texlive-euenc", + "fancybox.sty": "texlive-fancybox", + "fancyvrb.sty": "texlive-fancyvrb", + "float.sty": "texlive-float", + "fncychap.sty": "texlive-fncychap", + "footnote.sty": "texlive-mdwtools", + "framed.sty": "texlive-framed", + "luatex85.sty": "texlive-luatex85", + "multirow.sty": "texlive-multirow", + "needspace.sty": "texlive-needspace", + "palatino.sty": "texlive-psnfss", + "parskip.sty": "texlive-parskip", + "polyglossia.sty": "texlive-polyglossia", + "tabulary.sty": "texlive-tabulary", + "threeparttable.sty": "texlive-threeparttable", + "titlesec.sty": "texlive-titlesec", + "ucs.sty": "texlive-ucs", + "upquote.sty": "texlive-upquote", + "wrapfig.sty": "texlive-wrapfig", + } + + def __init__(self, args): + self.pdf = args.pdf + self.virtualenv = args.virtualenv + self.version_check = args.version_check + + self.missing = {} + + self.need = 0 + self.optional = 0 + self.need_symlink = 0 + self.need_sphinx = 0 + self.need_pip = 0 + self.need_virtualenv = 0 + self.rec_sphinx_upgrade = 0 + self.verbose_warn_install = 1 + + self.system_release = "" + self.install = "" + self.virtenv_dir = "" + self.python_cmd = "" + self.activate_cmd = "" + + self.min_version = (0, 0, 0) + self.cur_version = (0, 0, 0) + self.latest_avail_ver = (0, 0, 0) + self.venv_ver = (0, 0, 0) + + prefix = os.environ.get("srctree", ".") + "/" + + self.conf = prefix + "Documentation/conf.py" + self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" + self.virtenv_prefix = ["sphinx_", "Sphinx_" ] + + # + # Ancillary methods that don't depend on self + # + + @staticmethod + def which(prog): + for path in os.environ.get("PATH", "").split(":"): + full_path = os.path.join(path, prog) + if os.access(full_path, os.X_OK): + return full_path + + return None + + @staticmethod + def find_python_no_venv(): + # FIXME: does it makes sense now that this script is in Python? + + result = subprocess.run(["pwd"], capture_output=True, text=True) + cur_dir = result.stdout.strip() + + python_names = ["python3", "python"] + + for d in os.environ.get("PATH", "").split(":"): + if f"{cur_dir}/sphinx" in d: + continue + + for p in python_names: + if os.access(os.path.join(d, p), os.X_OK): + return os.path.join(d, p) + + # Python not found at the PATH + return python_names[-1] + + @staticmethod + def run(*args, **kwargs): + """Excecute a command, hiding its output by default""" + + if not kwargs.get('capture_output', False): + if 'stdout' not in kwargs: + kwargs['stdout'] = subprocess.DEVNULL + if 'stderr' not in kwargs: + kwargs['stderr'] = subprocess.DEVNULL + + return subprocess.run(*args, **kwargs) + + # + # Methods to check if a feature exists + # + + # Note: is_optional has 3 states: + # - 0: mandatory + # - 1: optional, but nice to have + # - 2: LaTeX optional - pdf builds without it, but may have visual impact + + def check_missing(self, progs): + for prog, is_optional in sorted(self.missing.items()): + # At least on some LTS distros like CentOS 7, texlive doesn't + # provide all packages we need. When such distros are + # detected, we have to disable PDF output. + # + # So, we need to ignore the packages that distros would + # need for LaTeX to work + if is_optional == 2 and not self.pdf: + self.optional -= 1 + continue + + if self.verbose_warn_install: + if is_optional: + print(f'Warning: better to also install "{prog}".') + else: + print(f'ERROR: please install "{prog}", otherwise, build won\'t work.') + + self.install += " " + progs.get(prog, prog) + + self.install = self.install.lstrip() + + def add_package(self, package, is_optional): + self.missing[package] = is_optional + if is_optional: + self.optional += 1 + else: + self.need += 1 + + def check_missing_file(self, files, package, is_optional): + for f in files: + if os.path.exists(f): + return + self.add_package(package, is_optional) + + def check_program(self, prog, is_optional): + found = self.which(prog) + if found: + return found + + self.add_package(prog, is_optional) + + return None + + def check_perl_module(self, prog, is_optional): + # While testing with lxc download template, one of the + # distros (Oracle) didn't have perl - nor even an option to install + # before installing oraclelinux-release-el9 package. + # + # Check it before running an error. If perl is not there, + # add it as a mandatory package, as some parts of the doc builder + # needs it. + if not self.which("perl"): + self.add_package("perl", 0) + self.add_package(prog, is_optional) + return + + try: + self.run(["perl", f"-M{prog}", "-e", "1"], check=True) + except subprocess.CalledProcessError: + self.add_package(prog, is_optional) + + def check_python_module(self, module, is_optional): + # FIXME: is it needed at the Python version? Maybe due to venv? + if not self.python_cmd: + return + + try: + self.run([self.python_cmd, "-c", f"import {module}"], check=True) + except subprocess.CalledProcessError: + self.add_package(module, is_optional) + + def check_rpm_missing(self, pkgs, is_optional): + for prog in pkgs: + try: + self.run(["rpm", "-q", prog], check=True) + except subprocess.CalledProcessError: + self.add_package(prog, is_optional) + + def check_pacman_missing(self, pkgs, is_optional): + for prog in pkgs: + try: + self.run(["pacman", "-Q", prog], check=True) + except subprocess.CalledProcessError: + self.add_package(prog, is_optional) + + def check_missing_tex(self, is_optional): + kpsewhich = self.which("kpsewhich") + for prog, package in self.texlive.items(): + + # If kpsewhich is not there, just add it to deps + if not kpsewhich: + self.add_package(package, is_optional) + continue + + # Check if the package is needed + try: + result = self.run( + [kpsewhich, prog], stdout=subprocess.PIPE, text=True, check=True + ) + + # Didn't find. Add it + if not result.stdout.strip(): + self.add_package(package, is_optional) + + except subprocess.CalledProcessError: + # kpsewhich returned an error. Add it, just in case + self.add_package(package, is_optional) + + def get_sphinx_fname(self): + if "SPHINXBUILD" in os.environ: + return os.environ["SPHINXBUILD"] + + fname = "sphinx-build" + if self.which(fname): + return fname + + fname = "sphinx-build-3" + if self.which(fname): + self.need_symlink = 1 + return fname + + return "" + + def get_sphinx_version(self, cmd): + try: + result = self.run([cmd, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, check=True) + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + for line in result.stdout.split("\n"): + match = re.match(r"^sphinx-build\s+([\d\.]+)(?:\+(?:/[\da-f]+)|b\d+)?\s*$", line) + if match: + return parse_version(match.group(1)) + + match = re.match(r"^Sphinx.*\s+([\d\.]+)\s*$", line) + if match: + return parse_version(match.group(1)) + + def check_sphinx(self): + try: + with open(self.conf, "r", encoding="utf-8") as f: + for line in f: + match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line) + if match: + self.min_version = parse_version(match.group(1)) + break + except IOError: + sys.exit(f"Can't open {self.conf}") + + if not self.min_version: + sys.exit(f"Can't get needs_sphinx version from {self.conf}") + + self.virtenv_dir = self.virtenv_prefix[0] + "latest" + + sphinx = self.get_sphinx_fname() + if not sphinx: + self.need_sphinx = 1 + return + + self.cur_version = self.get_sphinx_version(sphinx) + if not self.cur_version: + sys.exit(f"{sphinx} didn't return its version") + + if self.cur_version < self.min_version: + curver = ver_str(self.cur_version) + minver = ver_str(self.min_version) + + print(f"ERROR: Sphinx version is {curver}. It should be >= {minver}") + self.need_sphinx = 1 + return + + # On version check mode, just assume Sphinx has all mandatory deps + if self.version_check and self.cur_version >= RECOMMENDED_VERSION: + sys.exit(0) + + def catcheck(self, filename): + if os.path.exists(filename): + with open(filename, "r", encoding="utf-8") as f: + return f.read().strip() + return "" + + # + # Distro-specific hints methods + # + + def give_debian_hints(self): + progs = { + "Pod::Usage": "perl-modules", + "convert": "imagemagick", + "dot": "graphviz", + "ensurepip": "python3-venv", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2-bin", + "virtualenv": "virtualenv", + "xelatex": "texlive-xetex", + "yaml": "python3-yaml", + } + + if self.pdf: + pdf_pkgs = { + "texlive-lang-chinese": [ + "/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty", + ], + "fonts-dejavu": [ + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + ], + "fonts-noto-cjk": [ + "/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc", + ], + } + + for package, files in pdf_pkgs.items(): + self.check_missing_file(files, package, 2) + + self.check_program("dvipng", 2) + + self.check_missing(progs) + + if not self.need and not self.optional: + return + + if self.verbose_warn_install: + print("You should run:") + print(f"\n\tsudo apt-get install {self.install}") + + def give_redhat_hints(self): + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2-tools", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive-xetex-bin", + "yaml": "python3-pyyaml", + } + + fedora26_opt_pkgs = [ + "graphviz-gd", # Fedora 26: needed for PDF support + ] + + fedora_tex_pkgs = [ + "dejavu-sans-fonts", + "dejavu-sans-mono-fonts", + "dejavu-serif-fonts", + "texlive-collection-fontsrecommended", + "texlive-collection-latex", + "texlive-xecjk", + ] + + old = 0 + rel = None + pkg_manager = "dnf" + + match = re.search(r"(release|Linux)\s+(\d+)", self.system_release) + if match: + rel = int(match.group(2)) + + if not rel: + print("Couldn't identify release number") + old = 1 + self.pdf = False + elif re.search("Fedora", self.system_release): + # Fedora 38 and upper use this CJK font + + noto_sans_redhat = "google-noto-sans-cjk-fonts" + else: + # Almalinux, CentOS, RHEL, ... + + # at least up to version 9 (and Fedora < 38), that's the CJK font + noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts" + + progs["virtualenv"] = "python-virtualenv" + + if rel and rel < 8: + old = 1 + self.pdf = False + + # RHEL 7 is in ELS, currently up to Jun, 2026 + + print("Note: texlive packages on RHEL/CENTOS <= 7 are incomplete. Can't support PDF output") + print("If you want to build PDF, please read:") + print("\thttps://www.systutorials.com/241660/how-to-install-tex-live-on-centos-7-linux/") + + if self.pdf: + pdf_pkgs = [ + "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc", + ] + + self.check_missing_file(pdf_pkgs, noto_sans_redhat, 2) + + if not old: + self.check_rpm_missing(fedora26_opt_pkgs, 2) + self.check_rpm_missing(fedora_tex_pkgs, 2) + + self.check_missing_tex(2) + + self.check_missing(progs) + + if not self.need and not self.optional: + return + + if self.verbose_warn_install: + print("You should run:") + + if old: + # dnf is there since Fedora 18+ and RHEL 8 + pkg_manager = "yum" + + print(f"\n\tsudo {pkg_manager} install -y {self.install}") + + def give_opensuse_hints(self): + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive-xetex-bin", + "yaml": "python3-pyyaml", + } + + # On Tumbleweed, this package is also named rsvg-convert + if not re.search(r"Tumbleweed", self.system_release): + progs["rsvg-convert"] = "rsvg-view" + + suse_tex_pkgs = [ + "texlive-babel-english", + "texlive-caption", + "texlive-colortbl", + "texlive-courier", + "texlive-dvips", + "texlive-helvetic", + "texlive-makeindex", + "texlive-metafont", + "texlive-metapost", + "texlive-palatino", + "texlive-preview", + "texlive-times", + "texlive-zapfchan", + "texlive-zapfding", + ] + + progs["latexmk"] = "texlive-latexmk-bin" + + # FIXME: add support for installing CJK fonts + # + # I tried hard, but was unable to find a way to install + # "Noto Sans CJK SC" on openSUSE + + if self.pdf: + self.check_rpm_missing(suse_tex_pkgs, 2) + if self.pdf: + self.check_missing_tex(2) + self.check_missing(progs) + + if not self.need and not self.optional: + return + + if self.verbose_warn_install: + print("You should run:") + print(f"\n\tsudo zypper install --no-recommends {self.install}") + + def give_mageia_hints(self): + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive", + "yaml": "python3-yaml", + } + + tex_pkgs = [ + "texlive-fontsextra", + ] + + if re.search(r"OpenMandriva", self.system_release): + packager_cmd = "dnf install" + noto_sans = "noto-sans-cjk-fonts" + tex_pkgs = ["texlive-collection-fontsextra"] + else: + packager_cmd = "urpmi" + noto_sans = "google-noto-sans-cjk-ttc-fonts" + + progs["latexmk"] = "texlive-collection-basic" + + if self.pdf: + pdf_pkgs = [ + "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/TTF/NotoSans-Regular.ttf", + ] + + self.check_missing_file(pdf_pkgs, noto_sans, 2) + self.check_rpm_missing(tex_pkgs, 2) + + self.check_missing(progs) + + if not self.need and not self.optional: + return + if self.verbose_warn_install: + print("You should run:") + print(f"\n\tsudo {packager_cmd} {self.install}") + + def give_arch_linux_hints(self): + progs = { + "convert": "imagemagick", + "dot": "graphviz", + "latexmk": "texlive-core", + "rsvg-convert": "extra/librsvg", + "virtualenv": "python-virtualenv", + "xelatex": "texlive-xetex", + "yaml": "python-yaml", + } + + archlinux_tex_pkgs = [ + "texlive-core", + "texlive-latexextra", + "ttf-dejavu", + ] + + if self.pdf: + self.check_pacman_missing(archlinux_tex_pkgs, 2) + + self.check_missing_file( + ["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], + "noto-fonts-cjk", + 2, + ) + + self.check_missing(progs) + + if not self.need and not self.optional: + return + if self.verbose_warn_install: + print("You should run:") + print(f"\n\tsudo pacman -S {self.install}") + + def give_gentoo_hints(self): + progs = { + "convert": "media-gfx/imagemagick", + "dot": "media-gfx/graphviz", + "rsvg-convert": "gnome-base/librsvg", + "virtualenv": "dev-python/virtualenv", + "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", + "yaml": "dev-python/pyyaml", + } + + if self.pdf: + pdf_pkgs = { + "media-fonts/dejavu": [ + "/usr/share/fonts/dejavu/DejaVuSans.ttf", + ], + "media-fonts/noto-cjk": [ + "/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", + "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc", + ], + } + for package, files in pdf_pkgs.items(): + self.check_missing_file(files, package, 2) + + self.check_missing(progs) + + if not self.need and not self.optional: + return + + if self.verbose_warn_install: + print("You should run:") + print("\n") + + imagemagick = "media-gfx/imagemagick svg png" + cairo = "media-gfx/graphviz cairo pdf" + portage_imagemagick = "/etc/portage/package.use/imagemagick" + portage_cairo = "/etc/portage/package.use/graphviz" + + result = self.run(["grep", "imagemagick", portage_imagemagick], + stdout=subprocess.PIPE, text=True) + if not result.stdout.strip(): + print(f"\tsudo su -c 'echo \"{imagemagick}\" > {portage_imagemagick}'") + + result = self.run(["grep", "graphviz", portage_cairo], + stdout=subprocess.PIPE, text=True) + + if not result.stdout.strip(): + print(f"\tsudo su -c 'echo \"{cairo}\" > {portage_cairo}'") + + print(f"\tsudo emerge --ask {self.install}") + + # + # Dispatch the check to an os_specific hinter + # + + def check_distros(self): + # OS-specific hints logic + os_hints = { + re.compile("Red Hat Enterprise Linux"): self.give_redhat_hints, + re.compile("Fedora"): self.give_redhat_hints, + re.compile("AlmaLinux"): self.give_redhat_hints, + re.compile("Amazon Linux"): self.give_redhat_hints, + re.compile("CentOS"): self.give_redhat_hints, + re.compile("openEuler"): self.give_redhat_hints, + re.compile("Oracle Linux Server"): self.give_redhat_hints, + re.compile("Rocky Linux"): self.give_redhat_hints, + re.compile("Scientific Linux"): self.give_redhat_hints, + re.compile("Springdale Open Enterprise"): self.give_redhat_hints, + + re.compile("Ubuntu"): self.give_debian_hints, + re.compile("Debian"): self.give_debian_hints, + re.compile("Devuan"): self.give_debian_hints, + re.compile("Kali"): self.give_debian_hints, + re.compile("Mint"): self.give_debian_hints, + + re.compile("openSUSE"): self.give_opensuse_hints, + + re.compile("Mageia"): self.give_mageia_hints, + re.compile("OpenMandriva"): self.give_mageia_hints, + + re.compile("Arch Linux"): self.give_arch_linux_hints, + re.compile("Gentoo"): self.give_gentoo_hints, + } + + # If the OS is detected, use per-OS hint logic + for regex, os_hint in os_hints.items(): + if regex.search(self.system_release): + os_hint() + + return + + # + # Fall-back to generic hint code for other distros + # That's far from ideal, specially for LaTeX dependencies. + # + progs = {"sphinx-build": "sphinx"} + if self.pdf: + self.check_missing_tex(2) + + self.check_missing(progs) + + print(f"I don't know distro {self.system_release}.") + print("So, I can't provide you a hint with the install procedure.") + print("There are likely missing dependencies.") + + # + # Common dependencies + # + def deactivate_help(self): + print("\n If you want to exit the virtualenv, you can use:") + print("\tdeactivate") + + def get_virtenv(self): + cwd = os.getcwd() + + activates = [] + + # Add all sphinx prefixes with possible version numbers + for p in self.virtenv_prefix: + activates += glob(f"{cwd}/{p}[0-9]*/bin/activate") + + activates.sort(reverse=True, key=str.lower) + + # Place sphinx_latest first, if it exists + for p in self.virtenv_prefix: + activates = glob(f"{cwd}/{p}*latest/bin/activate") + activates + + ver = (0, 0, 0) + for f in activates: + # Discard too old Sphinx virtual environments + match = re.search(r"(\d+)\.(\d+)\.(\d+)", f) + if match: + ver = (int(match.group(1)), int(match.group(2)), int(match.group(3))) + + if ver < self.min_version: + continue + + sphinx_cmd = f.replace("activate", "sphinx-build") + if not os.path.isfile(sphinx_cmd): + continue + + ver = self.get_sphinx_version(sphinx_cmd) + + if not ver: + venv_dir = f.replace("/bin/activate", "") + print(f"Warning: virtual environment {venv_dir} is not working.\n" \ + "Python version upgrade? Remove it with:\n\n" \ + "\trm -rf {venv_dir}\n\n") + else: + if self.need_sphinx and ver >= self.min_version: + return (f, ver) + elif parse_version(ver) > self.cur_version: + return (f, ver) + + return ("", ver) + + def recommend_sphinx_upgrade(self): + # Avoid running sphinx-builds from venv if cur_version is good + if self.cur_version and self.cur_version >= RECOMMENDED_VERSION: + self.latest_avail_ver = self.cur_version + return None + + # Get the highest version from sphinx_*/bin/sphinx-build and the + # corresponding command to activate the venv/virtenv + self.activate_cmd, self.venv_ver = self.get_virtenv() + + # Store the highest version from Sphinx existing virtualenvs + if self.activate_cmd and self.venv_ver > self.cur_version: + self.latest_avail_ver = self.venv_ver + else: + if self.cur_version: + self.latest_avail_ver = self.cur_version + else: + self.latest_avail_ver = (0, 0, 0) + + # As we don't know package version of Sphinx, and there's no + # virtual environments, don't check if upgrades are needed + if not self.virtualenv: + if not self.latest_avail_ver: + return None + + return self.latest_avail_ver + + # Either there are already a virtual env or a new one should be created + self.need_pip = 1 + + if not self.latest_avail_ver: + return None + + # Return if the reason is due to an upgrade or not + if self.latest_avail_ver != (0, 0, 0): + if self.latest_avail_ver < RECOMMENDED_VERSION: + self.rec_sphinx_upgrade = 1 + + return self.latest_avail_ver + + def recommend_sphinx_version(self, virtualenv_cmd): + # The logic here is complex, as it have to deal with different versions: + # - minimal supported version; + # - minimal PDF version; + # - recommended version. + # It also needs to work fine with both distro's package and venv/virtualenv + + # Version is OK. Nothing to do. + if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: + return + + if not self.need_sphinx: + # sphinx-build is present and its version is >= $min_version + + # only recommend enabling a newer virtenv version if makes sense. + if self.latest_avail_ver and self.latest_avail_ver > self.cur_version: + print("\nYou may also use the newer Sphinx version {self.latest_avail_ver} with:") + if f"{self.virtenv_prefix}" in os.getcwd(): + print("\tdeactivate") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + + if self.latest_avail_ver and self.latest_avail_ver >= RECOMMENDED_VERSION: + return + + if not self.virtualenv: + # No sphinx either via package or via virtenv. As we can't + # Compare the versions here, just return, recommending the + # user to install it from the package distro. + if not self.latest_avail_ver or self.latest_avail_ver == (0, 0, 0): + return + + # User doesn't want a virtenv recommendation, but he already + # installed one via virtenv with a newer version. + # So, print commands to enable it + if self.latest_avail_ver > self.cur_version: + print("\nYou may also use the Sphinx virtualenv version {self.latest_avail_ver} with:") + if f"{self.virtenv_prefix}" in os.getcwd(): + print("\tdeactivate") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + print("\n") + else: + if self.need_sphinx: + self.need += 1 + + # Suggest newer versions if current ones are too old + if self.latest_avail_ver and self.latest_avail_ver >= self.min_version: + if self.latest_avail_ver >= RECOMMENDED_VERSION: + print("\nNeed to activate Sphinx (version {self.latest_avail_ver}) on virtualenv with:") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + + # Version is above the minimal required one, but may be + # below the recommended one. So, print warnings/notes + if self.latest_avail_ver < RECOMMENDED_VERSION: + print(f"Warning: It is recommended at least Sphinx version {RECOMMENDED_VERSION}.") + + # At this point, either it needs Sphinx or upgrade is recommended, + # both via pip + + if self.rec_sphinx_upgrade: + if not self.virtualenv: + print("Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n") + else: + print("To upgrade Sphinx, use:\n\n") + else: + print("\nSphinx needs to be installed either:\n1) via pip/pypi with:\n") + + self.python_cmd = self.find_python_no_venv() + + print(f"\t{virtualenv_cmd} {self.virtenv_dir}") + print(f"\t. {self.virtenv_dir}/bin/activate") + print(f"\tpip install -r {self.requirement_file}") + self.deactivate_help() + + print("\n2) As a package with:") + + old_need = self.need + old_optional = self.optional + self.missing = {} + self.pdf = False + self.optional = 0 + self.install = "" + old_verbose = self.verbose_warn_install + self.verbose_warn_install = 0 + + self.add_package("python-sphinx", 0) + + self.check_distros() + + self.need = old_need + self.optional = old_optional + self.verbose_warn_install = old_verbose + + print("\n" \ + " Please note that Sphinx >= 3.0 will currently produce false-positive\n" \ + " warning when the same name is used for more than one type (functions,\n" \ + " structs, enums,...). This is known Sphinx bug. For more details, see:\n" \ + "\thttps://github.com/sphinx-doc/sphinx/pull/8313") + + def check_needs(self): + self.get_system_release() + + # Check if Sphinx is already accessible from current environment + self.check_sphinx() + + if self.system_release: + print(f"Detected OS: {self.system_release}.") + else: + print("Unknown OS") + if self.cur_version != (0, 0, 0): + ver = ver_str(self.cur_version) + print(f"Sphinx version: {ver}\n") + + # FIXME: Check python command line, trying first python3 + self.python_cmd = self.which("python3") + if not self.python_cmd: + self.python_cmd = self.check_program("python", 0) + + # Check the type of virtual env, depending on Python version + if self.python_cmd: + if self.virtualenv: + try: + result = self.run( + [self.python_cmd, "--version"], + capture_output=True, + text=True, + check=True, + ) + + output = result.stdout + result.stderr + + match = re.search(r"(\d+)\.(\d+)\.", output) + if match: + major = int(match.group(1)) + minor = int(match.group(2)) + + if major < 3: + sys.exit("Python 3 is required to build the kernel docs") + if major == 3 and minor < 3: + self.need_virtualenv = True + else: + sys.exit(f"Warning: couldn't identify {self.python_cmd} version!") + + except subprocess.CalledProcessError as e: + sys.exit(f"Error checking Python version: {e}") + else: + self.add_package("python-sphinx", 0) + + self.venv_ver = self.recommend_sphinx_upgrade() + + virtualenv_cmd = "" + + if self.need_pip: + # Set virtualenv command line, if python < 3.3 + # FIXME: can be removed as we're now with an upper min requirement + # but then we need to check python version + if self.need_virtualenv: + virtualenv_cmd = self.which("virtualenv-3") + if not virtualenv_cmd: + virtualenv_cmd = self.which("virtualenv-3.5") + if not virtualenv_cmd: + self.check_program("virtualenv", 0) + virtualenv_cmd = "virtualenv" + else: + virtualenv_cmd = f"{self.python_cmd} -m venv" + self.check_python_module("ensurepip", 0) + + # Check for needed programs/tools + self.check_perl_module("Pod::Usage", 0) + self.check_python_module("yaml", 0) + self.check_program("make", 0) + self.check_program("gcc", 0) + self.check_program("dot", 1) + self.check_program("convert", 1) + + if self.pdf: + # Extra PDF files - should use 2 for LaTeX is_optional + self.check_program("xelatex", 2) + self.check_program("rsvg-convert", 2) + self.check_program("latexmk", 2) + + # Do distro-specific checks and output distro-install commands + self.check_distros() + + if not self.python_cmd: + if self.need == 1: + sys.exit("Can't build as 1 mandatory dependency is missing") + elif self.need: + sys.exit(f"Can't build as {self.need} mandatory dependencies are missing") + + # Check if sphinx-build is called sphinx-build-3 + if self.need_symlink: + sphinx_path = self.which("sphinx-build-3") + if sphinx_path: + print(f"\tsudo ln -sf {sphinx_path} /usr/bin/sphinx-build\n") + + self.recommend_sphinx_version(virtualenv_cmd) + print("") + + if not self.optional: + print("All optional dependencies are met.") + + if self.need == 1: + sys.exit("Can't build as 1 mandatory dependency is missing") + elif self.need: + sys.exit(f"Can't build as {self.need} mandatory dependencies are missing") + + print("Needed package dependencies are met.") + + def get_system_release(self): + """ + Determine the system type. There's no unique way that would work + with all distros with a minimal package install. So, several + methods are used here. + + By default, it will use lsb_release function. If not available, it will + fail back to reading the known different places where the distro name + is stored. + + Several modern distros now have /etc/os-release, which usually have + a decent coverage. + """ + + if self.which("lsb_release"): + result = self.run(["lsb_release", "-d"], capture_output=True, text=True) + self.system_release = result.stdout.replace("Description:", "").strip() + + release_files = [ + "/etc/system-release", + "/etc/redhat-release", + "/etc/lsb-release", + "/etc/gentoo-release", + ] + + if not self.system_release: + for f in release_files: + self.system_release = self.catcheck(f) + if self.system_release: + break + + # This seems more common than LSB these days + if not self.system_release: + os_var = {} + try: + with open("/etc/os-release", "r", encoding="utf-8") as f: + for line in f: + match = re.match(r"^([\w\d\_]+)=\"?([^\"]*)\"?\n", line) + if match: + os_var[match.group(1)] = match.group(2) + + self.system_release = os_var.get("NAME", "") + if "VERSION_ID" in os_var: + self.system_release += " " + os_var["VERSION_ID"] + elif "VERSION" in os_var: + self.system_release += " " + os_var["VERSION"] + except IOError: + pass + + if not self.system_release: + self.system_release = self.catcheck("/etc/issue") + + self.system_release = self.system_release.strip() + +DESCRIPTION = """ +Process some flags related to Sphinx installation and documentation build. +""" + + +def main(): + parser = argparse.ArgumentParser(description=DESCRIPTION) + + parser.add_argument( + "--no-virtualenv", + action="store_false", + dest="virtualenv", + help="Recommend installing Sphinx instead of using a virtualenv", + ) + + parser.add_argument( + "--no-pdf", + action="store_false", + dest="pdf", + help="Don't check for dependencies required to build PDF docs", + ) + + parser.add_argument( + "--version-check", + action="store_true", + dest="version_check", + help="If version is compatible, don't check for missing dependencies", + ) + + args = parser.parse_args() + + checker = SphinxDependencyChecker(args) + + checker.check_needs() + + +if __name__ == "__main__": + main() -- cgit From 56a8767751c4bd2b93ba8840a78eccae7018d57c Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:21 +0200 Subject: scripts: sphinx-pre-install: Make it compatible with Python 3.6 The minimal version requirements we have is 3.9. Yet, the script which detects it is this one. So, let's try supporting an old version here, as we may want to suggest to upgrade Python version to build the docs. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/39d6e27a047bc3cc8208ac5e11fe6ba44faff9c4.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index dcee2181b72f..71d86b230b22 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -115,7 +115,8 @@ class SphinxDependencyChecker: def find_python_no_venv(): # FIXME: does it makes sense now that this script is in Python? - result = subprocess.run(["pwd"], capture_output=True, text=True) + result = SphinxDependencyChecker.run(["pwd"], capture_output=True, + text=True) cur_dir = result.stdout.strip() python_names = ["python3", "python"] @@ -135,12 +136,23 @@ class SphinxDependencyChecker: def run(*args, **kwargs): """Excecute a command, hiding its output by default""" - if not kwargs.get('capture_output', False): + capture_output = kwargs.pop('capture_output', False) + + if capture_output: + if 'stdout' not in kwargs: + kwargs['stdout'] = subprocess.PIPE + if 'stderr' not in kwargs: + kwargs['stderr'] = subprocess.PIPE + else: if 'stdout' not in kwargs: kwargs['stdout'] = subprocess.DEVNULL if 'stderr' not in kwargs: kwargs['stderr'] = subprocess.DEVNULL + # Don't break with older Python versions + if 'text' in kwargs and sys.version_info < (3, 7): + kwargs['universal_newlines'] = kwargs.pop('text') + return subprocess.run(*args, **kwargs) # -- cgit From 728648b6a5faa1d6be1e90c04a71481ba33fd08d Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:22 +0200 Subject: scripts: sphinx-pre-install: run on a supported version The scripts/sphinx-pre-install is used to detect problems at the system environment and adjust it to build the Kernel documentation. If the version is too old, it won't run, though. Check if the version which started the script is valid. If not, seek for a new one that is compatible with documentation build. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/76627055a7f82f6a79296ddbd873fa5ac8f82a1d.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 65 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 71d86b230b22..3912359d2bae 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -5,6 +5,9 @@ # pylint: disable=C0103,C0114,C0115,C0116,C0301 # pylint: disable=R0902,R0904,R0912,R0915,R1705,R1710,E1121 +# Note: this script requires at least Python 3.6 to run. +# Don't add changes not compatible with it, it is meant to report +# incompatible python versions. import argparse import os @@ -16,7 +19,6 @@ from glob import glob def parse_version(version): """Convert a major.minor.patch version into a tuple""" -# return tuple(int(x) for x in version.split(".")) @@ -27,6 +29,7 @@ def ver_str(version): RECOMMENDED_VERSION = parse_version("3.4.3") +MIN_PYTHON_VERSION = parse_version("3.7") class SphinxDependencyChecker: @@ -132,6 +135,65 @@ class SphinxDependencyChecker: # Python not found at the PATH return python_names[-1] + @staticmethod + def get_python_version(cmd): + + result = SphinxDependencyChecker.run([cmd, "--version"], + capture_output=True, text=True) + version = result.stdout.strip() + + match = re.search(r"(\d+\.\d+\.\d+)", version) + if match: + return parse_version(match.group(1)) + + print(f"Can't parse version {version}") + return (0, 0, 0) + + @staticmethod + def find_python(): + + patterns = [ + "python3.[0-9]", + "python3.[0-9][0-9]", + ] + + new_python_cmd = None + + # Seek for a python binary newer than MIN_PYTHON_VERSION + for path in os.getenv("PATH", "").split(":"): + for pattern in patterns: + for cmd in glob(os.path.join(path, pattern)): + if os.path.isfile(cmd) and os.access(cmd, os.X_OK): + version = SphinxDependencyChecker.get_python_version(cmd) + if version >= MIN_PYTHON_VERSION: + return(cmd) + + @staticmethod + def check_python(): + + cur_ver = sys.version_info[:3] + if cur_ver >= MIN_PYTHON_VERSION: + return + + python_ver = ver_str(cur_ver) + + new_python_cmd = SphinxDependencyChecker.find_python() + if not new_python_cmd: + print(f"ERROR: Python version {python_ver} is not spported anymore") + print(f" Can't find a new version. This script may fail") + return + + # Restart script using the newer version + script_path = os.path.abspath(sys.argv[0]) + args = [new_python_cmd, script_path] + sys.argv[1:] + + print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") + + try: + os.execv(new_python_cmd, args) + except OSError as e: + sys.exit(f"Failed to restart with {new_python_cmd}: {e}") + @staticmethod def run(*args, **kwargs): """Excecute a command, hiding its output by default""" @@ -1107,6 +1169,7 @@ def main(): checker = SphinxDependencyChecker(args) + checker.check_python() checker.check_needs() -- cgit From ea5dd67722c1ac8ae65ae5a9390452931890f52e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:23 +0200 Subject: scripts: sphinx-pre-install: drop obsolete routines Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/afb97854f05b0852f4c6c1fcee102e9697c14cd8.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 3912359d2bae..b639acd455cc 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -114,27 +114,6 @@ class SphinxDependencyChecker: return None - @staticmethod - def find_python_no_venv(): - # FIXME: does it makes sense now that this script is in Python? - - result = SphinxDependencyChecker.run(["pwd"], capture_output=True, - text=True) - cur_dir = result.stdout.strip() - - python_names = ["python3", "python"] - - for d in os.environ.get("PATH", "").split(":"): - if f"{cur_dir}/sphinx" in d: - continue - - for p in python_names: - if os.access(os.path.join(d, p), os.X_OK): - return os.path.join(d, p) - - # Python not found at the PATH - return python_names[-1] - @staticmethod def get_python_version(cmd): @@ -940,7 +919,7 @@ class SphinxDependencyChecker: else: print("\nSphinx needs to be installed either:\n1) via pip/pypi with:\n") - self.python_cmd = self.find_python_no_venv() + self.python_cmd = os.path.abspath(sys.argv[0]) print(f"\t{virtualenv_cmd} {self.virtenv_dir}") print(f"\t. {self.virtenv_dir}/bin/activate") -- cgit From f25bf12afc03ce88ca82fd2ed67fbee5c1ed0cd8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:24 +0200 Subject: scripts: sphinx-pre-install: drop support for old virtualenv Up to Python 3.2, the virtual environment were created via virtualenv binary. As we dropped support for such old version, clean up the code. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/31afe394bcfd8f7e450263c1922d2c73b91d36d8.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 71 ++++++++++++------------------------------- 1 file changed, 19 insertions(+), 52 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index b639acd455cc..0a73b1b33842 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -80,7 +80,6 @@ class SphinxDependencyChecker: self.need_symlink = 0 self.need_sphinx = 0 self.need_pip = 0 - self.need_virtualenv = 0 self.rec_sphinx_upgrade = 0 self.verbose_warn_install = 1 @@ -919,12 +918,14 @@ class SphinxDependencyChecker: else: print("\nSphinx needs to be installed either:\n1) via pip/pypi with:\n") - self.python_cmd = os.path.abspath(sys.argv[0]) - - print(f"\t{virtualenv_cmd} {self.virtenv_dir}") - print(f"\t. {self.virtenv_dir}/bin/activate") - print(f"\tpip install -r {self.requirement_file}") - self.deactivate_help() + if not virtualenv_cmd: + print(" Currently not possible.\n") + print(" Please upgrade Python to a newer version and run this script again") + else: + print(f"\t{virtualenv_cmd} {self.virtenv_dir}") + print(f"\t. {self.virtenv_dir}/bin/activate") + print(f"\tpip install -r {self.requirement_file}") + self.deactivate_help() print("\n2) As a package with:") @@ -953,6 +954,7 @@ class SphinxDependencyChecker: def check_needs(self): self.get_system_release() + self.python_cmd = sys.executable # Check if Sphinx is already accessible from current environment self.check_sphinx() @@ -965,56 +967,21 @@ class SphinxDependencyChecker: ver = ver_str(self.cur_version) print(f"Sphinx version: {ver}\n") - # FIXME: Check python command line, trying first python3 - self.python_cmd = self.which("python3") - if not self.python_cmd: - self.python_cmd = self.check_program("python", 0) - # Check the type of virtual env, depending on Python version - if self.python_cmd: - if self.virtualenv: - try: - result = self.run( - [self.python_cmd, "--version"], - capture_output=True, - text=True, - check=True, - ) - - output = result.stdout + result.stderr - - match = re.search(r"(\d+)\.(\d+)\.", output) - if match: - major = int(match.group(1)) - minor = int(match.group(2)) - - if major < 3: - sys.exit("Python 3 is required to build the kernel docs") - if major == 3 and minor < 3: - self.need_virtualenv = True - else: - sys.exit(f"Warning: couldn't identify {self.python_cmd} version!") - - except subprocess.CalledProcessError as e: - sys.exit(f"Error checking Python version: {e}") - else: - self.add_package("python-sphinx", 0) + virtualenv_cmd = None - self.venv_ver = self.recommend_sphinx_upgrade() + if sys.version_info < MIN_PYTHON_VERSION: + min_ver = ver_str(MIN_PYTHON_VERSION) + print(f"ERROR: at least python {min_ver} is required to build the kernel docs") + self.need_sphinx = 1 - virtualenv_cmd = "" + self.venv_ver = self.recommend_sphinx_upgrade() if self.need_pip: - # Set virtualenv command line, if python < 3.3 - # FIXME: can be removed as we're now with an upper min requirement - # but then we need to check python version - if self.need_virtualenv: - virtualenv_cmd = self.which("virtualenv-3") - if not virtualenv_cmd: - virtualenv_cmd = self.which("virtualenv-3.5") - if not virtualenv_cmd: - self.check_program("virtualenv", 0) - virtualenv_cmd = "virtualenv" + if sys.version_info < MIN_PYTHON_VERSION: + self.need_pip = False + print("Warning: python version is not supported.") + else: virtualenv_cmd = f"{self.python_cmd} -m venv" self.check_python_module("ensurepip", 0) -- cgit From 2cb4877b74349e33be9e209fbcce6114c9a38b9f Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:25 +0200 Subject: scripts: sphinx-pre-install: Address issues with OpenSUSE Leap 15.x On openSUSE Leap 15.6, which is the current LTS version, has two Sphinx packages. The normal one requires Python 3.6, which we don't support anymore. However, it also has Python 3.11 with a newer Sphinx version (7.2.6). Suggest the newer version: Detected OS: openSUSE Leap 15.6. ERROR: at least python 3.7 is required to build the kernel docs Warning: python version is not supported. Warning: better to also install "convert". Warning: better to also install "dot". ERROR: please install "yaml", otherwise, build won't work. You should run: sudo zypper install --no-recommends ImageMagick graphviz python311-pyyaml Sphinx needs to be installed either: 1) via pip/pypi with: Currently not possible. Please upgrade Python to a newer version and run this script again 2) As a package with: sudo zypper install --no-recommends python311-Sphinx Please note that Sphinx >= 3.0 will currently produce false-positive warning when the same name is used for more than one type (functions, structs, enums,...). This is known Sphinx bug. For more details, see: https://github.com/sphinx-doc/sphinx/pull/8313 Can't build as 2 mandatory dependencies are missing Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/a1600e292b63f96f40163e350238812158ebd6c2.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 0a73b1b33842..eca42d90ed01 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -556,6 +556,22 @@ class SphinxDependencyChecker: progs["latexmk"] = "texlive-latexmk-bin" + match = re.search(r"(Leap)\s+(\d+).(\d)", self.system_release) + if match: + rel = int(match.group(2)) + + # Leap 15.x uses Python 3.6, which is not compatible with + # the build system anymore. Suggest Python 3.11 + if rel == 15: + if not self.which(self.python_cmd): + self.add_package(self.python_cmd, 0) + + progs.update({ + "python-sphinx": "python311-Sphinx", + "virtualenv": "python311-virtualenv", + "yaml": "python311-pyyaml", + }) + # FIXME: add support for installing CJK fonts # # I tried hard, but was unable to find a way to install -- cgit From c5ffae0fa965884d09458002e3d990c039602809 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:26 +0200 Subject: scripts: sphinx-pre-install: fix opensuse Leap hint for PyYAML On Leap, the name of the package is python311-PyYAML. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0266c5e28e136c62de33d2d7f7d5e69b273e0d01.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index eca42d90ed01..65438c198674 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -569,7 +569,7 @@ class SphinxDependencyChecker: progs.update({ "python-sphinx": "python311-Sphinx", "virtualenv": "python311-virtualenv", - "yaml": "python311-pyyaml", + "yaml": "python311-PyYAML", }) # FIXME: add support for installing CJK fonts -- cgit From 582b0f95c92e0a0ec79d23ed434786251bd4378b Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:27 +0200 Subject: scripts: sphinx-pre-install: fix support for gentoo Some gentoo packages have changes. Fix it. While here, improve emerge portage use logic. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/283987633aae7d99e0eff41e24eb6285c1b6b335.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 65438c198674..b793796329c8 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -669,12 +669,13 @@ class SphinxDependencyChecker: def give_gentoo_hints(self): progs = { - "convert": "media-gfx/imagemagick", - "dot": "media-gfx/graphviz", - "rsvg-convert": "gnome-base/librsvg", - "virtualenv": "dev-python/virtualenv", - "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", - "yaml": "dev-python/pyyaml", + "convert": "media-gfx/imagemagick", + "dot": "media-gfx/graphviz", + "rsvg-convert": "gnome-base/librsvg", + "virtualenv": "dev-python/virtualenv", + "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", + "yaml": "dev-python/pyyaml", + "python-sphinx": "dev-python/sphinx", } if self.pdf: @@ -699,21 +700,17 @@ class SphinxDependencyChecker: print("You should run:") print("\n") - imagemagick = "media-gfx/imagemagick svg png" - cairo = "media-gfx/graphviz cairo pdf" - portage_imagemagick = "/etc/portage/package.use/imagemagick" - portage_cairo = "/etc/portage/package.use/graphviz" - result = self.run(["grep", "imagemagick", portage_imagemagick], - stdout=subprocess.PIPE, text=True) - if not result.stdout.strip(): - print(f"\tsudo su -c 'echo \"{imagemagick}\" > {portage_imagemagick}'") - - result = self.run(["grep", "graphviz", portage_cairo], - stdout=subprocess.PIPE, text=True) + portages = [ + "media-gfx/imagemagick", + "media-gfx/graphviz", + ] - if not result.stdout.strip(): - print(f"\tsudo su -c 'echo \"{cairo}\" > {portage_cairo}'") + for p in portages: + result = self.run(["grep", p, "/etc/portage/package.use/*"], + stdout=subprocess.PIPE, text=True) + if not result.stdout.strip(): + print(f"\tsudo emerge -av1 {p}") print(f"\tsudo emerge --ask {self.install}") -- cgit From 94a161d998a103346088d97da6d1cfec4df97ddc Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:28 +0200 Subject: scripts: sphinx-pre-install: Address issues with OpenSUSE Tumbleweed On Tumbleweed, package names are named after python-313*, as it also has older python versions on it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/fe0b5f7c18d7b32e0229c0890371e1441ffea294.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index b793796329c8..0e165ad05fdb 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -571,6 +571,14 @@ class SphinxDependencyChecker: "virtualenv": "python311-virtualenv", "yaml": "python311-PyYAML", }) + else: + # Tumbleweed defaults to Python 3.11 + + progs.update({ + "python-sphinx": "python313-Sphinx", + "virtualenv": "python313-virtualenv", + "yaml": "python313-PyYAML", + }) # FIXME: add support for installing CJK fonts # -- cgit From 12bdcf898977172a93fc30668f8918da83c51f86 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:29 +0200 Subject: scripts: sphinx-pre-install: only show portage hints once On gentoo, doesn't repeat instructions about how to enable portage. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/d4e7ef1d5e1f0412eb9f9ae4913dc64c0821e002.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 0e165ad05fdb..808d31bfa790 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -89,6 +89,9 @@ class SphinxDependencyChecker: self.python_cmd = "" self.activate_cmd = "" + # Certain hints are meant to be shown only once + self.first_hint = True + self.min_version = (0, 0, 0) self.cur_version = (0, 0, 0) self.latest_avail_ver = (0, 0, 0) @@ -714,11 +717,14 @@ class SphinxDependencyChecker: "media-gfx/graphviz", ] - for p in portages: - result = self.run(["grep", p, "/etc/portage/package.use/*"], - stdout=subprocess.PIPE, text=True) - if not result.stdout.strip(): - print(f"\tsudo emerge -av1 {p}") + if self.first_hint: + for p in portages: + result = self.run(["grep", p, "/etc/portage/package.use/*"], + stdout=subprocess.PIPE, text=True) + if not result.stdout.strip(): + print(f"\tsudo emerge -av1 {p}") + + self.first_hint = False print(f"\tsudo emerge --ask {self.install}") -- cgit From 8f6f54c464d7fccfb8da46ca0afcb0860e6d73bb Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:30 +0200 Subject: scripts: sphinx-pre-install: cleanup rhel support Rhel < 8.0 is not supported anymore. Drop support for it. Rhel 8 is problematic: at least on the tests I did with a docker repo, it didn't work, but it could be due to the issue that it is actually different than a real One. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/62fe8ab243ad39f4964f1f74b965e43dc8f10e23.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 808d31bfa790..42f55e67256d 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -487,15 +487,31 @@ class SphinxDependencyChecker: progs["virtualenv"] = "python-virtualenv" - if rel and rel < 8: + if not rel or rel < 8: old = 1 self.pdf = False - # RHEL 7 is in ELS, currently up to Jun, 2026 + print("ERROR: Distro not supported. Too old?") + return + + # TODO: check if RHEL8 still works. + # On my tests with docker "redhat/ubi8" image, there's no + # python3-sphinx (or similar) package. It comes with Python 3.6, + # but there are other python packages over there, so it may be + # possible to work with venv. + + if self.first_hint: + print("Note: RHEL-based distros typically require extra repositories.\n" \ + "For most, enabling epel and crb are enough:\n" \ + "\tsudo dnf install -y epel-release", \ + "\tsudo dnf config-manager --set-enabled crb\n" \ + "Yet, some may have other required repositories. Those commands could be useful:" \ + "\tsudo dnf repolist all" \ + "\tsudo repoquery --available --info ", + "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want") + + self.first_hint = False - print("Note: texlive packages on RHEL/CENTOS <= 7 are incomplete. Can't support PDF output") - print("If you want to build PDF, please read:") - print("\thttps://www.systutorials.com/241660/how-to-install-tex-live-on-centos-7-linux/") if self.pdf: pdf_pkgs = [ -- cgit From 637fa6b38113fddca7da2f0507f709f46a44047a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:31 +0200 Subject: scripts: sphinx-pre-install: output Python and docutils version Specially when debugging issues, knowing the versions is important. Add it to the script output. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/96142ec916b9064c859b200639c3595115080322.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 42f55e67256d..de5bcfd052b5 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -154,6 +154,20 @@ class SphinxDependencyChecker: cur_ver = sys.version_info[:3] if cur_ver >= MIN_PYTHON_VERSION: + ver = ver_str(cur_ver) + print(f"Python version: {ver}") + + # This could be useful for debugging purposes + if SphinxDependencyChecker.which("docutils"): + result = SphinxDependencyChecker.run(["docutils", "--version"], + capture_output=True, text=True) + ver = result.stdout.strip() + match = re.search(r"(\d+\.\d+\.\d+)", ver) + if match: + ver = match.group(1) + + print(f"Docutils version: {ver}") + return python_ver = ver_str(cur_ver) -- cgit From cccc5389811a34eccbd0f531c89d284f1cebdd70 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:32 +0200 Subject: scripts: sphinx-pre-install: add a missing f-string marker I forgot one f-string marker, with turned to be affecting 3 lines, because of cut-and-paste ;-) Use the proper f-string marker to print Sphinx version at the hint lines. Yet, we don't want to print as a tuple, so call ver_str() for it. Ideally, we would be placing it directly at the f-string, but Python 3.6 f-string support was pretty much limited. Only 3.12 (PEP 701) makes it similar to Perl, allowing expressions inside it. It sounds that function call itself was introduced on 3.7. As we explicitly want this one to run on 3.6, as latest Leap comes with it, we can't use function calls on f-string. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/b0ad1795446b17a00ba2dd83f366e784253668e6.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index de5bcfd052b5..6a244105f7ef 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -914,12 +914,15 @@ class SphinxDependencyChecker: if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: return + if self.latest_avail_ver: + latest_avail_ver = ver_str(self.latest_avail_ver) + if not self.need_sphinx: # sphinx-build is present and its version is >= $min_version # only recommend enabling a newer virtenv version if makes sense. if self.latest_avail_ver and self.latest_avail_ver > self.cur_version: - print("\nYou may also use the newer Sphinx version {self.latest_avail_ver} with:") + print(f"\nYou may also use the newer Sphinx version {latest_avail_ver} with:") if f"{self.virtenv_prefix}" in os.getcwd(): print("\tdeactivate") print(f"\t. {self.activate_cmd}") @@ -940,7 +943,7 @@ class SphinxDependencyChecker: # installed one via virtenv with a newer version. # So, print commands to enable it if self.latest_avail_ver > self.cur_version: - print("\nYou may also use the Sphinx virtualenv version {self.latest_avail_ver} with:") + print(f"\nYou may also use the Sphinx virtualenv version {latest_avail_ver} with:") if f"{self.virtenv_prefix}" in os.getcwd(): print("\tdeactivate") print(f"\t. {self.activate_cmd}") @@ -954,7 +957,7 @@ class SphinxDependencyChecker: # Suggest newer versions if current ones are too old if self.latest_avail_ver and self.latest_avail_ver >= self.min_version: if self.latest_avail_ver >= RECOMMENDED_VERSION: - print("\nNeed to activate Sphinx (version {self.latest_avail_ver}) on virtualenv with:") + print(f"\nNeed to activate Sphinx (version {latest_avail_ver}) on virtualenv with:") print(f"\t. {self.activate_cmd}") self.deactivate_help() return -- cgit From 8b18e86f6c0063e33096bf84ffcd9fb9ced4844d Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:33 +0200 Subject: scripts: sphinx-pre-install: fix Leap support for rsvg-convert There is a test logic meant to be for Leap, renaming rsvg-convert package. Well, at least on latest Leap releases, this is wrong, causing install to fail. Drop it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/6fcc94533d860e2f6f4f2c7d6ceb6cca70f48bb6.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 6a244105f7ef..6b5f9eda91bb 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -566,10 +566,6 @@ class SphinxDependencyChecker: "yaml": "python3-pyyaml", } - # On Tumbleweed, this package is also named rsvg-convert - if not re.search(r"Tumbleweed", self.system_release): - progs["rsvg-convert"] = "rsvg-view" - suse_tex_pkgs = [ "texlive-babel-english", "texlive-caption", -- cgit From e53e6d395fe036bb2c24dc79dc728e9a82ee0b82 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:34 +0200 Subject: scripts: sphinx-pre-install: fix rhel recomendations Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/393a299a850ba9d94c6a8965e78db4da2dbf7e37.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 6b5f9eda91bb..1f64909953b9 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -519,9 +519,9 @@ class SphinxDependencyChecker: "For most, enabling epel and crb are enough:\n" \ "\tsudo dnf install -y epel-release", \ "\tsudo dnf config-manager --set-enabled crb\n" \ - "Yet, some may have other required repositories. Those commands could be useful:" \ - "\tsudo dnf repolist all" \ - "\tsudo repoquery --available --info ", + "Yet, some may have other required repositories. Those commands could be useful:\n" \ + "\tsudo dnf repolist all\n" \ + "\tsudo dnf repoquery --available --info \n", "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want") self.first_hint = False -- cgit From 42e3f9f360e70bea4ee7d4e895f042d86ee11500 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:35 +0200 Subject: scripts: sphinx-pre-install: remove Scientific Linux According with its website: https://scientificlinux.org/ Scientific Linux reached end of life in June 30, 2024. Also, it was based on RHEL 7, which is not compatible with our build system anymore. So, drop support for it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/dde5e0c95017022840f8a522ce44759e51f52aa1.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 1 - 1 file changed, 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 1f64909953b9..cd9c83b17971 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -769,7 +769,6 @@ class SphinxDependencyChecker: re.compile("openEuler"): self.give_redhat_hints, re.compile("Oracle Linux Server"): self.give_redhat_hints, re.compile("Rocky Linux"): self.give_redhat_hints, - re.compile("Scientific Linux"): self.give_redhat_hints, re.compile("Springdale Open Enterprise"): self.give_redhat_hints, re.compile("Ubuntu"): self.give_debian_hints, -- cgit From 1a7da749f18335d7996df3004225ca7418a1052d Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:36 +0200 Subject: scripts: sphinx-pre-install: improve Gentoo package deps logic It took me a lot of time, but I guess understand now what it takes to install a package on Gentoo. Handling dependencies is a nightmare, as Gentoo refuses to emerge some packages if there's no package.use file describing them. To make it worse, compilation flags shall also be present there for some packages. If USE is not perfect, error/warning messages like those are shown: gnome-base/librsvg dev-texlive/texlive-xetex media-fonts/dejavu dev-python/pyyaml ... !!! The following binary packages have been ignored due to non matching USE: =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_13 qt6 svg =media-gfx/graphviz-12.2.1-r1 X pdf python_single_target_python3_12 -python_single_target_python3_13 qt6 svg =media-gfx/graphviz-12.2.1-r1 X pdf qt6 svg =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 qt6 svg =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 python_single_target_python3_12 -python_single_target_python3_13 qt6 svg =media-fonts/noto-cjk-20190416 X =app-text/texlive-core-2024-r1 X cjk -xetex =app-text/texlive-core-2024-r1 X -xetex =app-text/texlive-core-2024-r1 -xetex =dev-libs/zziplib-0.13.79-r1 sdl If emerge is allowed, it will simply ignore the above packages, creating an incomplete installation, which will later fail when one tries to build docs with images or build PDFs. After the fix, command line commands to produce the needed USE chain will be emitted, if they don't exist yet. sudo su -c 'echo "media-gfx/graphviz" > /etc/portage/package.use/graphviz' sudo su -c 'echo "media-gfx/imagemagick" > /etc/portage/package.use/imagemagick' sudo su -c 'echo "media-libs/harfbuzz icu" > /etc/portage/package.use/media-libs' sudo su -c 'echo "media-fonts/noto-cjk" > /etc/portage/package.use/media-fonts' sudo su -c 'echo "app-text/texlive-core xetex" > /etc/portage/package.use/texlive' sudo su -c 'echo "dev-libs/zziplib sdl" > /etc/portage/package.use/zziblib' The new logic tries to be smart enough to detect for missing files and missing arguments. Yet, as Gentoo seems to require users to manage those package.use files by hand, the logic isn't perfect: users may still need to verify for conflicts on different use files. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/365fe5e7d568da932dcffde65f48f2c1256cb773.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 88 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 77 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index cd9c83b17971..94f3d2e32fd6 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -737,22 +737,88 @@ class SphinxDependencyChecker: print("You should run:") print("\n") - - portages = [ - "media-gfx/imagemagick", - "media-gfx/graphviz", - ] + # Handling dependencies is a nightmare, as Gentoo refuses to emerge + # some packages if there's no package.use file describing them. + # To make it worse, compilation flags shall also be present there + # for some packages. If USE is not perfect, error/warning messages + # like those are shown: + # + # !!! The following binary packages have been ignored due to non matching USE: + # + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_13 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf python_single_target_python3_12 -python_single_target_python3_13 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 python_single_target_python3_12 -python_single_target_python3_13 qt6 svg + # =media-fonts/noto-cjk-20190416 X + # =app-text/texlive-core-2024-r1 X cjk -xetex + # =app-text/texlive-core-2024-r1 X -xetex + # =app-text/texlive-core-2024-r1 -xetex + # =dev-libs/zziplib-0.13.79-r1 sdl + # + # And will ignore such packages, installing the remaining ones. That + # affects mostly the image extension and PDF generation. + + # Package dependencies and the minimal needed args: + portages = { + "graphviz": "media-gfx/graphviz", + "imagemagick": "media-gfx/imagemagick", + "media-libs": "media-libs/harfbuzz icu", + "media-fonts": "media-fonts/noto-cjk", + "texlive": "app-text/texlive-core xetex", + "zziblib": "dev-libs/zziplib sdl", + } if self.first_hint: - for p in portages: - result = self.run(["grep", p, "/etc/portage/package.use/*"], - stdout=subprocess.PIPE, text=True) - if not result.stdout.strip(): - print(f"\tsudo emerge -av1 {p}") + use_base = "/etc/portage/package.use" + files = glob(f"{use_base}/*") + + for fname, portage in portages.items(): + install = False + + while install == False: + if not files: + # No files under package.usage. Install all + install = True + break + + args = portage.split(" ") + + name = args.pop(0) + + cmd = ["grep", "-l", "-E", rf"^{name}\b" ] + files + result = self.run(cmd, stdout=subprocess.PIPE, text=True) + if result.returncode or not result.stdout.strip(): + # File containing portage name not found + install = True + break + + # Ensure that needed USE flags are present + if args: + match_fname = result.stdout.strip() + with open(match_fname, 'r', encoding='utf8', + errors='backslashreplace') as fp: + for line in fp: + for arg in args: + if arg.startswith("-"): + continue + + if not re.search(rf"\s*{arg}\b", line): + # Needed file argument not found + install = True + break + + # Everything looks ok, don't install + break + + # emit a code to setup missing USE + if install: + print(f"\tsudo su -c 'echo \"{portage}\" > {use_base}/{fname}'") self.first_hint = False - print(f"\tsudo emerge --ask {self.install}") + # Now, we can use emerge and let it respect USE + print(f"\tsudo emerge --ask --changed-use --binpkg-respect-use=y {self.install}") # # Dispatch the check to an os_specific hinter -- cgit From fb08659be07e646f8103449e02d5e58b32c284d9 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:37 +0200 Subject: scripts: sphinx-pre-install: fix OpenMandriva support OpenMandriva Lx 4.3 has different package names for ImageMagick and yaml. Fix them to ensure that system setup will pass. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0b4e7aa88c96e6a5b8f2e6f381b3e21124680d33.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 94f3d2e32fd6..2f6036eadc94 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -647,6 +647,11 @@ class SphinxDependencyChecker: packager_cmd = "dnf install" noto_sans = "noto-sans-cjk-fonts" tex_pkgs = ["texlive-collection-fontsextra"] + + # Tested on OpenMandriva Lx 4.3 + progs["convert"] = "imagemagick" + progs["yaml"] = "python-pyyaml" + else: packager_cmd = "urpmi" noto_sans = "google-noto-sans-cjk-ttc-fonts" -- cgit From 1ad72e9dfa5ef24ca489b7cbaa5f6f85b1fe87d8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:38 +0200 Subject: scripts: sphinx-pre-install: move package instructions to a new func Currently, if Python < 3.7, package install will fail. That happens with OpenSuse Leap and RHEL-based ver 8 distros. OpenSuse allows installing Sphinx with Python 3.11, but RHEL-based distros don't. Prepare to recomend only venv on such cases. For now, just split the recomendation on a new function that will check for a paramtere to be called. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/4fb2181c960e89774309a833f80209a1a3ab10d2.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 44 +++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 2f6036eadc94..2360ca2ed21c 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -89,6 +89,9 @@ class SphinxDependencyChecker: self.python_cmd = "" self.activate_cmd = "" + # Some distros may not have a Sphinx shipped package compatible with + # our minimal requirements + self.package_supported = True # Certain hints are meant to be shown only once self.first_hint = True @@ -969,6 +972,27 @@ class SphinxDependencyChecker: return self.latest_avail_ver + def recommend_package(self): + + print("\n2) As a package with:") + + old_need = self.need + old_optional = self.optional + self.missing = {} + self.pdf = False + self.optional = 0 + self.install = "" + old_verbose = self.verbose_warn_install + self.verbose_warn_install = 0 + + self.add_package("python-sphinx", 0) + + self.check_distros() + + self.need = old_need + self.optional = old_optional + self.verbose_warn_install = old_verbose + def recommend_sphinx_version(self, virtualenv_cmd): # The logic here is complex, as it have to deal with different versions: # - minimal supported version; @@ -1053,24 +1077,8 @@ class SphinxDependencyChecker: print(f"\tpip install -r {self.requirement_file}") self.deactivate_help() - print("\n2) As a package with:") - - old_need = self.need - old_optional = self.optional - self.missing = {} - self.pdf = False - self.optional = 0 - self.install = "" - old_verbose = self.verbose_warn_install - self.verbose_warn_install = 0 - - self.add_package("python-sphinx", 0) - - self.check_distros() - - self.need = old_need - self.optional = old_optional - self.verbose_warn_install = old_verbose + if self.package_supported: + self.recommend_package() print("\n" \ " Please note that Sphinx >= 3.0 will currently produce false-positive\n" \ -- cgit From 6db1d3977baf7ac895033624897d38fb60cbe3f7 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:39 +0200 Subject: scripts: sphinx-pre-install: adjust a warning message There is one extra space at the first line. Also, as now we only support Python 3.4+, update the text. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/74a17edd70364ca623a54b62bd97a344bb474988.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 2360ca2ed21c..365590f81551 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -1081,8 +1081,8 @@ class SphinxDependencyChecker: self.recommend_package() print("\n" \ - " Please note that Sphinx >= 3.0 will currently produce false-positive\n" \ - " warning when the same name is used for more than one type (functions,\n" \ + " Please note that Sphinx currentlys produce false-positive\n" \ + " warnings when the same name is used for more than one type (functions,\n" \ " structs, enums,...). This is known Sphinx bug. For more details, see:\n" \ "\thttps://github.com/sphinx-doc/sphinx/pull/8313") -- cgit From 8cd256524a920782268a0dd7e3e0404c34bfc65f Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:40 +0200 Subject: scripts: sphinx-pre-install: better handle Python min version Don't do any recommendations about Sphinx install with too old python versions. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/013aeb848ecc3f6b69b4518cf3d335bd2353b6e1.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 365590f81551..a5c777e529ec 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -92,6 +92,10 @@ class SphinxDependencyChecker: # Some distros may not have a Sphinx shipped package compatible with # our minimal requirements self.package_supported = True + + # Recommend a new python version + self.recommend_python = None + # Certain hints are meant to be shown only once self.first_hint = True @@ -511,11 +515,11 @@ class SphinxDependencyChecker: print("ERROR: Distro not supported. Too old?") return - # TODO: check if RHEL8 still works. - # On my tests with docker "redhat/ubi8" image, there's no - # python3-sphinx (or similar) package. It comes with Python 3.6, - # but there are other python packages over there, so it may be - # possible to work with venv. + # RHEL 8 uses Python 3.6, which is not compatible with + # the build system anymore. Suggest Python 3.11 + if rel == 8: + self.add_package("python39", 0) + self.recommend_python = True if self.first_hint: print("Note: RHEL-based distros typically require extra repositories.\n" \ @@ -596,6 +600,7 @@ class SphinxDependencyChecker: # the build system anymore. Suggest Python 3.11 if rel == 15: if not self.which(self.python_cmd): + self.recommend_python = True self.add_package(self.python_cmd, 0) progs.update({ @@ -1000,6 +1005,11 @@ class SphinxDependencyChecker: # - recommended version. # It also needs to work fine with both distro's package and venv/virtualenv + if self.recommend_python: + print("\nPython version is incompatible with doc build.\n" \ + "Please upgrade it and re-run.\n") + return + # Version is OK. Nothing to do. if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: return -- cgit From 9ecda2e10101537e97bcb2b05049e58306ac4e1b Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:41 +0200 Subject: scripts: sphinx-pre-install: convert is_optional to a class When is_optional was added in Perl, it was a boolean. With time, it ended becoming a sort of enum, which makes the module harder to maintain. Convert it to a enum-like class and add more options to it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/42290a24f3b1dbea9ebe19747cf5622bb2f2cf5c.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 192 ++++++++++++++++++++++++++++-------------- 1 file changed, 130 insertions(+), 62 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index a5c777e529ec..0963da21c27b 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -32,6 +32,53 @@ RECOMMENDED_VERSION = parse_version("3.4.3") MIN_PYTHON_VERSION = parse_version("3.7") +class DepType: + + # Internal types of dependencies. + _SYS_TYPE = 0 + _PHY_TYPE = 1 + _PDF_TYPE = 2 + + # Let's define keys as a tuple with the type and mandatory/optional. + # This way, checking for optional or type is easy. + + SYSTEM_MANDATORY = (_SYS_TYPE, True) + PYTHON_MANDATORY = (_PHY_TYPE, True) + PDF_MANDATORY = (_PDF_TYPE, True) + + # Currently we're not using all optional types, but let's keep all + # combinations here, as we may end needing them in the future. Also, + # it allows a name() function that handles all possibilities. + SYSTEM_OPTIONAL = (_SYS_TYPE, False) + PYTHON_OPTIONAL = (_PHY_TYPE, False) + PDF_OPTIONAL = (_PDF_TYPE, True) + + @staticmethod + def name(dtype): + if dtype[0] == DepType._SYS_TYPE: + msg = "build" + elif dtype[0] == DepType._PHY_TYPE: + msg = "Python" + else: + msg = "PDF" + + if dtype[1]: + return f"ERROR: {msg} mandatory deps missing" + else: + out = f"Warning: {msg} optional deps missing" + + @staticmethod + def is_optional(dtype): + return not dtype[1] + + @staticmethod + def is_pdf(dtype): + if (dtype[0] == DepType._PDF_TYPE): + return True + + return False + + class SphinxDependencyChecker: # List of required texlive packages on Fedora and OpenSuse texlive = { @@ -223,56 +270,68 @@ class SphinxDependencyChecker: # Methods to check if a feature exists # - # Note: is_optional has 3 states: - # - 0: mandatory - # - 1: optional, but nice to have - # - 2: LaTeX optional - pdf builds without it, but may have visual impact - def check_missing(self, progs): - for prog, is_optional in sorted(self.missing.items()): + run = {} + + for prog, dtype in sorted(self.missing.items()): # At least on some LTS distros like CentOS 7, texlive doesn't # provide all packages we need. When such distros are # detected, we have to disable PDF output. # # So, we need to ignore the packages that distros would # need for LaTeX to work - if is_optional == 2 and not self.pdf: + if DepType.is_pdf(dtype) and not self.pdf: self.optional -= 1 continue + if not dtype in run: + run[dtype] = [] + + run[dtype].append(prog) + + output_msg = "" + + for dtype in sorted(run.keys()): + progs = " ".join(run[dtype]) + if self.verbose_warn_install: - if is_optional: - print(f'Warning: better to also install "{prog}".') - else: - print(f'ERROR: please install "{prog}", otherwise, build won\'t work.') + try: + name = DepType.name(dtype) + output_msg += f'{name}:\t{progs}\n' + except KeyError: + raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") - self.install += " " + progs.get(prog, prog) + self.install += " " + progs + + if output_msg: + print(f"\n{output_msg}\n") self.install = self.install.lstrip() - def add_package(self, package, is_optional): - self.missing[package] = is_optional + def add_package(self, package, dtype): + is_optional = DepType.is_optional(dtype) + self.missing[package] = dtype if is_optional: self.optional += 1 else: self.need += 1 - def check_missing_file(self, files, package, is_optional): + def check_missing_file(self, files, package, dtype): for f in files: if os.path.exists(f): return - self.add_package(package, is_optional) + self.add_package(package, dtype) - def check_program(self, prog, is_optional): + def check_program(self, prog, dtype): found = self.which(prog) if found: return found - self.add_package(prog, is_optional) + self.add_package(prog, dtype) return None - def check_perl_module(self, prog, is_optional): + def check_perl_module(self, prog, dtype): # While testing with lxc download template, one of the # distros (Oracle) didn't have perl - nor even an option to install # before installing oraclelinux-release-el9 package. @@ -281,46 +340,52 @@ class SphinxDependencyChecker: # add it as a mandatory package, as some parts of the doc builder # needs it. if not self.which("perl"): - self.add_package("perl", 0) - self.add_package(prog, is_optional) + self.add_package("perl", DepType.SYSTEM_MANDATORY) + self.add_package(prog, dtype) return try: self.run(["perl", f"-M{prog}", "-e", "1"], check=True) except subprocess.CalledProcessError: - self.add_package(prog, is_optional) + self.add_package(prog, dtype) - def check_python_module(self, module, is_optional): - # FIXME: is it needed at the Python version? Maybe due to venv? - if not self.python_cmd: - return + def check_python_module(self, module, is_optional=False): + if is_optional: + dtype = DepType.PYTHON_OPTIONAL + else: + dtype = DepType.PYTHON_MANDATORY try: self.run([self.python_cmd, "-c", f"import {module}"], check=True) except subprocess.CalledProcessError: - self.add_package(module, is_optional) + self.add_package(module, dtype) - def check_rpm_missing(self, pkgs, is_optional): + def check_rpm_missing(self, pkgs, dtype): for prog in pkgs: try: self.run(["rpm", "-q", prog], check=True) except subprocess.CalledProcessError: - self.add_package(prog, is_optional) + self.add_package(prog, dtype) - def check_pacman_missing(self, pkgs, is_optional): + def check_pacman_missing(self, pkgs, dtype): for prog in pkgs: try: self.run(["pacman", "-Q", prog], check=True) except subprocess.CalledProcessError: - self.add_package(prog, is_optional) + self.add_package(prog, dtype) + + def check_missing_tex(self, is_optional=False): + if is_optional: + dtype = DepType.PDF_OPTIONAL + else: + dtype = DepType.PDF_MANDATORY - def check_missing_tex(self, is_optional): kpsewhich = self.which("kpsewhich") for prog, package in self.texlive.items(): # If kpsewhich is not there, just add it to deps if not kpsewhich: - self.add_package(package, is_optional) + self.add_package(package, dtype) continue # Check if the package is needed @@ -331,11 +396,11 @@ class SphinxDependencyChecker: # Didn't find. Add it if not result.stdout.strip(): - self.add_package(package, is_optional) + self.add_package(package, dtype) except subprocess.CalledProcessError: # kpsewhich returned an error. Add it, just in case - self.add_package(package, is_optional) + self.add_package(package, dtype) def get_sphinx_fname(self): if "SPHINXBUILD" in os.environ: @@ -446,9 +511,9 @@ class SphinxDependencyChecker: } for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, 2) + self.check_missing_file(files, package, DepType.PDF_MANDATORY) - self.check_program("dvipng", 2) + self.check_program("dvipng", DepType.PDF_MANDATORY) self.check_missing(progs) @@ -518,7 +583,7 @@ class SphinxDependencyChecker: # RHEL 8 uses Python 3.6, which is not compatible with # the build system anymore. Suggest Python 3.11 if rel == 8: - self.add_package("python39", 0) + self.add_package("python39", DepType.SYSTEM_MANDATORY) self.recommend_python = True if self.first_hint: @@ -540,13 +605,13 @@ class SphinxDependencyChecker: "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc", ] - self.check_missing_file(pdf_pkgs, noto_sans_redhat, 2) + self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepType.PDF_MANDATORY) if not old: - self.check_rpm_missing(fedora26_opt_pkgs, 2) - self.check_rpm_missing(fedora_tex_pkgs, 2) + self.check_rpm_missing(fedora26_opt_pkgs, DepType.PDF_MANDATORY) + self.check_rpm_missing(fedora_tex_pkgs, DepType.PDF_MANDATORY) - self.check_missing_tex(2) + self.check_missing_tex() self.check_missing(progs) @@ -601,7 +666,7 @@ class SphinxDependencyChecker: if rel == 15: if not self.which(self.python_cmd): self.recommend_python = True - self.add_package(self.python_cmd, 0) + self.add_package(self.python_cmd, DepType.SYSTEM_MANDATORY) progs.update({ "python-sphinx": "python311-Sphinx", @@ -623,9 +688,9 @@ class SphinxDependencyChecker: # "Noto Sans CJK SC" on openSUSE if self.pdf: - self.check_rpm_missing(suse_tex_pkgs, 2) + self.check_rpm_missing(suse_tex_pkgs, DepType.PDF_MANDATORY) if self.pdf: - self.check_missing_tex(2) + self.check_missing_tex() self.check_missing(progs) if not self.need and not self.optional: @@ -672,8 +737,8 @@ class SphinxDependencyChecker: "/usr/share/fonts/TTF/NotoSans-Regular.ttf", ] - self.check_missing_file(pdf_pkgs, noto_sans, 2) - self.check_rpm_missing(tex_pkgs, 2) + self.check_missing_file(pdf_pkgs, noto_sans, DepType.PDF_MANDATORY) + self.check_rpm_missing(tex_pkgs, DepType.PDF_MANDATORY) self.check_missing(progs) @@ -701,7 +766,7 @@ class SphinxDependencyChecker: ] if self.pdf: - self.check_pacman_missing(archlinux_tex_pkgs, 2) + self.check_pacman_missing(archlinux_tex_pkgs, DepType.PDF_MANDATORY) self.check_missing_file( ["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], @@ -739,7 +804,7 @@ class SphinxDependencyChecker: ], } for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, 2) + self.check_missing_file(files, package, DepType.PDF_MANDATORY) self.check_missing(progs) @@ -878,7 +943,7 @@ class SphinxDependencyChecker: # progs = {"sphinx-build": "sphinx"} if self.pdf: - self.check_missing_tex(2) + self.check_missing_tex() self.check_missing(progs) @@ -990,7 +1055,7 @@ class SphinxDependencyChecker: old_verbose = self.verbose_warn_install self.verbose_warn_install = 0 - self.add_package("python-sphinx", 0) + self.add_package("python-sphinx", DepType.PYTHON_MANDATORY) self.check_distros() @@ -1010,6 +1075,7 @@ class SphinxDependencyChecker: "Please upgrade it and re-run.\n") return + # Version is OK. Nothing to do. if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: return @@ -1128,21 +1194,23 @@ class SphinxDependencyChecker: else: virtualenv_cmd = f"{self.python_cmd} -m venv" - self.check_python_module("ensurepip", 0) + self.check_python_module("ensurepip") # Check for needed programs/tools - self.check_perl_module("Pod::Usage", 0) - self.check_python_module("yaml", 0) - self.check_program("make", 0) - self.check_program("gcc", 0) - self.check_program("dot", 1) - self.check_program("convert", 1) + self.check_perl_module("Pod::Usage", DepType.SYSTEM_MANDATORY) + + self.check_program("make", DepType.SYSTEM_MANDATORY) + self.check_program("gcc", DepType.SYSTEM_MANDATORY) + + self.check_program("dot", DepType.SYSTEM_OPTIONAL) + self.check_program("convert", DepType.SYSTEM_OPTIONAL) + + self.check_python_module("yaml") if self.pdf: - # Extra PDF files - should use 2 for LaTeX is_optional - self.check_program("xelatex", 2) - self.check_program("rsvg-convert", 2) - self.check_program("latexmk", 2) + self.check_program("xelatex", DepType.PDF_MANDATORY) + self.check_program("rsvg-convert", DepType.PDF_MANDATORY) + self.check_program("latexmk", DepType.PDF_MANDATORY) # Do distro-specific checks and output distro-install commands self.check_distros() -- cgit From 272f5e0390dd9b5f62b480eae482a67d2493630e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:42 +0200 Subject: scripts: sphinx-pre-install: better handle RHEL-based distros Better implement support for RHEL-based distros. While here, get rid of a Fedora 28 support which cause troubles with server distros. Also, get rid of yum, as RHEL8 already suppords dnf, and this is not the minimal version we may still support. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/4d1b27d3a381f011e150bb50176babba83af9e1a.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 0963da21c27b..592223fa686f 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -536,10 +536,6 @@ class SphinxDependencyChecker: "yaml": "python3-pyyaml", } - fedora26_opt_pkgs = [ - "graphviz-gd", # Fedora 26: needed for PDF support - ] - fedora_tex_pkgs = [ "dejavu-sans-fonts", "dejavu-sans-mono-fonts", @@ -549,9 +545,8 @@ class SphinxDependencyChecker: "texlive-xecjk", ] - old = 0 + fedora = False rel = None - pkg_manager = "dnf" match = re.search(r"(release|Linux)\s+(\d+)", self.system_release) if match: @@ -559,12 +554,12 @@ class SphinxDependencyChecker: if not rel: print("Couldn't identify release number") - old = 1 self.pdf = False elif re.search("Fedora", self.system_release): # Fedora 38 and upper use this CJK font noto_sans_redhat = "google-noto-sans-cjk-fonts" + fedora = True else: # Almalinux, CentOS, RHEL, ... @@ -574,9 +569,6 @@ class SphinxDependencyChecker: progs["virtualenv"] = "python-virtualenv" if not rel or rel < 8: - old = 1 - self.pdf = False - print("ERROR: Distro not supported. Too old?") return @@ -607,11 +599,16 @@ class SphinxDependencyChecker: self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepType.PDF_MANDATORY) - if not old: - self.check_rpm_missing(fedora26_opt_pkgs, DepType.PDF_MANDATORY) - self.check_rpm_missing(fedora_tex_pkgs, DepType.PDF_MANDATORY) + self.check_rpm_missing(fedora_tex_pkgs, DepType.PDF_MANDATORY) + + self.check_missing_tex(DepType.PDF_MANDATORY) + + # There's no texlive-ctex on RHEL 8 repositories. This will + # likely affect CJK pdf build only. + if not fedora and rel == 8: + if "texlive-ctex" in self.missing: + del self.missing["texlive-ctex"] - self.check_missing_tex() self.check_missing(progs) @@ -621,11 +618,7 @@ class SphinxDependencyChecker: if self.verbose_warn_install: print("You should run:") - if old: - # dnf is there since Fedora 18+ and RHEL 8 - pkg_manager = "yum" - - print(f"\n\tsudo {pkg_manager} install -y {self.install}") + print(f"\n\tsudo dnf install -y {self.install}") def give_opensuse_hints(self): progs = { -- cgit From 2cab00fb178a31bca45adc1ff5f0994679d116a4 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:43 +0200 Subject: scripts: sphinx-pre-install: move missing logic to a separate class Better manage dependencies by placing them on a distro-independent class. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/f4f5bf276e07dc494f5dc83c4c2d087be7f790e6.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 251 +++++++++++++++++++++++------------------- 1 file changed, 138 insertions(+), 113 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 592223fa686f..47dce1fcddfb 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -32,7 +32,7 @@ RECOMMENDED_VERSION = parse_version("3.4.3") MIN_PYTHON_VERSION = parse_version("3.7") -class DepType: +class DepManager: # Internal types of dependencies. _SYS_TYPE = 0 @@ -53,11 +53,18 @@ class DepType: PYTHON_OPTIONAL = (_PHY_TYPE, False) PDF_OPTIONAL = (_PDF_TYPE, True) + def __init__(self, pdf): + self.missing = {} + self.run = {} + self.need = 0 + self.optional = 0 + self.pdf = pdf + @staticmethod def name(dtype): - if dtype[0] == DepType._SYS_TYPE: + if dtype[0] == DepManager._SYS_TYPE: msg = "build" - elif dtype[0] == DepType._PHY_TYPE: + elif dtype[0] == DepManager._PHY_TYPE: msg = "Python" else: msg = "PDF" @@ -73,11 +80,75 @@ class DepType: @staticmethod def is_pdf(dtype): - if (dtype[0] == DepType._PDF_TYPE): + if (dtype[0] == DepManager._PDF_TYPE): return True return False + def add_package(self, package, dtype): + is_optional = DepManager.is_optional(dtype) + self.missing[package] = dtype + if is_optional: + self.optional += 1 + else: + self.need += 1 + + def del_package(self, package): + if package in self.missing: + del self.missing[package] + + def clear_deps(self): + """ + Clear dependencies without changing needed/optional. + + This is an ackward way to have a separate section to recommend + a package after system main dependencies. + + TODO: rework the logic to prevent needing it + """ + + self.missing = {} + + def check_missing(self, progs): + self.run = {} + + for prog, dtype in sorted(self.missing.items()): + # At least on some LTS distros like CentOS 7, texlive doesn't + # provide all packages we need. When such distros are + # detected, we have to disable PDF output. + # + # So, we need to ignore the packages that distros would + # need for LaTeX to work + if DepManager.is_pdf(dtype) and not self.pdf: + self.optional -= 1 + continue + + if not dtype in self.run: + self.run[dtype] = [] + + self.run[dtype].append(progs.get(prog, prog)) + + install = [] + for dtype in self.run.keys(): + install += self.run[dtype] + + return " ".join(sorted(set(install))) + + def warn_install(self): + + output_msg = "" + + for dtype in sorted(self.run.keys()): + progs = " ".join(sorted(set(self.run[dtype]))) + + try: + name = DepManager.name(dtype) + output_msg += f'{name}:\t{progs}\n' + except KeyError: + raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") + + if output_msg: + print(f"\n{output_msg}\n") class SphinxDependencyChecker: # List of required texlive packages on Fedora and OpenSuse @@ -120,10 +191,8 @@ class SphinxDependencyChecker: self.virtualenv = args.virtualenv self.version_check = args.version_check - self.missing = {} + self.deps = DepManager(self.pdf) - self.need = 0 - self.optional = 0 self.need_symlink = 0 self.need_sphinx = 0 self.need_pip = 0 @@ -270,64 +339,18 @@ class SphinxDependencyChecker: # Methods to check if a feature exists # - def check_missing(self, progs): - run = {} - - for prog, dtype in sorted(self.missing.items()): - # At least on some LTS distros like CentOS 7, texlive doesn't - # provide all packages we need. When such distros are - # detected, we have to disable PDF output. - # - # So, we need to ignore the packages that distros would - # need for LaTeX to work - if DepType.is_pdf(dtype) and not self.pdf: - self.optional -= 1 - continue - - if not dtype in run: - run[dtype] = [] - - run[dtype].append(prog) - - output_msg = "" - - for dtype in sorted(run.keys()): - progs = " ".join(run[dtype]) - - if self.verbose_warn_install: - try: - name = DepType.name(dtype) - output_msg += f'{name}:\t{progs}\n' - except KeyError: - raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") - - self.install += " " + progs - - if output_msg: - print(f"\n{output_msg}\n") - - self.install = self.install.lstrip() - - def add_package(self, package, dtype): - is_optional = DepType.is_optional(dtype) - self.missing[package] = dtype - if is_optional: - self.optional += 1 - else: - self.need += 1 - def check_missing_file(self, files, package, dtype): for f in files: if os.path.exists(f): return - self.add_package(package, dtype) + self.deps.add_package(package, dtype) def check_program(self, prog, dtype): found = self.which(prog) if found: return found - self.add_package(prog, dtype) + self.deps.add_package(prog, dtype) return None @@ -340,52 +363,52 @@ class SphinxDependencyChecker: # add it as a mandatory package, as some parts of the doc builder # needs it. if not self.which("perl"): - self.add_package("perl", DepType.SYSTEM_MANDATORY) - self.add_package(prog, dtype) + self.deps.add_package("perl", DepManager.SYSTEM_MANDATORY) + self.deps.add_package(prog, dtype) return try: self.run(["perl", f"-M{prog}", "-e", "1"], check=True) except subprocess.CalledProcessError: - self.add_package(prog, dtype) + self.deps.add_package(prog, dtype) def check_python_module(self, module, is_optional=False): if is_optional: - dtype = DepType.PYTHON_OPTIONAL + dtype = DepManager.PYTHON_OPTIONAL else: - dtype = DepType.PYTHON_MANDATORY + dtype = DepManager.PYTHON_MANDATORY try: self.run([self.python_cmd, "-c", f"import {module}"], check=True) except subprocess.CalledProcessError: - self.add_package(module, dtype) + self.deps.add_package(module, dtype) def check_rpm_missing(self, pkgs, dtype): for prog in pkgs: try: self.run(["rpm", "-q", prog], check=True) except subprocess.CalledProcessError: - self.add_package(prog, dtype) + self.deps.add_package(prog, dtype) def check_pacman_missing(self, pkgs, dtype): for prog in pkgs: try: self.run(["pacman", "-Q", prog], check=True) except subprocess.CalledProcessError: - self.add_package(prog, dtype) + self.deps.add_package(prog, dtype) def check_missing_tex(self, is_optional=False): if is_optional: - dtype = DepType.PDF_OPTIONAL + dtype = DepManager.PDF_OPTIONAL else: - dtype = DepType.PDF_MANDATORY + dtype = DepManager.PDF_MANDATORY kpsewhich = self.which("kpsewhich") for prog, package in self.texlive.items(): # If kpsewhich is not there, just add it to deps if not kpsewhich: - self.add_package(package, dtype) + self.deps.add_package(package, dtype) continue # Check if the package is needed @@ -396,11 +419,11 @@ class SphinxDependencyChecker: # Didn't find. Add it if not result.stdout.strip(): - self.add_package(package, dtype) + self.deps.add_package(package, dtype) except subprocess.CalledProcessError: # kpsewhich returned an error. Add it, just in case - self.add_package(package, dtype) + self.deps.add_package(package, dtype) def get_sphinx_fname(self): if "SPHINXBUILD" in os.environ: @@ -478,6 +501,17 @@ class SphinxDependencyChecker: return f.read().strip() return "" + def check_missing(self, progs): + self.install += self.deps.check_missing(progs) + if self.verbose_warn_install: + self.deps.warn_install() + + if not self.deps.need and not self.deps.optional: + return False + + return True + + # # Distro-specific hints methods # @@ -511,13 +545,11 @@ class SphinxDependencyChecker: } for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, DepType.PDF_MANDATORY) - - self.check_program("dvipng", DepType.PDF_MANDATORY) + self.check_missing_file(files, package, DepManager.PDF_MANDATORY) - self.check_missing(progs) + self.check_program("dvipng", DepManager.PDF_MANDATORY) - if not self.need and not self.optional: + if self.check_missing(progs): return if self.verbose_warn_install: @@ -575,7 +607,7 @@ class SphinxDependencyChecker: # RHEL 8 uses Python 3.6, which is not compatible with # the build system anymore. Suggest Python 3.11 if rel == 8: - self.add_package("python39", DepType.SYSTEM_MANDATORY) + self.deps.add_package("python39", DepManager.SYSTEM_MANDATORY) self.recommend_python = True if self.first_hint: @@ -597,22 +629,18 @@ class SphinxDependencyChecker: "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc", ] - self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepType.PDF_MANDATORY) + self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepManager.PDF_MANDATORY) - self.check_rpm_missing(fedora_tex_pkgs, DepType.PDF_MANDATORY) + self.check_rpm_missing(fedora_tex_pkgs, DepManager.PDF_MANDATORY) - self.check_missing_tex(DepType.PDF_MANDATORY) + self.check_missing_tex(DepManager.PDF_MANDATORY) # There's no texlive-ctex on RHEL 8 repositories. This will # likely affect CJK pdf build only. if not fedora and rel == 8: - if "texlive-ctex" in self.missing: - del self.missing["texlive-ctex"] - - - self.check_missing(progs) + self.deps.del_package("texlive-ctex") - if not self.need and not self.optional: + if self.check_missing(progs): return if self.verbose_warn_install: @@ -659,7 +687,7 @@ class SphinxDependencyChecker: if rel == 15: if not self.which(self.python_cmd): self.recommend_python = True - self.add_package(self.python_cmd, DepType.SYSTEM_MANDATORY) + self.deps.add_package(self.python_cmd, DepManager.SYSTEM_MANDATORY) progs.update({ "python-sphinx": "python311-Sphinx", @@ -681,12 +709,11 @@ class SphinxDependencyChecker: # "Noto Sans CJK SC" on openSUSE if self.pdf: - self.check_rpm_missing(suse_tex_pkgs, DepType.PDF_MANDATORY) + self.check_rpm_missing(suse_tex_pkgs, DepManager.PDF_MANDATORY) if self.pdf: self.check_missing_tex() - self.check_missing(progs) - if not self.need and not self.optional: + if self.check_missing(progs): return if self.verbose_warn_install: @@ -730,13 +757,12 @@ class SphinxDependencyChecker: "/usr/share/fonts/TTF/NotoSans-Regular.ttf", ] - self.check_missing_file(pdf_pkgs, noto_sans, DepType.PDF_MANDATORY) - self.check_rpm_missing(tex_pkgs, DepType.PDF_MANDATORY) - - self.check_missing(progs) + self.check_missing_file(pdf_pkgs, noto_sans, DepManager.PDF_MANDATORY) + self.check_rpm_missing(tex_pkgs, DepManager.PDF_MANDATORY) - if not self.need and not self.optional: + if self.check_missing(progs): return + if self.verbose_warn_install: print("You should run:") print(f"\n\tsudo {packager_cmd} {self.install}") @@ -759,7 +785,7 @@ class SphinxDependencyChecker: ] if self.pdf: - self.check_pacman_missing(archlinux_tex_pkgs, DepType.PDF_MANDATORY) + self.check_pacman_missing(archlinux_tex_pkgs, DepManager.PDF_MANDATORY) self.check_missing_file( ["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], @@ -767,10 +793,9 @@ class SphinxDependencyChecker: 2, ) - self.check_missing(progs) - - if not self.need and not self.optional: + if self.check_missing(progs): return + if self.verbose_warn_install: print("You should run:") print(f"\n\tsudo pacman -S {self.install}") @@ -797,11 +822,9 @@ class SphinxDependencyChecker: ], } for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, DepType.PDF_MANDATORY) - - self.check_missing(progs) + self.check_missing_file(files, package, DepManager.PDF_MANDATORY) - if not self.need and not self.optional: + if self.check_missing(progs): return if self.verbose_warn_install: @@ -1039,16 +1062,18 @@ class SphinxDependencyChecker: print("\n2) As a package with:") - old_need = self.need - old_optional = self.optional - self.missing = {} + old_need = self.deps.need + old_optional = self.deps.optional + self.pdf = False self.optional = 0 self.install = "" old_verbose = self.verbose_warn_install self.verbose_warn_install = 0 - self.add_package("python-sphinx", DepType.PYTHON_MANDATORY) + self.deps.clear_deps() + + self.deps.add_package("python-sphinx", DepManager.PYTHON_MANDATORY) self.check_distros() @@ -1111,7 +1136,7 @@ class SphinxDependencyChecker: print("\n") else: if self.need_sphinx: - self.need += 1 + self.deps.need += 1 # Suggest newer versions if current ones are too old if self.latest_avail_ver and self.latest_avail_ver >= self.min_version: @@ -1190,20 +1215,20 @@ class SphinxDependencyChecker: self.check_python_module("ensurepip") # Check for needed programs/tools - self.check_perl_module("Pod::Usage", DepType.SYSTEM_MANDATORY) + self.check_perl_module("Pod::Usage", DepManager.SYSTEM_MANDATORY) - self.check_program("make", DepType.SYSTEM_MANDATORY) - self.check_program("gcc", DepType.SYSTEM_MANDATORY) + self.check_program("make", DepManager.SYSTEM_MANDATORY) + self.check_program("gcc", DepManager.SYSTEM_MANDATORY) - self.check_program("dot", DepType.SYSTEM_OPTIONAL) - self.check_program("convert", DepType.SYSTEM_OPTIONAL) + self.check_program("dot", DepManager.SYSTEM_OPTIONAL) + self.check_program("convert", DepManager.SYSTEM_OPTIONAL) self.check_python_module("yaml") if self.pdf: - self.check_program("xelatex", DepType.PDF_MANDATORY) - self.check_program("rsvg-convert", DepType.PDF_MANDATORY) - self.check_program("latexmk", DepType.PDF_MANDATORY) + self.check_program("xelatex", DepManager.PDF_MANDATORY) + self.check_program("rsvg-convert", DepManager.PDF_MANDATORY) + self.check_program("latexmk", DepManager.PDF_MANDATORY) # Do distro-specific checks and output distro-install commands self.check_distros() -- cgit From 1e9ba3b6d4cef8b1b87d6226ab7026c9745cc1d7 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:44 +0200 Subject: scripts: sphinx-pre-install: move ancillary checkers to a separate class The code there are just a bunch of static functions that are used by the main class. group them altogether to better organize the code. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/e2671eb14fae7a8510f5305ac44ad8063e237a5f.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 160 +++++++++++++++++++++--------------------- 1 file changed, 81 insertions(+), 79 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 47dce1fcddfb..b00e50028f4d 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -150,85 +150,11 @@ class DepManager: if output_msg: print(f"\n{output_msg}\n") -class SphinxDependencyChecker: - # List of required texlive packages on Fedora and OpenSuse - texlive = { - "amsfonts.sty": "texlive-amsfonts", - "amsmath.sty": "texlive-amsmath", - "amssymb.sty": "texlive-amsfonts", - "amsthm.sty": "texlive-amscls", - "anyfontsize.sty": "texlive-anyfontsize", - "atbegshi.sty": "texlive-oberdiek", - "bm.sty": "texlive-tools", - "capt-of.sty": "texlive-capt-of", - "cmap.sty": "texlive-cmap", - "ctexhook.sty": "texlive-ctex", - "ecrm1000.tfm": "texlive-ec", - "eqparbox.sty": "texlive-eqparbox", - "eu1enc.def": "texlive-euenc", - "fancybox.sty": "texlive-fancybox", - "fancyvrb.sty": "texlive-fancyvrb", - "float.sty": "texlive-float", - "fncychap.sty": "texlive-fncychap", - "footnote.sty": "texlive-mdwtools", - "framed.sty": "texlive-framed", - "luatex85.sty": "texlive-luatex85", - "multirow.sty": "texlive-multirow", - "needspace.sty": "texlive-needspace", - "palatino.sty": "texlive-psnfss", - "parskip.sty": "texlive-parskip", - "polyglossia.sty": "texlive-polyglossia", - "tabulary.sty": "texlive-tabulary", - "threeparttable.sty": "texlive-threeparttable", - "titlesec.sty": "texlive-titlesec", - "ucs.sty": "texlive-ucs", - "upquote.sty": "texlive-upquote", - "wrapfig.sty": "texlive-wrapfig", - } - - def __init__(self, args): - self.pdf = args.pdf - self.virtualenv = args.virtualenv - self.version_check = args.version_check - - self.deps = DepManager(self.pdf) - - self.need_symlink = 0 - self.need_sphinx = 0 - self.need_pip = 0 - self.rec_sphinx_upgrade = 0 - self.verbose_warn_install = 1 - - self.system_release = "" - self.install = "" - self.virtenv_dir = "" - self.python_cmd = "" - self.activate_cmd = "" - - # Some distros may not have a Sphinx shipped package compatible with - # our minimal requirements - self.package_supported = True - - # Recommend a new python version - self.recommend_python = None - - # Certain hints are meant to be shown only once - self.first_hint = True - - self.min_version = (0, 0, 0) - self.cur_version = (0, 0, 0) - self.latest_avail_ver = (0, 0, 0) - self.venv_ver = (0, 0, 0) - - prefix = os.environ.get("srctree", ".") + "/" - - self.conf = prefix + "Documentation/conf.py" - self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" - self.virtenv_prefix = ["sphinx_", "Sphinx_" ] - - # - # Ancillary methods that don't depend on self - # +class AncillaryCheckers: + """ + Ancillary methods that checks for missing dependencies for different + types of types, like binaries, python modules, rpm deps, etc. + """ @staticmethod def which(prog): @@ -335,6 +261,82 @@ class SphinxDependencyChecker: return subprocess.run(*args, **kwargs) +class SphinxDependencyChecker(AncillaryCheckers): + # List of required texlive packages on Fedora and OpenSuse + texlive = { + "amsfonts.sty": "texlive-amsfonts", + "amsmath.sty": "texlive-amsmath", + "amssymb.sty": "texlive-amsfonts", + "amsthm.sty": "texlive-amscls", + "anyfontsize.sty": "texlive-anyfontsize", + "atbegshi.sty": "texlive-oberdiek", + "bm.sty": "texlive-tools", + "capt-of.sty": "texlive-capt-of", + "cmap.sty": "texlive-cmap", + "ctexhook.sty": "texlive-ctex", + "ecrm1000.tfm": "texlive-ec", + "eqparbox.sty": "texlive-eqparbox", + "eu1enc.def": "texlive-euenc", + "fancybox.sty": "texlive-fancybox", + "fancyvrb.sty": "texlive-fancyvrb", + "float.sty": "texlive-float", + "fncychap.sty": "texlive-fncychap", + "footnote.sty": "texlive-mdwtools", + "framed.sty": "texlive-framed", + "luatex85.sty": "texlive-luatex85", + "multirow.sty": "texlive-multirow", + "needspace.sty": "texlive-needspace", + "palatino.sty": "texlive-psnfss", + "parskip.sty": "texlive-parskip", + "polyglossia.sty": "texlive-polyglossia", + "tabulary.sty": "texlive-tabulary", + "threeparttable.sty": "texlive-threeparttable", + "titlesec.sty": "texlive-titlesec", + "ucs.sty": "texlive-ucs", + "upquote.sty": "texlive-upquote", + "wrapfig.sty": "texlive-wrapfig", + } + + def __init__(self, args): + self.pdf = args.pdf + self.virtualenv = args.virtualenv + self.version_check = args.version_check + + self.deps = DepManager(self.pdf) + + self.need_symlink = 0 + self.need_sphinx = 0 + self.need_pip = 0 + self.rec_sphinx_upgrade = 0 + self.verbose_warn_install = 1 + + self.system_release = "" + self.install = "" + self.virtenv_dir = "" + self.python_cmd = "" + self.activate_cmd = "" + + # Some distros may not have a Sphinx shipped package compatible with + # our minimal requirements + self.package_supported = True + + # Recommend a new python version + self.recommend_python = None + + # Certain hints are meant to be shown only once + self.first_hint = True + + self.min_version = (0, 0, 0) + self.cur_version = (0, 0, 0) + self.latest_avail_ver = (0, 0, 0) + self.venv_ver = (0, 0, 0) + + prefix = os.environ.get("srctree", ".") + "/" + + self.conf = prefix + "Documentation/conf.py" + self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" + self.virtenv_prefix = ["sphinx_", "Sphinx_" ] + # # Methods to check if a feature exists # -- cgit From 9bb5f0dc18d037635f4c5075747de1e47493b538 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:45 +0200 Subject: scripts: sphinx-pre-install: add more generic checkers on a class Better organize the code by moving the more generic methods to MissingCheckers. Such class contain only binary and package dependent missing checkers, but no distro-specific data or code. All distro-specific data/code remains at SphinxDependencyChecker class. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/11a252fe816bd7c85583d26ade0666eb2b481bf0.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 142 ++++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 66 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index b00e50028f4d..9127487bd4d7 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -150,7 +150,7 @@ class DepManager: if output_msg: print(f"\n{output_msg}\n") -class AncillaryCheckers: +class AncillaryMethods: """ Ancillary methods that checks for missing dependencies for different types of types, like binaries, python modules, rpm deps, etc. @@ -261,81 +261,24 @@ class AncillaryCheckers: return subprocess.run(*args, **kwargs) -class SphinxDependencyChecker(AncillaryCheckers): - # List of required texlive packages on Fedora and OpenSuse - texlive = { - "amsfonts.sty": "texlive-amsfonts", - "amsmath.sty": "texlive-amsmath", - "amssymb.sty": "texlive-amsfonts", - "amsthm.sty": "texlive-amscls", - "anyfontsize.sty": "texlive-anyfontsize", - "atbegshi.sty": "texlive-oberdiek", - "bm.sty": "texlive-tools", - "capt-of.sty": "texlive-capt-of", - "cmap.sty": "texlive-cmap", - "ctexhook.sty": "texlive-ctex", - "ecrm1000.tfm": "texlive-ec", - "eqparbox.sty": "texlive-eqparbox", - "eu1enc.def": "texlive-euenc", - "fancybox.sty": "texlive-fancybox", - "fancyvrb.sty": "texlive-fancyvrb", - "float.sty": "texlive-float", - "fncychap.sty": "texlive-fncychap", - "footnote.sty": "texlive-mdwtools", - "framed.sty": "texlive-framed", - "luatex85.sty": "texlive-luatex85", - "multirow.sty": "texlive-multirow", - "needspace.sty": "texlive-needspace", - "palatino.sty": "texlive-psnfss", - "parskip.sty": "texlive-parskip", - "polyglossia.sty": "texlive-polyglossia", - "tabulary.sty": "texlive-tabulary", - "threeparttable.sty": "texlive-threeparttable", - "titlesec.sty": "texlive-titlesec", - "ucs.sty": "texlive-ucs", - "upquote.sty": "texlive-upquote", - "wrapfig.sty": "texlive-wrapfig", - } +class MissingCheckers(AncillaryMethods): - def __init__(self, args): + def __init__(self, args, texlive): self.pdf = args.pdf self.virtualenv = args.virtualenv self.version_check = args.version_check + self.texlive = texlive self.deps = DepManager(self.pdf) self.need_symlink = 0 self.need_sphinx = 0 - self.need_pip = 0 - self.rec_sphinx_upgrade = 0 + self.verbose_warn_install = 1 - self.system_release = "" - self.install = "" self.virtenv_dir = "" - self.python_cmd = "" - self.activate_cmd = "" - # Some distros may not have a Sphinx shipped package compatible with - # our minimal requirements - self.package_supported = True - - # Recommend a new python version - self.recommend_python = None - - # Certain hints are meant to be shown only once - self.first_hint = True - - self.min_version = (0, 0, 0) - self.cur_version = (0, 0, 0) - self.latest_avail_ver = (0, 0, 0) - self.venv_ver = (0, 0, 0) - - prefix = os.environ.get("srctree", ".") + "/" - - self.conf = prefix + "Documentation/conf.py" - self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" - self.virtenv_prefix = ["sphinx_", "Sphinx_" ] + self.install = "" # # Methods to check if a feature exists @@ -460,9 +403,9 @@ class SphinxDependencyChecker(AncillaryCheckers): if match: return parse_version(match.group(1)) - def check_sphinx(self): + def check_sphinx(self, conf): try: - with open(self.conf, "r", encoding="utf-8") as f: + with open(conf, "r", encoding="utf-8") as f: for line in f: match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line) if match: @@ -513,6 +456,73 @@ class SphinxDependencyChecker(AncillaryCheckers): return True +class SphinxDependencyChecker(MissingCheckers): + + def __init__(self, args): + # List of required texlive packages on Fedora and OpenSuse + texlive = { + "amsfonts.sty": "texlive-amsfonts", + "amsmath.sty": "texlive-amsmath", + "amssymb.sty": "texlive-amsfonts", + "amsthm.sty": "texlive-amscls", + "anyfontsize.sty": "texlive-anyfontsize", + "atbegshi.sty": "texlive-oberdiek", + "bm.sty": "texlive-tools", + "capt-of.sty": "texlive-capt-of", + "cmap.sty": "texlive-cmap", + "ctexhook.sty": "texlive-ctex", + "ecrm1000.tfm": "texlive-ec", + "eqparbox.sty": "texlive-eqparbox", + "eu1enc.def": "texlive-euenc", + "fancybox.sty": "texlive-fancybox", + "fancyvrb.sty": "texlive-fancyvrb", + "float.sty": "texlive-float", + "fncychap.sty": "texlive-fncychap", + "footnote.sty": "texlive-mdwtools", + "framed.sty": "texlive-framed", + "luatex85.sty": "texlive-luatex85", + "multirow.sty": "texlive-multirow", + "needspace.sty": "texlive-needspace", + "palatino.sty": "texlive-psnfss", + "parskip.sty": "texlive-parskip", + "polyglossia.sty": "texlive-polyglossia", + "tabulary.sty": "texlive-tabulary", + "threeparttable.sty": "texlive-threeparttable", + "titlesec.sty": "texlive-titlesec", + "ucs.sty": "texlive-ucs", + "upquote.sty": "texlive-upquote", + "wrapfig.sty": "texlive-wrapfig", + } + + super().__init__(args, texlive) + + self.need_pip = 0 + self.rec_sphinx_upgrade = 0 + + self.system_release = "" + self.python_cmd = "" + self.activate_cmd = "" + + # Some distros may not have a Sphinx shipped package compatible with + # our minimal requirements + self.package_supported = True + + # Recommend a new python version + self.recommend_python = None + + # Certain hints are meant to be shown only once + self.first_hint = True + + self.min_version = (0, 0, 0) + self.cur_version = (0, 0, 0) + self.latest_avail_ver = (0, 0, 0) + self.venv_ver = (0, 0, 0) + + prefix = os.environ.get("srctree", ".") + "/" + + self.conf = prefix + "Documentation/conf.py" + self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" + self.virtenv_prefix = ["sphinx_", "Sphinx_" ] # # Distro-specific hints methods @@ -1187,7 +1197,7 @@ class SphinxDependencyChecker(AncillaryCheckers): self.python_cmd = sys.executable # Check if Sphinx is already accessible from current environment - self.check_sphinx() + self.check_sphinx(self.conf) if self.system_release: print(f"Detected OS: {self.system_release}.") -- cgit From fb22e438b23eabd828a0ca1076be5ecf9b0262db Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:46 +0200 Subject: scripts: sphinx-pre-install: move get_system_release() The code at get_system_release() is actually a helper function, independent from the actual Sphinx verification checker. Move it to MissingCheckers class, where other checkers are present. With that, the entire distro-specific handler logic, with all its complexity is confined at SphinxDependencyChecker class. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/4b42a85bbb6575bb34a58cf66019038c4afa1d5b.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 115 ++++++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 56 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 9127487bd4d7..593982f350b3 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -456,6 +456,64 @@ class MissingCheckers(AncillaryMethods): return True + def get_system_release(self): + """ + Determine the system type. There's no unique way that would work + with all distros with a minimal package install. So, several + methods are used here. + + By default, it will use lsb_release function. If not available, it will + fail back to reading the known different places where the distro name + is stored. + + Several modern distros now have /etc/os-release, which usually have + a decent coverage. + """ + + system_release = "" + + if self.which("lsb_release"): + result = self.run(["lsb_release", "-d"], capture_output=True, text=True) + system_release = result.stdout.replace("Description:", "").strip() + + release_files = [ + "/etc/system-release", + "/etc/redhat-release", + "/etc/lsb-release", + "/etc/gentoo-release", + ] + + if not system_release: + for f in release_files: + system_release = self.catcheck(f) + if system_release: + break + + # This seems more common than LSB these days + if not system_release: + os_var = {} + try: + with open("/etc/os-release", "r", encoding="utf-8") as f: + for line in f: + match = re.match(r"^([\w\d\_]+)=\"?([^\"]*)\"?\n", line) + if match: + os_var[match.group(1)] = match.group(2) + + system_release = os_var.get("NAME", "") + if "VERSION_ID" in os_var: + system_release += " " + os_var["VERSION_ID"] + elif "VERSION" in os_var: + system_release += " " + os_var["VERSION"] + except IOError: + pass + + if not system_release: + system_release = self.catcheck("/etc/issue") + + system_release = system_release.strip() + + return system_release + class SphinxDependencyChecker(MissingCheckers): def __init__(self, args): @@ -499,7 +557,7 @@ class SphinxDependencyChecker(MissingCheckers): self.need_pip = 0 self.rec_sphinx_upgrade = 0 - self.system_release = "" + self.system_release = self.get_system_release() self.python_cmd = "" self.activate_cmd = "" @@ -1193,7 +1251,6 @@ class SphinxDependencyChecker(MissingCheckers): "\thttps://github.com/sphinx-doc/sphinx/pull/8313") def check_needs(self): - self.get_system_release() self.python_cmd = sys.executable # Check if Sphinx is already accessible from current environment @@ -1270,60 +1327,6 @@ class SphinxDependencyChecker(MissingCheckers): print("Needed package dependencies are met.") - def get_system_release(self): - """ - Determine the system type. There's no unique way that would work - with all distros with a minimal package install. So, several - methods are used here. - - By default, it will use lsb_release function. If not available, it will - fail back to reading the known different places where the distro name - is stored. - - Several modern distros now have /etc/os-release, which usually have - a decent coverage. - """ - - if self.which("lsb_release"): - result = self.run(["lsb_release", "-d"], capture_output=True, text=True) - self.system_release = result.stdout.replace("Description:", "").strip() - - release_files = [ - "/etc/system-release", - "/etc/redhat-release", - "/etc/lsb-release", - "/etc/gentoo-release", - ] - - if not self.system_release: - for f in release_files: - self.system_release = self.catcheck(f) - if self.system_release: - break - - # This seems more common than LSB these days - if not self.system_release: - os_var = {} - try: - with open("/etc/os-release", "r", encoding="utf-8") as f: - for line in f: - match = re.match(r"^([\w\d\_]+)=\"?([^\"]*)\"?\n", line) - if match: - os_var[match.group(1)] = match.group(2) - - self.system_release = os_var.get("NAME", "") - if "VERSION_ID" in os_var: - self.system_release += " " + os_var["VERSION_ID"] - elif "VERSION" in os_var: - self.system_release += " " + os_var["VERSION"] - except IOError: - pass - - if not self.system_release: - self.system_release = self.catcheck("/etc/issue") - - self.system_release = self.system_release.strip() - DESCRIPTION = """ Process some flags related to Sphinx installation and documentation build. """ -- cgit From f477c6d71d3947ab8f2e9e7d5fd8448f7a26c1ab Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:47 +0200 Subject: scripts: sphinx-pre-install: add documentation for the ancillary classes. While here, rename a parameter to have its usage better documented. No functional changes. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/7421112b14edf5c21cc4cf0f2ee320fcaf874b40.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 162 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 140 insertions(+), 22 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 593982f350b3..1dc3f19804ab 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -33,35 +33,62 @@ MIN_PYTHON_VERSION = parse_version("3.7") class DepManager: + """ + Manage package dependencies. There are three types of dependencies: + + - System: dependencies required for docs build; + - Python: python dependencies for a native distro Sphinx install; + - PDF: dependencies needed by PDF builds. + + Each dependency can be mandatory or optional. Not installing an optional + dependency won't break the build, but will cause degradation at the + docs output. + """ - # Internal types of dependencies. + # Internal types of dependencies. Don't use them outside DepManager class. _SYS_TYPE = 0 _PHY_TYPE = 1 _PDF_TYPE = 2 - # Let's define keys as a tuple with the type and mandatory/optional. - # This way, checking for optional or type is easy. + # Dependencies visible outside the class. + # The keys are tuple with: (type, is_mandatory flag). + # + # Currently we're not using all optional dep types. Yet, we'll keep all + # possible combinations here. They're not many, and that makes easier + # if later needed and for the name() method below SYSTEM_MANDATORY = (_SYS_TYPE, True) PYTHON_MANDATORY = (_PHY_TYPE, True) PDF_MANDATORY = (_PDF_TYPE, True) - # Currently we're not using all optional types, but let's keep all - # combinations here, as we may end needing them in the future. Also, - # it allows a name() function that handles all possibilities. SYSTEM_OPTIONAL = (_SYS_TYPE, False) PYTHON_OPTIONAL = (_PHY_TYPE, False) PDF_OPTIONAL = (_PDF_TYPE, True) def __init__(self, pdf): + """ + Initialize internal vars: + + - missing: missing dependencies list, containing a distro-independent + name for a missing dependency and its type. + - missing_pkg: ancillary dict containing missing dependencies in + distro namespace, organized by type. + - need: total number of needed dependencies. Never cleaned. + - optional: total number of optional dependencies. Never cleaned. + - pdf: PDF support is enabled? + """ self.missing = {} - self.run = {} + self.missing_pkg = {} self.need = 0 self.optional = 0 self.pdf = pdf @staticmethod def name(dtype): + """ + Ancillary routine to output a warn/error message reporting + missing dependencies. + """ if dtype[0] == DepManager._SYS_TYPE: msg = "build" elif dtype[0] == DepManager._PHY_TYPE: @@ -76,16 +103,22 @@ class DepManager: @staticmethod def is_optional(dtype): + """Ancillary routine to report if a dependency is optional""" return not dtype[1] @staticmethod def is_pdf(dtype): + """Ancillary routine to report if a dependency is for PDF generation""" if (dtype[0] == DepManager._PDF_TYPE): return True return False def add_package(self, package, dtype): + """ + Add a package at the self.missing() dictionary. + Doesn't update missing_pkg. + """ is_optional = DepManager.is_optional(dtype) self.missing[package] = dtype if is_optional: @@ -94,6 +127,10 @@ class DepManager: self.need += 1 def del_package(self, package): + """ + Remove a package at the self.missing() dictionary. + Doesn't update missing_pkg. + """ if package in self.missing: del self.missing[package] @@ -104,13 +141,22 @@ class DepManager: This is an ackward way to have a separate section to recommend a package after system main dependencies. - TODO: rework the logic to prevent needing it + TODO: rework the logic to prevent needing it. """ self.missing = {} + self.missing_pkg = {} def check_missing(self, progs): - self.run = {} + """ + Update self.missing_pkg, using progs dict to convert from the + agnostic package name to distro-specific one. + + Returns an string with the packages to be installed, sorted and + with eventual duplicates removed. + """ + + self.missing_pkg = {} for prog, dtype in sorted(self.missing.items()): # At least on some LTS distros like CentOS 7, texlive doesn't @@ -123,23 +169,26 @@ class DepManager: self.optional -= 1 continue - if not dtype in self.run: - self.run[dtype] = [] + if not dtype in self.missing_pkg: + self.missing_pkg[dtype] = [] - self.run[dtype].append(progs.get(prog, prog)) + self.missing_pkg[dtype].append(progs.get(prog, prog)) install = [] - for dtype in self.run.keys(): - install += self.run[dtype] + for dtype in self.missing_pkg.keys(): + install += self.missing_pkg[dtype] return " ".join(sorted(set(install))) def warn_install(self): + """ + Emit warnings/errors related to missing packages. + """ output_msg = "" - for dtype in sorted(self.run.keys()): - progs = " ".join(sorted(set(self.run[dtype]))) + for dtype in sorted(self.missing_pkg.keys()): + progs = " ".join(sorted(set(self.missing_pkg[dtype]))) try: name = DepManager.name(dtype) @@ -158,6 +207,11 @@ class AncillaryMethods: @staticmethod def which(prog): + """ + Our own implementation of which(). We could instead use + shutil.which(), but this function is simple enough. + Probably faster to use this implementation than to import shutil. + """ for path in os.environ.get("PATH", "").split(":"): full_path = os.path.join(path, prog) if os.access(full_path, os.X_OK): @@ -167,6 +221,10 @@ class AncillaryMethods: @staticmethod def get_python_version(cmd): + """ + Get python version from a Python binary. As we need to detect if + are out there newer python binaries, we can't rely on sys.release here. + """ result = SphinxDependencyChecker.run([cmd, "--version"], capture_output=True, text=True) @@ -181,7 +239,13 @@ class AncillaryMethods: @staticmethod def find_python(): + """ + Detect if are out there any python 3.xy version newer than the + current one. + Note: this routine is limited to up to 2 digits for python3. We + may need to update it one day, hopefully on a distant future. + """ patterns = [ "python3.[0-9]", "python3.[0-9][0-9]", @@ -200,7 +264,10 @@ class AncillaryMethods: @staticmethod def check_python(): - + """ + Check if the current python binary satisfies our minimal requirement + for Sphinx build. If not, re-run with a newer version if found. + """ cur_ver = sys.version_info[:3] if cur_ver >= MIN_PYTHON_VERSION: ver = ver_str(cur_ver) @@ -240,7 +307,10 @@ class AncillaryMethods: @staticmethod def run(*args, **kwargs): - """Excecute a command, hiding its output by default""" + """ + Excecute a command, hiding its output by default. + Preserve comatibility with older Python versions. + """ capture_output = kwargs.pop('capture_output', False) @@ -262,8 +332,15 @@ class AncillaryMethods: return subprocess.run(*args, **kwargs) class MissingCheckers(AncillaryMethods): + """ + Contains some ancillary checkers for different types of binaries and + package managers. + """ def __init__(self, args, texlive): + """ + Initialize its internal variables + """ self.pdf = args.pdf self.virtualenv = args.virtualenv self.version_check = args.version_check @@ -280,17 +357,20 @@ class MissingCheckers(AncillaryMethods): self.install = "" - # - # Methods to check if a feature exists - # - def check_missing_file(self, files, package, dtype): + """ + Does the file exists? If not, add it to missing dependencies. + """ for f in files: if os.path.exists(f): return self.deps.add_package(package, dtype) def check_program(self, prog, dtype): + """ + Does the program exists and it is at the PATH? + If not, add it to missing dependencies. + """ found = self.which(prog) if found: return found @@ -300,6 +380,18 @@ class MissingCheckers(AncillaryMethods): return None def check_perl_module(self, prog, dtype): + """ + Does perl have a dependency? Is it available? + If not, add it to missing dependencies. + + Right now, we still need Perl for doc build, as it is required + by some tools called at docs or kernel build time, like: + + scripts/documentation-file-ref-check + + Also, checkpatch is on Perl. + """ + # While testing with lxc download template, one of the # distros (Oracle) didn't have perl - nor even an option to install # before installing oraclelinux-release-el9 package. @@ -318,6 +410,10 @@ class MissingCheckers(AncillaryMethods): self.deps.add_package(prog, dtype) def check_python_module(self, module, is_optional=False): + """ + Does a python module exists outside venv? If not, add it to missing + dependencies. + """ if is_optional: dtype = DepManager.PYTHON_OPTIONAL else: @@ -329,6 +425,9 @@ class MissingCheckers(AncillaryMethods): self.deps.add_package(module, dtype) def check_rpm_missing(self, pkgs, dtype): + """ + Does a rpm package exists? If not, add it to missing dependencies. + """ for prog in pkgs: try: self.run(["rpm", "-q", prog], check=True) @@ -336,6 +435,9 @@ class MissingCheckers(AncillaryMethods): self.deps.add_package(prog, dtype) def check_pacman_missing(self, pkgs, dtype): + """ + Does a pacman package exists? If not, add it to missing dependencies. + """ for prog in pkgs: try: self.run(["pacman", "-Q", prog], check=True) @@ -343,6 +445,9 @@ class MissingCheckers(AncillaryMethods): self.deps.add_package(prog, dtype) def check_missing_tex(self, is_optional=False): + """ + Does a LaTeX package exists? If not, add it to missing dependencies. + """ if is_optional: dtype = DepManager.PDF_OPTIONAL else: @@ -371,6 +476,9 @@ class MissingCheckers(AncillaryMethods): self.deps.add_package(package, dtype) def get_sphinx_fname(self): + """ + Gets the binary filename for sphinx-build. + """ if "SPHINXBUILD" in os.environ: return os.environ["SPHINXBUILD"] @@ -386,6 +494,9 @@ class MissingCheckers(AncillaryMethods): return "" def get_sphinx_version(self, cmd): + """ + Gets sphinx-build version. + """ try: result = self.run([cmd, "--version"], stdout=subprocess.PIPE, @@ -404,6 +515,9 @@ class MissingCheckers(AncillaryMethods): return parse_version(match.group(1)) def check_sphinx(self, conf): + """ + Checks Sphinx minimal requirements + """ try: with open(conf, "r", encoding="utf-8") as f: for line in f: @@ -441,6 +555,10 @@ class MissingCheckers(AncillaryMethods): sys.exit(0) def catcheck(self, filename): + """ + Reads a file if it exists, returning as string. + If not found, returns an empty string. + """ if os.path.exists(filename): with open(filename, "r", encoding="utf-8") as f: return f.read().strip() -- cgit From 6d5f4f3da1a82786650d38067bb4c5e4ed84dc61 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:48 +0200 Subject: scripts: sphinx-pre-install: add docstring documentation This program is somewhat complex. Add some docstring documentation, explaining what each function and class is supposed to do. Most of the focus here were to describe the ancillary functions used to detect dependency needs. The main SphinxDependencyChecker still requires a lot of care, and probably need to be reorganized to clearly split the 4 types of output it produces: - Need to upgrade Python binary; - System install needs; - Virtual env install needs; - Python install needs via system packages, to run Sphinx natively. Yet, for now, I'm happy of having it a lot better documented than its Perl version. - While here, rename a parameter to have its usage better documented. No functional changes. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0cadab2cab3f78ae6d9f378e92a45125fbc5188f.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 110 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 102 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 1dc3f19804ab..1c96f6692e9a 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -9,6 +9,22 @@ # Don't add changes not compatible with it, it is meant to report # incompatible python versions. +""" +Dependency checker for Sphinx documentation Kernel build. + +This module provides tools to check for all required dependencies needed to +build documentation using Sphinx, including system packages, Python modules +and LaTeX packages for PDF generation. + +It detect packages for a subset of Linux distributions used by Kernel +maintainers, showing hints and missing dependencies. + +The main class SphinxDependencyChecker handles the dependency checking logic +and provides recommendations for installing missing packages. It supports both +system package installations and Python virtual environments. By default, +system pacage install is recommended. +""" + import argparse import os import re @@ -75,7 +91,7 @@ class DepManager: distro namespace, organized by type. - need: total number of needed dependencies. Never cleaned. - optional: total number of optional dependencies. Never cleaned. - - pdf: PDF support is enabled? + - pdf: Is PDF support enabled? """ self.missing = {} self.missing_pkg = {} @@ -565,6 +581,13 @@ class MissingCheckers(AncillaryMethods): return "" def check_missing(self, progs): + """ + Check for missing dependencies using the provided program mapping. + + The actual distro-specific programs are mapped via progs argument. + + Returns True if there are missing dependencies, False otherwise. + """ self.install += self.deps.check_missing(progs) if self.verbose_warn_install: self.deps.warn_install() @@ -633,8 +656,18 @@ class MissingCheckers(AncillaryMethods): return system_release class SphinxDependencyChecker(MissingCheckers): + """ + Main class for checking Sphinx documentation build dependencies. + - Check for missing system packages; + - Check for missing Python modules; + - Check for missing LaTeX packages needed by PDF generation; + - Propose Sphinx install via Python Virtual environment; + - Propose Sphinx install via distro-specific package install. + """ def __init__(self, args): + """Initialize checker variables""" + # List of required texlive packages on Fedora and OpenSuse texlive = { "amsfonts.sty": "texlive-amsfonts", @@ -705,6 +738,9 @@ class SphinxDependencyChecker(MissingCheckers): # def give_debian_hints(self): + """ + Provide package installation hints for Debian-based distros. + """ progs = { "Pod::Usage": "perl-modules", "convert": "imagemagick", @@ -745,6 +781,10 @@ class SphinxDependencyChecker(MissingCheckers): print(f"\n\tsudo apt-get install {self.install}") def give_redhat_hints(self): + """ + Provide package installation hints for RedHat-based distros + (Fedora, RHEL and RHEL-based variants). + """ progs = { "Pod::Usage": "perl-Pod-Usage", "convert": "ImageMagick", @@ -837,6 +877,10 @@ class SphinxDependencyChecker(MissingCheckers): print(f"\n\tsudo dnf install -y {self.install}") def give_opensuse_hints(self): + """ + Provide package installation hints for openSUSE-based distros + (Leap and Tumbleweed). + """ progs = { "Pod::Usage": "perl-Pod-Usage", "convert": "ImageMagick", @@ -909,6 +953,9 @@ class SphinxDependencyChecker(MissingCheckers): print(f"\n\tsudo zypper install --no-recommends {self.install}") def give_mageia_hints(self): + """ + Provide package installation hints for Mageia and OpenMandriva. + """ progs = { "Pod::Usage": "perl-Pod-Usage", "convert": "ImageMagick", @@ -956,6 +1003,9 @@ class SphinxDependencyChecker(MissingCheckers): print(f"\n\tsudo {packager_cmd} {self.install}") def give_arch_linux_hints(self): + """ + Provide package installation hints for ArchLinux. + """ progs = { "convert": "imagemagick", "dot": "graphviz", @@ -989,6 +1039,9 @@ class SphinxDependencyChecker(MissingCheckers): print(f"\n\tsudo pacman -S {self.install}") def give_gentoo_hints(self): + """ + Provide package installation hints for Gentoo. + """ progs = { "convert": "media-gfx/imagemagick", "dot": "media-gfx/graphviz", @@ -1107,7 +1160,12 @@ class SphinxDependencyChecker(MissingCheckers): # def check_distros(self): - # OS-specific hints logic + """ + OS-specific hints logic. Seeks for a hinter. If found, provide + package-manager-specific install commands. + + Otherwise, just lists the missing dependencies. + """ os_hints = { re.compile("Red Hat Enterprise Linux"): self.give_redhat_hints, re.compile("Fedora"): self.give_redhat_hints, @@ -1159,10 +1217,22 @@ class SphinxDependencyChecker(MissingCheckers): # Common dependencies # def deactivate_help(self): + """ + Print a helper message to disable a virtual environment. + """ + print("\n If you want to exit the virtualenv, you can use:") print("\tdeactivate") def get_virtenv(self): + """ + Give a hint about how to activate an already-existing virtual + environment containing sphinx-build. + + Returns a tuble with (activate_cmd_path, sphinx_version) with + the newest available virtual env. + """ + cwd = os.getcwd() activates = [] @@ -1207,6 +1277,14 @@ class SphinxDependencyChecker(MissingCheckers): return ("", ver) def recommend_sphinx_upgrade(self): + """ + Check if Sphinx needs to be upgraded. + + Returns a tuple with the higest available Sphinx version if found. + Otherwise, returns None to indicate either that no upgrade is needed + or no venv was found. + """ + # Avoid running sphinx-builds from venv if cur_version is good if self.cur_version and self.cur_version >= RECOMMENDED_VERSION: self.latest_avail_ver = self.cur_version @@ -1247,6 +1325,9 @@ class SphinxDependencyChecker(MissingCheckers): return self.latest_avail_ver def recommend_package(self): + """ + Recommend installing Sphinx as a distro-specific package. + """ print("\n2) As a package with:") @@ -1270,11 +1351,19 @@ class SphinxDependencyChecker(MissingCheckers): self.verbose_warn_install = old_verbose def recommend_sphinx_version(self, virtualenv_cmd): - # The logic here is complex, as it have to deal with different versions: - # - minimal supported version; - # - minimal PDF version; - # - recommended version. - # It also needs to work fine with both distro's package and venv/virtualenv + """ + Provide recommendations for installing or upgrading Sphinx based + on current version. + + The logic here is complex, as it have to deal with different versions: + + - minimal supported version; + - minimal PDF version; + - recommended version. + + It also needs to work fine with both distro's package and + venv/virtualenv + """ if self.recommend_python: print("\nPython version is incompatible with doc build.\n" \ @@ -1369,6 +1458,10 @@ class SphinxDependencyChecker(MissingCheckers): "\thttps://github.com/sphinx-doc/sphinx/pull/8313") def check_needs(self): + """ + Main method that checks needed dependencies and provides + recommendations. + """ self.python_cmd = sys.executable # Check if Sphinx is already accessible from current environment @@ -1451,6 +1544,7 @@ Process some flags related to Sphinx installation and documentation build. def main(): + """Main function""" parser = argparse.ArgumentParser(description=DESCRIPTION) parser.add_argument( @@ -1481,6 +1575,6 @@ def main(): checker.check_python() checker.check_needs() - +# Call main if not used as module if __name__ == "__main__": main() -- cgit From 8b45effaa2ce2790e3b391e1ac4bb668e0d37560 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:49 +0200 Subject: scripts: sphinx-pre-install: fix several codingstyle issues Address most pylint issues. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/5139b18535e1436e4b1773706224a9ec3a386697.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 52 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 1c96f6692e9a..1b11162da9fb 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (c) 2017-2025 Mauro Carvalho Chehab # -# pylint: disable=C0103,C0114,C0115,C0116,C0301 -# pylint: disable=R0902,R0904,R0912,R0915,R1705,R1710,E1121 +# pylint: disable=C0103,C0114,C0115,C0116,C0301,C0302 +# pylint: disable=R0902,R0904,R0911,R0912,R0914,R0915,R1705,R1710,E1121 # Note: this script requires at least Python 3.6 to run. # Don't add changes not compatible with it, it is meant to report @@ -115,7 +115,7 @@ class DepManager: if dtype[1]: return f"ERROR: {msg} mandatory deps missing" else: - out = f"Warning: {msg} optional deps missing" + return f"Warning: {msg} optional deps missing" @staticmethod def is_optional(dtype): @@ -125,7 +125,7 @@ class DepManager: @staticmethod def is_pdf(dtype): """Ancillary routine to report if a dependency is for PDF generation""" - if (dtype[0] == DepManager._PDF_TYPE): + if dtype[0] == DepManager._PDF_TYPE: return True return False @@ -191,8 +191,8 @@ class DepManager: self.missing_pkg[dtype].append(progs.get(prog, prog)) install = [] - for dtype in self.missing_pkg.keys(): - install += self.missing_pkg[dtype] + for dtype, pkgs in self.missing_pkg.items(): + install += pkgs return " ".join(sorted(set(install))) @@ -267,8 +267,6 @@ class AncillaryMethods: "python3.[0-9][0-9]", ] - new_python_cmd = None - # Seek for a python binary newer than MIN_PYTHON_VERSION for path in os.getenv("PATH", "").split(":"): for pattern in patterns: @@ -276,7 +274,7 @@ class AncillaryMethods: if os.path.isfile(cmd) and os.access(cmd, os.X_OK): version = SphinxDependencyChecker.get_python_version(cmd) if version >= MIN_PYTHON_VERSION: - return(cmd) + return cmd @staticmethod def check_python(): @@ -306,8 +304,8 @@ class AncillaryMethods: new_python_cmd = SphinxDependencyChecker.find_python() if not new_python_cmd: - print(f"ERROR: Python version {python_ver} is not spported anymore") - print(f" Can't find a new version. This script may fail") + print(f"ERROR: Python version {python_ver} is not spported anymore\n") + print(" Can't find a new version. This script may fail") return # Restart script using the newer version @@ -362,6 +360,9 @@ class MissingCheckers(AncillaryMethods): self.version_check = args.version_check self.texlive = texlive + self.min_version = (0, 0, 0) + self.cur_version = (0, 0, 0) + self.deps = DepManager(self.pdf) self.need_symlink = 0 @@ -370,8 +371,10 @@ class MissingCheckers(AncillaryMethods): self.verbose_warn_install = 1 self.virtenv_dir = "" - self.install = "" + self.python_cmd = "" + + self.virtenv_prefix = ["sphinx_", "Sphinx_" ] def check_missing_file(self, files, package, dtype): """ @@ -542,10 +545,10 @@ class MissingCheckers(AncillaryMethods): self.min_version = parse_version(match.group(1)) break except IOError: - sys.exit(f"Can't open {self.conf}") + sys.exit(f"Can't open {conf}") if not self.min_version: - sys.exit(f"Can't get needs_sphinx version from {self.conf}") + sys.exit(f"Can't get needs_sphinx version from {conf}") self.virtenv_dir = self.virtenv_prefix[0] + "latest" @@ -709,7 +712,6 @@ class SphinxDependencyChecker(MissingCheckers): self.rec_sphinx_upgrade = 0 self.system_release = self.get_system_release() - self.python_cmd = "" self.activate_cmd = "" # Some distros may not have a Sphinx shipped package compatible with @@ -722,8 +724,6 @@ class SphinxDependencyChecker(MissingCheckers): # Certain hints are meant to be shown only once self.first_hint = True - self.min_version = (0, 0, 0) - self.cur_version = (0, 0, 0) self.latest_avail_ver = (0, 0, 0) self.venv_ver = (0, 0, 0) @@ -731,7 +731,6 @@ class SphinxDependencyChecker(MissingCheckers): self.conf = prefix + "Documentation/conf.py" self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" - self.virtenv_prefix = ["sphinx_", "Sphinx_" ] # # Distro-specific hints methods @@ -814,6 +813,7 @@ class SphinxDependencyChecker(MissingCheckers): if not rel: print("Couldn't identify release number") + noto_sans_redhat = None self.pdf = False elif re.search("Fedora", self.system_release): # Fedora 38 and upper use this CJK font @@ -1111,7 +1111,7 @@ class SphinxDependencyChecker(MissingCheckers): for fname, portage in portages.items(): install = False - while install == False: + while install is False: if not files: # No files under package.usage. Install all install = True @@ -1335,7 +1335,7 @@ class SphinxDependencyChecker(MissingCheckers): old_optional = self.deps.optional self.pdf = False - self.optional = 0 + self.deps.optional = 0 self.install = "" old_verbose = self.verbose_warn_install self.verbose_warn_install = 0 @@ -1346,8 +1346,8 @@ class SphinxDependencyChecker(MissingCheckers): self.check_distros() - self.need = old_need - self.optional = old_optional + self.deps.need = old_need + self.deps.optional = old_optional self.verbose_warn_install = old_verbose def recommend_sphinx_version(self, virtualenv_cmd): @@ -1528,13 +1528,13 @@ class SphinxDependencyChecker(MissingCheckers): self.recommend_sphinx_version(virtualenv_cmd) print("") - if not self.optional: + if not self.deps.optional: print("All optional dependencies are met.") - if self.need == 1: + if self.deps.need == 1: sys.exit("Can't build as 1 mandatory dependency is missing") - elif self.need: - sys.exit(f"Can't build as {self.need} mandatory dependencies are missing") + elif self.deps.need: + sys.exit(f"Can't build as {self.deps.need} mandatory dependencies are missing") print("Needed package dependencies are met.") -- cgit From 24a34b3b453dc8fff0366a28c472b603f98a5c20 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:50 +0200 Subject: scripts: sphinx-pre-install: rework install command logic Cleanup the code to remove some redundancy and to let it be clearer about the command install instructions. Ensure that special instructions will be shown only once, before the actual install command. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/a6120449d9cc14346e867d1ef8944ae28ddbf3f6.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.py | 179 +++++++++++++++++++----------------------- 1 file changed, 82 insertions(+), 97 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py index 1b11162da9fb..7dfe5c2a6cc2 100755 --- a/scripts/sphinx-pre-install.py +++ b/scripts/sphinx-pre-install.py @@ -213,7 +213,7 @@ class DepManager: raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") if output_msg: - print(f"\n{output_msg}\n") + print(f"\n{output_msg}") class AncillaryMethods: """ @@ -583,23 +583,6 @@ class MissingCheckers(AncillaryMethods): return f.read().strip() return "" - def check_missing(self, progs): - """ - Check for missing dependencies using the provided program mapping. - - The actual distro-specific programs are mapped via progs argument. - - Returns True if there are missing dependencies, False otherwise. - """ - self.install += self.deps.check_missing(progs) - if self.verbose_warn_install: - self.deps.warn_install() - - if not self.deps.need and not self.deps.optional: - return False - - return True - def get_system_release(self): """ Determine the system type. There's no unique way that would work @@ -722,7 +705,7 @@ class SphinxDependencyChecker(MissingCheckers): self.recommend_python = None # Certain hints are meant to be shown only once - self.first_hint = True + self.distro_msg = None self.latest_avail_ver = (0, 0, 0) self.venv_ver = (0, 0, 0) @@ -732,6 +715,33 @@ class SphinxDependencyChecker(MissingCheckers): self.conf = prefix + "Documentation/conf.py" self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" + def get_install_progs(self, progs, cmd, extra=None): + """ + Check for missing dependencies using the provided program mapping. + + The actual distro-specific programs are mapped via progs argument. + """ + install = self.deps.check_missing(progs) + + if self.verbose_warn_install: + self.deps.warn_install() + + if not install: + return + + if cmd: + if self.verbose_warn_install: + msg = "You should run:" + else: + msg = "" + + if extra: + msg += "\n\t" + extra.replace("\n", "\n\t") + + return(msg + "\n\tsudo " + cmd + " " + install) + + return None + # # Distro-specific hints methods # @@ -772,12 +782,7 @@ class SphinxDependencyChecker(MissingCheckers): self.check_program("dvipng", DepManager.PDF_MANDATORY) - if self.check_missing(progs): - return - - if self.verbose_warn_install: - print("You should run:") - print(f"\n\tsudo apt-get install {self.install}") + return self.get_install_progs(progs, "apt-get install") def give_redhat_hints(self): """ @@ -838,18 +843,16 @@ class SphinxDependencyChecker(MissingCheckers): self.deps.add_package("python39", DepManager.SYSTEM_MANDATORY) self.recommend_python = True - if self.first_hint: - print("Note: RHEL-based distros typically require extra repositories.\n" \ - "For most, enabling epel and crb are enough:\n" \ - "\tsudo dnf install -y epel-release", \ - "\tsudo dnf config-manager --set-enabled crb\n" \ - "Yet, some may have other required repositories. Those commands could be useful:\n" \ - "\tsudo dnf repolist all\n" \ - "\tsudo dnf repoquery --available --info \n", - "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want") - - self.first_hint = False - + if not self.distro_msg: + self.distro_msg = \ + "Note: RHEL-based distros typically require extra repositories.\n" \ + "For most, enabling epel and crb are enough:\n" \ + "\tsudo dnf install -y epel-release\n" \ + "\tsudo dnf config-manager --set-enabled crb\n" \ + "Yet, some may have other required repositories. Those commands could be useful:\n" \ + "\tsudo dnf repolist all\n" \ + "\tsudo dnf repoquery --available --info \n" \ + "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want" if self.pdf: pdf_pkgs = [ @@ -868,13 +871,7 @@ class SphinxDependencyChecker(MissingCheckers): if not fedora and rel == 8: self.deps.del_package("texlive-ctex") - if self.check_missing(progs): - return - - if self.verbose_warn_install: - print("You should run:") - - print(f"\n\tsudo dnf install -y {self.install}") + return self.get_install_progs(progs, "dnf install") def give_opensuse_hints(self): """ @@ -945,12 +942,7 @@ class SphinxDependencyChecker(MissingCheckers): if self.pdf: self.check_missing_tex() - if self.check_missing(progs): - return - - if self.verbose_warn_install: - print("You should run:") - print(f"\n\tsudo zypper install --no-recommends {self.install}") + return self.get_install_progs(progs, "zypper install --no-recommends") def give_mageia_hints(self): """ @@ -995,12 +987,7 @@ class SphinxDependencyChecker(MissingCheckers): self.check_missing_file(pdf_pkgs, noto_sans, DepManager.PDF_MANDATORY) self.check_rpm_missing(tex_pkgs, DepManager.PDF_MANDATORY) - if self.check_missing(progs): - return - - if self.verbose_warn_install: - print("You should run:") - print(f"\n\tsudo {packager_cmd} {self.install}") + return self.get_install_progs(progs, packager_cmd) def give_arch_linux_hints(self): """ @@ -1023,20 +1010,15 @@ class SphinxDependencyChecker(MissingCheckers): ] if self.pdf: - self.check_pacman_missing(archlinux_tex_pkgs, DepManager.PDF_MANDATORY) + self.check_pacman_missing(archlinux_tex_pkgs, + DepManager.PDF_MANDATORY) - self.check_missing_file( - ["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], - "noto-fonts-cjk", - 2, - ) + self.check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], + "noto-fonts-cjk", + DepManager.PDF_MANDATORY) - if self.check_missing(progs): - return - if self.verbose_warn_install: - print("You should run:") - print(f"\n\tsudo pacman -S {self.install}") + return self.get_install_progs(progs, "pacman -S") def give_gentoo_hints(self): """ @@ -1065,13 +1047,6 @@ class SphinxDependencyChecker(MissingCheckers): for package, files in pdf_pkgs.items(): self.check_missing_file(files, package, DepManager.PDF_MANDATORY) - if self.check_missing(progs): - return - - if self.verbose_warn_install: - print("You should run:") - print("\n") - # Handling dependencies is a nightmare, as Gentoo refuses to emerge # some packages if there's no package.use file describing them. # To make it worse, compilation flags shall also be present there @@ -1104,7 +1079,10 @@ class SphinxDependencyChecker(MissingCheckers): "zziblib": "dev-libs/zziplib sdl", } - if self.first_hint: + extra_cmds = "" + if not self.distro_msg: + self.distro_msg = "Note: Gentoo requires package.use to be adjusted before emerging packages" + use_base = "/etc/portage/package.use" files = glob(f"{use_base}/*") @@ -1148,23 +1126,23 @@ class SphinxDependencyChecker(MissingCheckers): # emit a code to setup missing USE if install: - print(f"\tsudo su -c 'echo \"{portage}\" > {use_base}/{fname}'") - - self.first_hint = False + extra_cmds += (f"sudo su -c 'echo \"{portage}\" > {use_base}/{fname}'\n") # Now, we can use emerge and let it respect USE - print(f"\tsudo emerge --ask --changed-use --binpkg-respect-use=y {self.install}") + return self.get_install_progs(progs, + "emerge --ask --changed-use --binpkg-respect-use=y", + extra_cmds) - # - # Dispatch the check to an os_specific hinter - # - - def check_distros(self): + def get_install(self): """ - OS-specific hints logic. Seeks for a hinter. If found, provide - package-manager-specific install commands. + OS-specific hints logic. Seeks for a hinter. If found, use it to + provide package-manager specific install commands. + + Otherwise, outputs install instructions for the meta-packages. - Otherwise, just lists the missing dependencies. + Returns a string with the command to be executed to install the + the needed packages, if distro found. Otherwise, return just a + list of packages that require installation. """ os_hints = { re.compile("Red Hat Enterprise Linux"): self.give_redhat_hints, @@ -1195,9 +1173,7 @@ class SphinxDependencyChecker(MissingCheckers): # If the OS is detected, use per-OS hint logic for regex, os_hint in os_hints.items(): if regex.search(self.system_release): - os_hint() - - return + return os_hint() # # Fall-back to generic hint code for other distros @@ -1207,11 +1183,12 @@ class SphinxDependencyChecker(MissingCheckers): if self.pdf: self.check_missing_tex() - self.check_missing(progs) + self.distro_msg = \ + f"I don't know distro {self.system_release}.\n" \ + "So, I can't provide you a hint with the install procedure.\n" \ + "There are likely missing dependencies.\n" - print(f"I don't know distro {self.system_release}.") - print("So, I can't provide you a hint with the install procedure.") - print("There are likely missing dependencies.") + return self.get_install_progs(progs, None) # # Common dependencies @@ -1336,7 +1313,6 @@ class SphinxDependencyChecker(MissingCheckers): self.pdf = False self.deps.optional = 0 - self.install = "" old_verbose = self.verbose_warn_install self.verbose_warn_install = 0 @@ -1344,7 +1320,9 @@ class SphinxDependencyChecker(MissingCheckers): self.deps.add_package("python-sphinx", DepManager.PYTHON_MANDATORY) - self.check_distros() + cmd = self.get_install() + if cmd: + print(cmd) self.deps.need = old_need self.deps.optional = old_optional @@ -1511,7 +1489,14 @@ class SphinxDependencyChecker(MissingCheckers): self.check_program("latexmk", DepManager.PDF_MANDATORY) # Do distro-specific checks and output distro-install commands - self.check_distros() + cmd = self.get_install() + if cmd: + print(cmd) + + # If distro requires some special instructions, print here. + # Please notice that get_install() needs to be called first. + if self.distro_msg: + print("\n" + self.distro_msg) if not self.python_cmd: if self.need == 1: -- cgit From d43cd965f3a646d22851192c659b787dba311d87 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:51 +0200 Subject: docs: Makefile: switch to the new scripts/sphinx-pre-install.py Now that we have a better, improved Python script, use it when checking for documentation build dependencies. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/79508fb071512c33e807f5411bbff1904751b5d3.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 1565 +++++++++++++++++++++++++++++++++++++++++ scripts/sphinx-pre-install.py | 1565 ----------------------------------------- 2 files changed, 1565 insertions(+), 1565 deletions(-) create mode 100755 scripts/sphinx-pre-install delete mode 100755 scripts/sphinx-pre-install.py (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install new file mode 100755 index 000000000000..7dfe5c2a6cc2 --- /dev/null +++ b/scripts/sphinx-pre-install @@ -0,0 +1,1565 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (c) 2017-2025 Mauro Carvalho Chehab +# +# pylint: disable=C0103,C0114,C0115,C0116,C0301,C0302 +# pylint: disable=R0902,R0904,R0911,R0912,R0914,R0915,R1705,R1710,E1121 + +# Note: this script requires at least Python 3.6 to run. +# Don't add changes not compatible with it, it is meant to report +# incompatible python versions. + +""" +Dependency checker for Sphinx documentation Kernel build. + +This module provides tools to check for all required dependencies needed to +build documentation using Sphinx, including system packages, Python modules +and LaTeX packages for PDF generation. + +It detect packages for a subset of Linux distributions used by Kernel +maintainers, showing hints and missing dependencies. + +The main class SphinxDependencyChecker handles the dependency checking logic +and provides recommendations for installing missing packages. It supports both +system package installations and Python virtual environments. By default, +system pacage install is recommended. +""" + +import argparse +import os +import re +import subprocess +import sys +from glob import glob + + +def parse_version(version): + """Convert a major.minor.patch version into a tuple""" + return tuple(int(x) for x in version.split(".")) + + +def ver_str(version): + """Returns a version tuple as major.minor.patch""" + + return ".".join([str(x) for x in version]) + + +RECOMMENDED_VERSION = parse_version("3.4.3") +MIN_PYTHON_VERSION = parse_version("3.7") + + +class DepManager: + """ + Manage package dependencies. There are three types of dependencies: + + - System: dependencies required for docs build; + - Python: python dependencies for a native distro Sphinx install; + - PDF: dependencies needed by PDF builds. + + Each dependency can be mandatory or optional. Not installing an optional + dependency won't break the build, but will cause degradation at the + docs output. + """ + + # Internal types of dependencies. Don't use them outside DepManager class. + _SYS_TYPE = 0 + _PHY_TYPE = 1 + _PDF_TYPE = 2 + + # Dependencies visible outside the class. + # The keys are tuple with: (type, is_mandatory flag). + # + # Currently we're not using all optional dep types. Yet, we'll keep all + # possible combinations here. They're not many, and that makes easier + # if later needed and for the name() method below + + SYSTEM_MANDATORY = (_SYS_TYPE, True) + PYTHON_MANDATORY = (_PHY_TYPE, True) + PDF_MANDATORY = (_PDF_TYPE, True) + + SYSTEM_OPTIONAL = (_SYS_TYPE, False) + PYTHON_OPTIONAL = (_PHY_TYPE, False) + PDF_OPTIONAL = (_PDF_TYPE, True) + + def __init__(self, pdf): + """ + Initialize internal vars: + + - missing: missing dependencies list, containing a distro-independent + name for a missing dependency and its type. + - missing_pkg: ancillary dict containing missing dependencies in + distro namespace, organized by type. + - need: total number of needed dependencies. Never cleaned. + - optional: total number of optional dependencies. Never cleaned. + - pdf: Is PDF support enabled? + """ + self.missing = {} + self.missing_pkg = {} + self.need = 0 + self.optional = 0 + self.pdf = pdf + + @staticmethod + def name(dtype): + """ + Ancillary routine to output a warn/error message reporting + missing dependencies. + """ + if dtype[0] == DepManager._SYS_TYPE: + msg = "build" + elif dtype[0] == DepManager._PHY_TYPE: + msg = "Python" + else: + msg = "PDF" + + if dtype[1]: + return f"ERROR: {msg} mandatory deps missing" + else: + return f"Warning: {msg} optional deps missing" + + @staticmethod + def is_optional(dtype): + """Ancillary routine to report if a dependency is optional""" + return not dtype[1] + + @staticmethod + def is_pdf(dtype): + """Ancillary routine to report if a dependency is for PDF generation""" + if dtype[0] == DepManager._PDF_TYPE: + return True + + return False + + def add_package(self, package, dtype): + """ + Add a package at the self.missing() dictionary. + Doesn't update missing_pkg. + """ + is_optional = DepManager.is_optional(dtype) + self.missing[package] = dtype + if is_optional: + self.optional += 1 + else: + self.need += 1 + + def del_package(self, package): + """ + Remove a package at the self.missing() dictionary. + Doesn't update missing_pkg. + """ + if package in self.missing: + del self.missing[package] + + def clear_deps(self): + """ + Clear dependencies without changing needed/optional. + + This is an ackward way to have a separate section to recommend + a package after system main dependencies. + + TODO: rework the logic to prevent needing it. + """ + + self.missing = {} + self.missing_pkg = {} + + def check_missing(self, progs): + """ + Update self.missing_pkg, using progs dict to convert from the + agnostic package name to distro-specific one. + + Returns an string with the packages to be installed, sorted and + with eventual duplicates removed. + """ + + self.missing_pkg = {} + + for prog, dtype in sorted(self.missing.items()): + # At least on some LTS distros like CentOS 7, texlive doesn't + # provide all packages we need. When such distros are + # detected, we have to disable PDF output. + # + # So, we need to ignore the packages that distros would + # need for LaTeX to work + if DepManager.is_pdf(dtype) and not self.pdf: + self.optional -= 1 + continue + + if not dtype in self.missing_pkg: + self.missing_pkg[dtype] = [] + + self.missing_pkg[dtype].append(progs.get(prog, prog)) + + install = [] + for dtype, pkgs in self.missing_pkg.items(): + install += pkgs + + return " ".join(sorted(set(install))) + + def warn_install(self): + """ + Emit warnings/errors related to missing packages. + """ + + output_msg = "" + + for dtype in sorted(self.missing_pkg.keys()): + progs = " ".join(sorted(set(self.missing_pkg[dtype]))) + + try: + name = DepManager.name(dtype) + output_msg += f'{name}:\t{progs}\n' + except KeyError: + raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") + + if output_msg: + print(f"\n{output_msg}") + +class AncillaryMethods: + """ + Ancillary methods that checks for missing dependencies for different + types of types, like binaries, python modules, rpm deps, etc. + """ + + @staticmethod + def which(prog): + """ + Our own implementation of which(). We could instead use + shutil.which(), but this function is simple enough. + Probably faster to use this implementation than to import shutil. + """ + for path in os.environ.get("PATH", "").split(":"): + full_path = os.path.join(path, prog) + if os.access(full_path, os.X_OK): + return full_path + + return None + + @staticmethod + def get_python_version(cmd): + """ + Get python version from a Python binary. As we need to detect if + are out there newer python binaries, we can't rely on sys.release here. + """ + + result = SphinxDependencyChecker.run([cmd, "--version"], + capture_output=True, text=True) + version = result.stdout.strip() + + match = re.search(r"(\d+\.\d+\.\d+)", version) + if match: + return parse_version(match.group(1)) + + print(f"Can't parse version {version}") + return (0, 0, 0) + + @staticmethod + def find_python(): + """ + Detect if are out there any python 3.xy version newer than the + current one. + + Note: this routine is limited to up to 2 digits for python3. We + may need to update it one day, hopefully on a distant future. + """ + patterns = [ + "python3.[0-9]", + "python3.[0-9][0-9]", + ] + + # Seek for a python binary newer than MIN_PYTHON_VERSION + for path in os.getenv("PATH", "").split(":"): + for pattern in patterns: + for cmd in glob(os.path.join(path, pattern)): + if os.path.isfile(cmd) and os.access(cmd, os.X_OK): + version = SphinxDependencyChecker.get_python_version(cmd) + if version >= MIN_PYTHON_VERSION: + return cmd + + @staticmethod + def check_python(): + """ + Check if the current python binary satisfies our minimal requirement + for Sphinx build. If not, re-run with a newer version if found. + """ + cur_ver = sys.version_info[:3] + if cur_ver >= MIN_PYTHON_VERSION: + ver = ver_str(cur_ver) + print(f"Python version: {ver}") + + # This could be useful for debugging purposes + if SphinxDependencyChecker.which("docutils"): + result = SphinxDependencyChecker.run(["docutils", "--version"], + capture_output=True, text=True) + ver = result.stdout.strip() + match = re.search(r"(\d+\.\d+\.\d+)", ver) + if match: + ver = match.group(1) + + print(f"Docutils version: {ver}") + + return + + python_ver = ver_str(cur_ver) + + new_python_cmd = SphinxDependencyChecker.find_python() + if not new_python_cmd: + print(f"ERROR: Python version {python_ver} is not spported anymore\n") + print(" Can't find a new version. This script may fail") + return + + # Restart script using the newer version + script_path = os.path.abspath(sys.argv[0]) + args = [new_python_cmd, script_path] + sys.argv[1:] + + print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") + + try: + os.execv(new_python_cmd, args) + except OSError as e: + sys.exit(f"Failed to restart with {new_python_cmd}: {e}") + + @staticmethod + def run(*args, **kwargs): + """ + Excecute a command, hiding its output by default. + Preserve comatibility with older Python versions. + """ + + capture_output = kwargs.pop('capture_output', False) + + if capture_output: + if 'stdout' not in kwargs: + kwargs['stdout'] = subprocess.PIPE + if 'stderr' not in kwargs: + kwargs['stderr'] = subprocess.PIPE + else: + if 'stdout' not in kwargs: + kwargs['stdout'] = subprocess.DEVNULL + if 'stderr' not in kwargs: + kwargs['stderr'] = subprocess.DEVNULL + + # Don't break with older Python versions + if 'text' in kwargs and sys.version_info < (3, 7): + kwargs['universal_newlines'] = kwargs.pop('text') + + return subprocess.run(*args, **kwargs) + +class MissingCheckers(AncillaryMethods): + """ + Contains some ancillary checkers for different types of binaries and + package managers. + """ + + def __init__(self, args, texlive): + """ + Initialize its internal variables + """ + self.pdf = args.pdf + self.virtualenv = args.virtualenv + self.version_check = args.version_check + self.texlive = texlive + + self.min_version = (0, 0, 0) + self.cur_version = (0, 0, 0) + + self.deps = DepManager(self.pdf) + + self.need_symlink = 0 + self.need_sphinx = 0 + + self.verbose_warn_install = 1 + + self.virtenv_dir = "" + self.install = "" + self.python_cmd = "" + + self.virtenv_prefix = ["sphinx_", "Sphinx_" ] + + def check_missing_file(self, files, package, dtype): + """ + Does the file exists? If not, add it to missing dependencies. + """ + for f in files: + if os.path.exists(f): + return + self.deps.add_package(package, dtype) + + def check_program(self, prog, dtype): + """ + Does the program exists and it is at the PATH? + If not, add it to missing dependencies. + """ + found = self.which(prog) + if found: + return found + + self.deps.add_package(prog, dtype) + + return None + + def check_perl_module(self, prog, dtype): + """ + Does perl have a dependency? Is it available? + If not, add it to missing dependencies. + + Right now, we still need Perl for doc build, as it is required + by some tools called at docs or kernel build time, like: + + scripts/documentation-file-ref-check + + Also, checkpatch is on Perl. + """ + + # While testing with lxc download template, one of the + # distros (Oracle) didn't have perl - nor even an option to install + # before installing oraclelinux-release-el9 package. + # + # Check it before running an error. If perl is not there, + # add it as a mandatory package, as some parts of the doc builder + # needs it. + if not self.which("perl"): + self.deps.add_package("perl", DepManager.SYSTEM_MANDATORY) + self.deps.add_package(prog, dtype) + return + + try: + self.run(["perl", f"-M{prog}", "-e", "1"], check=True) + except subprocess.CalledProcessError: + self.deps.add_package(prog, dtype) + + def check_python_module(self, module, is_optional=False): + """ + Does a python module exists outside venv? If not, add it to missing + dependencies. + """ + if is_optional: + dtype = DepManager.PYTHON_OPTIONAL + else: + dtype = DepManager.PYTHON_MANDATORY + + try: + self.run([self.python_cmd, "-c", f"import {module}"], check=True) + except subprocess.CalledProcessError: + self.deps.add_package(module, dtype) + + def check_rpm_missing(self, pkgs, dtype): + """ + Does a rpm package exists? If not, add it to missing dependencies. + """ + for prog in pkgs: + try: + self.run(["rpm", "-q", prog], check=True) + except subprocess.CalledProcessError: + self.deps.add_package(prog, dtype) + + def check_pacman_missing(self, pkgs, dtype): + """ + Does a pacman package exists? If not, add it to missing dependencies. + """ + for prog in pkgs: + try: + self.run(["pacman", "-Q", prog], check=True) + except subprocess.CalledProcessError: + self.deps.add_package(prog, dtype) + + def check_missing_tex(self, is_optional=False): + """ + Does a LaTeX package exists? If not, add it to missing dependencies. + """ + if is_optional: + dtype = DepManager.PDF_OPTIONAL + else: + dtype = DepManager.PDF_MANDATORY + + kpsewhich = self.which("kpsewhich") + for prog, package in self.texlive.items(): + + # If kpsewhich is not there, just add it to deps + if not kpsewhich: + self.deps.add_package(package, dtype) + continue + + # Check if the package is needed + try: + result = self.run( + [kpsewhich, prog], stdout=subprocess.PIPE, text=True, check=True + ) + + # Didn't find. Add it + if not result.stdout.strip(): + self.deps.add_package(package, dtype) + + except subprocess.CalledProcessError: + # kpsewhich returned an error. Add it, just in case + self.deps.add_package(package, dtype) + + def get_sphinx_fname(self): + """ + Gets the binary filename for sphinx-build. + """ + if "SPHINXBUILD" in os.environ: + return os.environ["SPHINXBUILD"] + + fname = "sphinx-build" + if self.which(fname): + return fname + + fname = "sphinx-build-3" + if self.which(fname): + self.need_symlink = 1 + return fname + + return "" + + def get_sphinx_version(self, cmd): + """ + Gets sphinx-build version. + """ + try: + result = self.run([cmd, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, check=True) + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + for line in result.stdout.split("\n"): + match = re.match(r"^sphinx-build\s+([\d\.]+)(?:\+(?:/[\da-f]+)|b\d+)?\s*$", line) + if match: + return parse_version(match.group(1)) + + match = re.match(r"^Sphinx.*\s+([\d\.]+)\s*$", line) + if match: + return parse_version(match.group(1)) + + def check_sphinx(self, conf): + """ + Checks Sphinx minimal requirements + """ + try: + with open(conf, "r", encoding="utf-8") as f: + for line in f: + match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line) + if match: + self.min_version = parse_version(match.group(1)) + break + except IOError: + sys.exit(f"Can't open {conf}") + + if not self.min_version: + sys.exit(f"Can't get needs_sphinx version from {conf}") + + self.virtenv_dir = self.virtenv_prefix[0] + "latest" + + sphinx = self.get_sphinx_fname() + if not sphinx: + self.need_sphinx = 1 + return + + self.cur_version = self.get_sphinx_version(sphinx) + if not self.cur_version: + sys.exit(f"{sphinx} didn't return its version") + + if self.cur_version < self.min_version: + curver = ver_str(self.cur_version) + minver = ver_str(self.min_version) + + print(f"ERROR: Sphinx version is {curver}. It should be >= {minver}") + self.need_sphinx = 1 + return + + # On version check mode, just assume Sphinx has all mandatory deps + if self.version_check and self.cur_version >= RECOMMENDED_VERSION: + sys.exit(0) + + def catcheck(self, filename): + """ + Reads a file if it exists, returning as string. + If not found, returns an empty string. + """ + if os.path.exists(filename): + with open(filename, "r", encoding="utf-8") as f: + return f.read().strip() + return "" + + def get_system_release(self): + """ + Determine the system type. There's no unique way that would work + with all distros with a minimal package install. So, several + methods are used here. + + By default, it will use lsb_release function. If not available, it will + fail back to reading the known different places where the distro name + is stored. + + Several modern distros now have /etc/os-release, which usually have + a decent coverage. + """ + + system_release = "" + + if self.which("lsb_release"): + result = self.run(["lsb_release", "-d"], capture_output=True, text=True) + system_release = result.stdout.replace("Description:", "").strip() + + release_files = [ + "/etc/system-release", + "/etc/redhat-release", + "/etc/lsb-release", + "/etc/gentoo-release", + ] + + if not system_release: + for f in release_files: + system_release = self.catcheck(f) + if system_release: + break + + # This seems more common than LSB these days + if not system_release: + os_var = {} + try: + with open("/etc/os-release", "r", encoding="utf-8") as f: + for line in f: + match = re.match(r"^([\w\d\_]+)=\"?([^\"]*)\"?\n", line) + if match: + os_var[match.group(1)] = match.group(2) + + system_release = os_var.get("NAME", "") + if "VERSION_ID" in os_var: + system_release += " " + os_var["VERSION_ID"] + elif "VERSION" in os_var: + system_release += " " + os_var["VERSION"] + except IOError: + pass + + if not system_release: + system_release = self.catcheck("/etc/issue") + + system_release = system_release.strip() + + return system_release + +class SphinxDependencyChecker(MissingCheckers): + """ + Main class for checking Sphinx documentation build dependencies. + + - Check for missing system packages; + - Check for missing Python modules; + - Check for missing LaTeX packages needed by PDF generation; + - Propose Sphinx install via Python Virtual environment; + - Propose Sphinx install via distro-specific package install. + """ + def __init__(self, args): + """Initialize checker variables""" + + # List of required texlive packages on Fedora and OpenSuse + texlive = { + "amsfonts.sty": "texlive-amsfonts", + "amsmath.sty": "texlive-amsmath", + "amssymb.sty": "texlive-amsfonts", + "amsthm.sty": "texlive-amscls", + "anyfontsize.sty": "texlive-anyfontsize", + "atbegshi.sty": "texlive-oberdiek", + "bm.sty": "texlive-tools", + "capt-of.sty": "texlive-capt-of", + "cmap.sty": "texlive-cmap", + "ctexhook.sty": "texlive-ctex", + "ecrm1000.tfm": "texlive-ec", + "eqparbox.sty": "texlive-eqparbox", + "eu1enc.def": "texlive-euenc", + "fancybox.sty": "texlive-fancybox", + "fancyvrb.sty": "texlive-fancyvrb", + "float.sty": "texlive-float", + "fncychap.sty": "texlive-fncychap", + "footnote.sty": "texlive-mdwtools", + "framed.sty": "texlive-framed", + "luatex85.sty": "texlive-luatex85", + "multirow.sty": "texlive-multirow", + "needspace.sty": "texlive-needspace", + "palatino.sty": "texlive-psnfss", + "parskip.sty": "texlive-parskip", + "polyglossia.sty": "texlive-polyglossia", + "tabulary.sty": "texlive-tabulary", + "threeparttable.sty": "texlive-threeparttable", + "titlesec.sty": "texlive-titlesec", + "ucs.sty": "texlive-ucs", + "upquote.sty": "texlive-upquote", + "wrapfig.sty": "texlive-wrapfig", + } + + super().__init__(args, texlive) + + self.need_pip = 0 + self.rec_sphinx_upgrade = 0 + + self.system_release = self.get_system_release() + self.activate_cmd = "" + + # Some distros may not have a Sphinx shipped package compatible with + # our minimal requirements + self.package_supported = True + + # Recommend a new python version + self.recommend_python = None + + # Certain hints are meant to be shown only once + self.distro_msg = None + + self.latest_avail_ver = (0, 0, 0) + self.venv_ver = (0, 0, 0) + + prefix = os.environ.get("srctree", ".") + "/" + + self.conf = prefix + "Documentation/conf.py" + self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" + + def get_install_progs(self, progs, cmd, extra=None): + """ + Check for missing dependencies using the provided program mapping. + + The actual distro-specific programs are mapped via progs argument. + """ + install = self.deps.check_missing(progs) + + if self.verbose_warn_install: + self.deps.warn_install() + + if not install: + return + + if cmd: + if self.verbose_warn_install: + msg = "You should run:" + else: + msg = "" + + if extra: + msg += "\n\t" + extra.replace("\n", "\n\t") + + return(msg + "\n\tsudo " + cmd + " " + install) + + return None + + # + # Distro-specific hints methods + # + + def give_debian_hints(self): + """ + Provide package installation hints for Debian-based distros. + """ + progs = { + "Pod::Usage": "perl-modules", + "convert": "imagemagick", + "dot": "graphviz", + "ensurepip": "python3-venv", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2-bin", + "virtualenv": "virtualenv", + "xelatex": "texlive-xetex", + "yaml": "python3-yaml", + } + + if self.pdf: + pdf_pkgs = { + "texlive-lang-chinese": [ + "/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty", + ], + "fonts-dejavu": [ + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + ], + "fonts-noto-cjk": [ + "/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc", + ], + } + + for package, files in pdf_pkgs.items(): + self.check_missing_file(files, package, DepManager.PDF_MANDATORY) + + self.check_program("dvipng", DepManager.PDF_MANDATORY) + + return self.get_install_progs(progs, "apt-get install") + + def give_redhat_hints(self): + """ + Provide package installation hints for RedHat-based distros + (Fedora, RHEL and RHEL-based variants). + """ + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2-tools", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive-xetex-bin", + "yaml": "python3-pyyaml", + } + + fedora_tex_pkgs = [ + "dejavu-sans-fonts", + "dejavu-sans-mono-fonts", + "dejavu-serif-fonts", + "texlive-collection-fontsrecommended", + "texlive-collection-latex", + "texlive-xecjk", + ] + + fedora = False + rel = None + + match = re.search(r"(release|Linux)\s+(\d+)", self.system_release) + if match: + rel = int(match.group(2)) + + if not rel: + print("Couldn't identify release number") + noto_sans_redhat = None + self.pdf = False + elif re.search("Fedora", self.system_release): + # Fedora 38 and upper use this CJK font + + noto_sans_redhat = "google-noto-sans-cjk-fonts" + fedora = True + else: + # Almalinux, CentOS, RHEL, ... + + # at least up to version 9 (and Fedora < 38), that's the CJK font + noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts" + + progs["virtualenv"] = "python-virtualenv" + + if not rel or rel < 8: + print("ERROR: Distro not supported. Too old?") + return + + # RHEL 8 uses Python 3.6, which is not compatible with + # the build system anymore. Suggest Python 3.11 + if rel == 8: + self.deps.add_package("python39", DepManager.SYSTEM_MANDATORY) + self.recommend_python = True + + if not self.distro_msg: + self.distro_msg = \ + "Note: RHEL-based distros typically require extra repositories.\n" \ + "For most, enabling epel and crb are enough:\n" \ + "\tsudo dnf install -y epel-release\n" \ + "\tsudo dnf config-manager --set-enabled crb\n" \ + "Yet, some may have other required repositories. Those commands could be useful:\n" \ + "\tsudo dnf repolist all\n" \ + "\tsudo dnf repoquery --available --info \n" \ + "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want" + + if self.pdf: + pdf_pkgs = [ + "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc", + ] + + self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepManager.PDF_MANDATORY) + + self.check_rpm_missing(fedora_tex_pkgs, DepManager.PDF_MANDATORY) + + self.check_missing_tex(DepManager.PDF_MANDATORY) + + # There's no texlive-ctex on RHEL 8 repositories. This will + # likely affect CJK pdf build only. + if not fedora and rel == 8: + self.deps.del_package("texlive-ctex") + + return self.get_install_progs(progs, "dnf install") + + def give_opensuse_hints(self): + """ + Provide package installation hints for openSUSE-based distros + (Leap and Tumbleweed). + """ + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive-xetex-bin", + "yaml": "python3-pyyaml", + } + + suse_tex_pkgs = [ + "texlive-babel-english", + "texlive-caption", + "texlive-colortbl", + "texlive-courier", + "texlive-dvips", + "texlive-helvetic", + "texlive-makeindex", + "texlive-metafont", + "texlive-metapost", + "texlive-palatino", + "texlive-preview", + "texlive-times", + "texlive-zapfchan", + "texlive-zapfding", + ] + + progs["latexmk"] = "texlive-latexmk-bin" + + match = re.search(r"(Leap)\s+(\d+).(\d)", self.system_release) + if match: + rel = int(match.group(2)) + + # Leap 15.x uses Python 3.6, which is not compatible with + # the build system anymore. Suggest Python 3.11 + if rel == 15: + if not self.which(self.python_cmd): + self.recommend_python = True + self.deps.add_package(self.python_cmd, DepManager.SYSTEM_MANDATORY) + + progs.update({ + "python-sphinx": "python311-Sphinx", + "virtualenv": "python311-virtualenv", + "yaml": "python311-PyYAML", + }) + else: + # Tumbleweed defaults to Python 3.11 + + progs.update({ + "python-sphinx": "python313-Sphinx", + "virtualenv": "python313-virtualenv", + "yaml": "python313-PyYAML", + }) + + # FIXME: add support for installing CJK fonts + # + # I tried hard, but was unable to find a way to install + # "Noto Sans CJK SC" on openSUSE + + if self.pdf: + self.check_rpm_missing(suse_tex_pkgs, DepManager.PDF_MANDATORY) + if self.pdf: + self.check_missing_tex() + + return self.get_install_progs(progs, "zypper install --no-recommends") + + def give_mageia_hints(self): + """ + Provide package installation hints for Mageia and OpenMandriva. + """ + progs = { + "Pod::Usage": "perl-Pod-Usage", + "convert": "ImageMagick", + "dot": "graphviz", + "python-sphinx": "python3-sphinx", + "rsvg-convert": "librsvg2", + "virtualenv": "python3-virtualenv", + "xelatex": "texlive", + "yaml": "python3-yaml", + } + + tex_pkgs = [ + "texlive-fontsextra", + ] + + if re.search(r"OpenMandriva", self.system_release): + packager_cmd = "dnf install" + noto_sans = "noto-sans-cjk-fonts" + tex_pkgs = ["texlive-collection-fontsextra"] + + # Tested on OpenMandriva Lx 4.3 + progs["convert"] = "imagemagick" + progs["yaml"] = "python-pyyaml" + + else: + packager_cmd = "urpmi" + noto_sans = "google-noto-sans-cjk-ttc-fonts" + + progs["latexmk"] = "texlive-collection-basic" + + if self.pdf: + pdf_pkgs = [ + "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/TTF/NotoSans-Regular.ttf", + ] + + self.check_missing_file(pdf_pkgs, noto_sans, DepManager.PDF_MANDATORY) + self.check_rpm_missing(tex_pkgs, DepManager.PDF_MANDATORY) + + return self.get_install_progs(progs, packager_cmd) + + def give_arch_linux_hints(self): + """ + Provide package installation hints for ArchLinux. + """ + progs = { + "convert": "imagemagick", + "dot": "graphviz", + "latexmk": "texlive-core", + "rsvg-convert": "extra/librsvg", + "virtualenv": "python-virtualenv", + "xelatex": "texlive-xetex", + "yaml": "python-yaml", + } + + archlinux_tex_pkgs = [ + "texlive-core", + "texlive-latexextra", + "ttf-dejavu", + ] + + if self.pdf: + self.check_pacman_missing(archlinux_tex_pkgs, + DepManager.PDF_MANDATORY) + + self.check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], + "noto-fonts-cjk", + DepManager.PDF_MANDATORY) + + + return self.get_install_progs(progs, "pacman -S") + + def give_gentoo_hints(self): + """ + Provide package installation hints for Gentoo. + """ + progs = { + "convert": "media-gfx/imagemagick", + "dot": "media-gfx/graphviz", + "rsvg-convert": "gnome-base/librsvg", + "virtualenv": "dev-python/virtualenv", + "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", + "yaml": "dev-python/pyyaml", + "python-sphinx": "dev-python/sphinx", + } + + if self.pdf: + pdf_pkgs = { + "media-fonts/dejavu": [ + "/usr/share/fonts/dejavu/DejaVuSans.ttf", + ], + "media-fonts/noto-cjk": [ + "/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", + "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc", + ], + } + for package, files in pdf_pkgs.items(): + self.check_missing_file(files, package, DepManager.PDF_MANDATORY) + + # Handling dependencies is a nightmare, as Gentoo refuses to emerge + # some packages if there's no package.use file describing them. + # To make it worse, compilation flags shall also be present there + # for some packages. If USE is not perfect, error/warning messages + # like those are shown: + # + # !!! The following binary packages have been ignored due to non matching USE: + # + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_13 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf python_single_target_python3_12 -python_single_target_python3_13 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 qt6 svg + # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 python_single_target_python3_12 -python_single_target_python3_13 qt6 svg + # =media-fonts/noto-cjk-20190416 X + # =app-text/texlive-core-2024-r1 X cjk -xetex + # =app-text/texlive-core-2024-r1 X -xetex + # =app-text/texlive-core-2024-r1 -xetex + # =dev-libs/zziplib-0.13.79-r1 sdl + # + # And will ignore such packages, installing the remaining ones. That + # affects mostly the image extension and PDF generation. + + # Package dependencies and the minimal needed args: + portages = { + "graphviz": "media-gfx/graphviz", + "imagemagick": "media-gfx/imagemagick", + "media-libs": "media-libs/harfbuzz icu", + "media-fonts": "media-fonts/noto-cjk", + "texlive": "app-text/texlive-core xetex", + "zziblib": "dev-libs/zziplib sdl", + } + + extra_cmds = "" + if not self.distro_msg: + self.distro_msg = "Note: Gentoo requires package.use to be adjusted before emerging packages" + + use_base = "/etc/portage/package.use" + files = glob(f"{use_base}/*") + + for fname, portage in portages.items(): + install = False + + while install is False: + if not files: + # No files under package.usage. Install all + install = True + break + + args = portage.split(" ") + + name = args.pop(0) + + cmd = ["grep", "-l", "-E", rf"^{name}\b" ] + files + result = self.run(cmd, stdout=subprocess.PIPE, text=True) + if result.returncode or not result.stdout.strip(): + # File containing portage name not found + install = True + break + + # Ensure that needed USE flags are present + if args: + match_fname = result.stdout.strip() + with open(match_fname, 'r', encoding='utf8', + errors='backslashreplace') as fp: + for line in fp: + for arg in args: + if arg.startswith("-"): + continue + + if not re.search(rf"\s*{arg}\b", line): + # Needed file argument not found + install = True + break + + # Everything looks ok, don't install + break + + # emit a code to setup missing USE + if install: + extra_cmds += (f"sudo su -c 'echo \"{portage}\" > {use_base}/{fname}'\n") + + # Now, we can use emerge and let it respect USE + return self.get_install_progs(progs, + "emerge --ask --changed-use --binpkg-respect-use=y", + extra_cmds) + + def get_install(self): + """ + OS-specific hints logic. Seeks for a hinter. If found, use it to + provide package-manager specific install commands. + + Otherwise, outputs install instructions for the meta-packages. + + Returns a string with the command to be executed to install the + the needed packages, if distro found. Otherwise, return just a + list of packages that require installation. + """ + os_hints = { + re.compile("Red Hat Enterprise Linux"): self.give_redhat_hints, + re.compile("Fedora"): self.give_redhat_hints, + re.compile("AlmaLinux"): self.give_redhat_hints, + re.compile("Amazon Linux"): self.give_redhat_hints, + re.compile("CentOS"): self.give_redhat_hints, + re.compile("openEuler"): self.give_redhat_hints, + re.compile("Oracle Linux Server"): self.give_redhat_hints, + re.compile("Rocky Linux"): self.give_redhat_hints, + re.compile("Springdale Open Enterprise"): self.give_redhat_hints, + + re.compile("Ubuntu"): self.give_debian_hints, + re.compile("Debian"): self.give_debian_hints, + re.compile("Devuan"): self.give_debian_hints, + re.compile("Kali"): self.give_debian_hints, + re.compile("Mint"): self.give_debian_hints, + + re.compile("openSUSE"): self.give_opensuse_hints, + + re.compile("Mageia"): self.give_mageia_hints, + re.compile("OpenMandriva"): self.give_mageia_hints, + + re.compile("Arch Linux"): self.give_arch_linux_hints, + re.compile("Gentoo"): self.give_gentoo_hints, + } + + # If the OS is detected, use per-OS hint logic + for regex, os_hint in os_hints.items(): + if regex.search(self.system_release): + return os_hint() + + # + # Fall-back to generic hint code for other distros + # That's far from ideal, specially for LaTeX dependencies. + # + progs = {"sphinx-build": "sphinx"} + if self.pdf: + self.check_missing_tex() + + self.distro_msg = \ + f"I don't know distro {self.system_release}.\n" \ + "So, I can't provide you a hint with the install procedure.\n" \ + "There are likely missing dependencies.\n" + + return self.get_install_progs(progs, None) + + # + # Common dependencies + # + def deactivate_help(self): + """ + Print a helper message to disable a virtual environment. + """ + + print("\n If you want to exit the virtualenv, you can use:") + print("\tdeactivate") + + def get_virtenv(self): + """ + Give a hint about how to activate an already-existing virtual + environment containing sphinx-build. + + Returns a tuble with (activate_cmd_path, sphinx_version) with + the newest available virtual env. + """ + + cwd = os.getcwd() + + activates = [] + + # Add all sphinx prefixes with possible version numbers + for p in self.virtenv_prefix: + activates += glob(f"{cwd}/{p}[0-9]*/bin/activate") + + activates.sort(reverse=True, key=str.lower) + + # Place sphinx_latest first, if it exists + for p in self.virtenv_prefix: + activates = glob(f"{cwd}/{p}*latest/bin/activate") + activates + + ver = (0, 0, 0) + for f in activates: + # Discard too old Sphinx virtual environments + match = re.search(r"(\d+)\.(\d+)\.(\d+)", f) + if match: + ver = (int(match.group(1)), int(match.group(2)), int(match.group(3))) + + if ver < self.min_version: + continue + + sphinx_cmd = f.replace("activate", "sphinx-build") + if not os.path.isfile(sphinx_cmd): + continue + + ver = self.get_sphinx_version(sphinx_cmd) + + if not ver: + venv_dir = f.replace("/bin/activate", "") + print(f"Warning: virtual environment {venv_dir} is not working.\n" \ + "Python version upgrade? Remove it with:\n\n" \ + "\trm -rf {venv_dir}\n\n") + else: + if self.need_sphinx and ver >= self.min_version: + return (f, ver) + elif parse_version(ver) > self.cur_version: + return (f, ver) + + return ("", ver) + + def recommend_sphinx_upgrade(self): + """ + Check if Sphinx needs to be upgraded. + + Returns a tuple with the higest available Sphinx version if found. + Otherwise, returns None to indicate either that no upgrade is needed + or no venv was found. + """ + + # Avoid running sphinx-builds from venv if cur_version is good + if self.cur_version and self.cur_version >= RECOMMENDED_VERSION: + self.latest_avail_ver = self.cur_version + return None + + # Get the highest version from sphinx_*/bin/sphinx-build and the + # corresponding command to activate the venv/virtenv + self.activate_cmd, self.venv_ver = self.get_virtenv() + + # Store the highest version from Sphinx existing virtualenvs + if self.activate_cmd and self.venv_ver > self.cur_version: + self.latest_avail_ver = self.venv_ver + else: + if self.cur_version: + self.latest_avail_ver = self.cur_version + else: + self.latest_avail_ver = (0, 0, 0) + + # As we don't know package version of Sphinx, and there's no + # virtual environments, don't check if upgrades are needed + if not self.virtualenv: + if not self.latest_avail_ver: + return None + + return self.latest_avail_ver + + # Either there are already a virtual env or a new one should be created + self.need_pip = 1 + + if not self.latest_avail_ver: + return None + + # Return if the reason is due to an upgrade or not + if self.latest_avail_ver != (0, 0, 0): + if self.latest_avail_ver < RECOMMENDED_VERSION: + self.rec_sphinx_upgrade = 1 + + return self.latest_avail_ver + + def recommend_package(self): + """ + Recommend installing Sphinx as a distro-specific package. + """ + + print("\n2) As a package with:") + + old_need = self.deps.need + old_optional = self.deps.optional + + self.pdf = False + self.deps.optional = 0 + old_verbose = self.verbose_warn_install + self.verbose_warn_install = 0 + + self.deps.clear_deps() + + self.deps.add_package("python-sphinx", DepManager.PYTHON_MANDATORY) + + cmd = self.get_install() + if cmd: + print(cmd) + + self.deps.need = old_need + self.deps.optional = old_optional + self.verbose_warn_install = old_verbose + + def recommend_sphinx_version(self, virtualenv_cmd): + """ + Provide recommendations for installing or upgrading Sphinx based + on current version. + + The logic here is complex, as it have to deal with different versions: + + - minimal supported version; + - minimal PDF version; + - recommended version. + + It also needs to work fine with both distro's package and + venv/virtualenv + """ + + if self.recommend_python: + print("\nPython version is incompatible with doc build.\n" \ + "Please upgrade it and re-run.\n") + return + + + # Version is OK. Nothing to do. + if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: + return + + if self.latest_avail_ver: + latest_avail_ver = ver_str(self.latest_avail_ver) + + if not self.need_sphinx: + # sphinx-build is present and its version is >= $min_version + + # only recommend enabling a newer virtenv version if makes sense. + if self.latest_avail_ver and self.latest_avail_ver > self.cur_version: + print(f"\nYou may also use the newer Sphinx version {latest_avail_ver} with:") + if f"{self.virtenv_prefix}" in os.getcwd(): + print("\tdeactivate") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + + if self.latest_avail_ver and self.latest_avail_ver >= RECOMMENDED_VERSION: + return + + if not self.virtualenv: + # No sphinx either via package or via virtenv. As we can't + # Compare the versions here, just return, recommending the + # user to install it from the package distro. + if not self.latest_avail_ver or self.latest_avail_ver == (0, 0, 0): + return + + # User doesn't want a virtenv recommendation, but he already + # installed one via virtenv with a newer version. + # So, print commands to enable it + if self.latest_avail_ver > self.cur_version: + print(f"\nYou may also use the Sphinx virtualenv version {latest_avail_ver} with:") + if f"{self.virtenv_prefix}" in os.getcwd(): + print("\tdeactivate") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + print("\n") + else: + if self.need_sphinx: + self.deps.need += 1 + + # Suggest newer versions if current ones are too old + if self.latest_avail_ver and self.latest_avail_ver >= self.min_version: + if self.latest_avail_ver >= RECOMMENDED_VERSION: + print(f"\nNeed to activate Sphinx (version {latest_avail_ver}) on virtualenv with:") + print(f"\t. {self.activate_cmd}") + self.deactivate_help() + return + + # Version is above the minimal required one, but may be + # below the recommended one. So, print warnings/notes + if self.latest_avail_ver < RECOMMENDED_VERSION: + print(f"Warning: It is recommended at least Sphinx version {RECOMMENDED_VERSION}.") + + # At this point, either it needs Sphinx or upgrade is recommended, + # both via pip + + if self.rec_sphinx_upgrade: + if not self.virtualenv: + print("Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n") + else: + print("To upgrade Sphinx, use:\n\n") + else: + print("\nSphinx needs to be installed either:\n1) via pip/pypi with:\n") + + if not virtualenv_cmd: + print(" Currently not possible.\n") + print(" Please upgrade Python to a newer version and run this script again") + else: + print(f"\t{virtualenv_cmd} {self.virtenv_dir}") + print(f"\t. {self.virtenv_dir}/bin/activate") + print(f"\tpip install -r {self.requirement_file}") + self.deactivate_help() + + if self.package_supported: + self.recommend_package() + + print("\n" \ + " Please note that Sphinx currentlys produce false-positive\n" \ + " warnings when the same name is used for more than one type (functions,\n" \ + " structs, enums,...). This is known Sphinx bug. For more details, see:\n" \ + "\thttps://github.com/sphinx-doc/sphinx/pull/8313") + + def check_needs(self): + """ + Main method that checks needed dependencies and provides + recommendations. + """ + self.python_cmd = sys.executable + + # Check if Sphinx is already accessible from current environment + self.check_sphinx(self.conf) + + if self.system_release: + print(f"Detected OS: {self.system_release}.") + else: + print("Unknown OS") + if self.cur_version != (0, 0, 0): + ver = ver_str(self.cur_version) + print(f"Sphinx version: {ver}\n") + + # Check the type of virtual env, depending on Python version + virtualenv_cmd = None + + if sys.version_info < MIN_PYTHON_VERSION: + min_ver = ver_str(MIN_PYTHON_VERSION) + print(f"ERROR: at least python {min_ver} is required to build the kernel docs") + self.need_sphinx = 1 + + self.venv_ver = self.recommend_sphinx_upgrade() + + if self.need_pip: + if sys.version_info < MIN_PYTHON_VERSION: + self.need_pip = False + print("Warning: python version is not supported.") + + else: + virtualenv_cmd = f"{self.python_cmd} -m venv" + self.check_python_module("ensurepip") + + # Check for needed programs/tools + self.check_perl_module("Pod::Usage", DepManager.SYSTEM_MANDATORY) + + self.check_program("make", DepManager.SYSTEM_MANDATORY) + self.check_program("gcc", DepManager.SYSTEM_MANDATORY) + + self.check_program("dot", DepManager.SYSTEM_OPTIONAL) + self.check_program("convert", DepManager.SYSTEM_OPTIONAL) + + self.check_python_module("yaml") + + if self.pdf: + self.check_program("xelatex", DepManager.PDF_MANDATORY) + self.check_program("rsvg-convert", DepManager.PDF_MANDATORY) + self.check_program("latexmk", DepManager.PDF_MANDATORY) + + # Do distro-specific checks and output distro-install commands + cmd = self.get_install() + if cmd: + print(cmd) + + # If distro requires some special instructions, print here. + # Please notice that get_install() needs to be called first. + if self.distro_msg: + print("\n" + self.distro_msg) + + if not self.python_cmd: + if self.need == 1: + sys.exit("Can't build as 1 mandatory dependency is missing") + elif self.need: + sys.exit(f"Can't build as {self.need} mandatory dependencies are missing") + + # Check if sphinx-build is called sphinx-build-3 + if self.need_symlink: + sphinx_path = self.which("sphinx-build-3") + if sphinx_path: + print(f"\tsudo ln -sf {sphinx_path} /usr/bin/sphinx-build\n") + + self.recommend_sphinx_version(virtualenv_cmd) + print("") + + if not self.deps.optional: + print("All optional dependencies are met.") + + if self.deps.need == 1: + sys.exit("Can't build as 1 mandatory dependency is missing") + elif self.deps.need: + sys.exit(f"Can't build as {self.deps.need} mandatory dependencies are missing") + + print("Needed package dependencies are met.") + +DESCRIPTION = """ +Process some flags related to Sphinx installation and documentation build. +""" + + +def main(): + """Main function""" + parser = argparse.ArgumentParser(description=DESCRIPTION) + + parser.add_argument( + "--no-virtualenv", + action="store_false", + dest="virtualenv", + help="Recommend installing Sphinx instead of using a virtualenv", + ) + + parser.add_argument( + "--no-pdf", + action="store_false", + dest="pdf", + help="Don't check for dependencies required to build PDF docs", + ) + + parser.add_argument( + "--version-check", + action="store_true", + dest="version_check", + help="If version is compatible, don't check for missing dependencies", + ) + + args = parser.parse_args() + + checker = SphinxDependencyChecker(args) + + checker.check_python() + checker.check_needs() + +# Call main if not used as module +if __name__ == "__main__": + main() diff --git a/scripts/sphinx-pre-install.py b/scripts/sphinx-pre-install.py deleted file mode 100755 index 7dfe5c2a6cc2..000000000000 --- a/scripts/sphinx-pre-install.py +++ /dev/null @@ -1,1565 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-or-later -# Copyright (c) 2017-2025 Mauro Carvalho Chehab -# -# pylint: disable=C0103,C0114,C0115,C0116,C0301,C0302 -# pylint: disable=R0902,R0904,R0911,R0912,R0914,R0915,R1705,R1710,E1121 - -# Note: this script requires at least Python 3.6 to run. -# Don't add changes not compatible with it, it is meant to report -# incompatible python versions. - -""" -Dependency checker for Sphinx documentation Kernel build. - -This module provides tools to check for all required dependencies needed to -build documentation using Sphinx, including system packages, Python modules -and LaTeX packages for PDF generation. - -It detect packages for a subset of Linux distributions used by Kernel -maintainers, showing hints and missing dependencies. - -The main class SphinxDependencyChecker handles the dependency checking logic -and provides recommendations for installing missing packages. It supports both -system package installations and Python virtual environments. By default, -system pacage install is recommended. -""" - -import argparse -import os -import re -import subprocess -import sys -from glob import glob - - -def parse_version(version): - """Convert a major.minor.patch version into a tuple""" - return tuple(int(x) for x in version.split(".")) - - -def ver_str(version): - """Returns a version tuple as major.minor.patch""" - - return ".".join([str(x) for x in version]) - - -RECOMMENDED_VERSION = parse_version("3.4.3") -MIN_PYTHON_VERSION = parse_version("3.7") - - -class DepManager: - """ - Manage package dependencies. There are three types of dependencies: - - - System: dependencies required for docs build; - - Python: python dependencies for a native distro Sphinx install; - - PDF: dependencies needed by PDF builds. - - Each dependency can be mandatory or optional. Not installing an optional - dependency won't break the build, but will cause degradation at the - docs output. - """ - - # Internal types of dependencies. Don't use them outside DepManager class. - _SYS_TYPE = 0 - _PHY_TYPE = 1 - _PDF_TYPE = 2 - - # Dependencies visible outside the class. - # The keys are tuple with: (type, is_mandatory flag). - # - # Currently we're not using all optional dep types. Yet, we'll keep all - # possible combinations here. They're not many, and that makes easier - # if later needed and for the name() method below - - SYSTEM_MANDATORY = (_SYS_TYPE, True) - PYTHON_MANDATORY = (_PHY_TYPE, True) - PDF_MANDATORY = (_PDF_TYPE, True) - - SYSTEM_OPTIONAL = (_SYS_TYPE, False) - PYTHON_OPTIONAL = (_PHY_TYPE, False) - PDF_OPTIONAL = (_PDF_TYPE, True) - - def __init__(self, pdf): - """ - Initialize internal vars: - - - missing: missing dependencies list, containing a distro-independent - name for a missing dependency and its type. - - missing_pkg: ancillary dict containing missing dependencies in - distro namespace, organized by type. - - need: total number of needed dependencies. Never cleaned. - - optional: total number of optional dependencies. Never cleaned. - - pdf: Is PDF support enabled? - """ - self.missing = {} - self.missing_pkg = {} - self.need = 0 - self.optional = 0 - self.pdf = pdf - - @staticmethod - def name(dtype): - """ - Ancillary routine to output a warn/error message reporting - missing dependencies. - """ - if dtype[0] == DepManager._SYS_TYPE: - msg = "build" - elif dtype[0] == DepManager._PHY_TYPE: - msg = "Python" - else: - msg = "PDF" - - if dtype[1]: - return f"ERROR: {msg} mandatory deps missing" - else: - return f"Warning: {msg} optional deps missing" - - @staticmethod - def is_optional(dtype): - """Ancillary routine to report if a dependency is optional""" - return not dtype[1] - - @staticmethod - def is_pdf(dtype): - """Ancillary routine to report if a dependency is for PDF generation""" - if dtype[0] == DepManager._PDF_TYPE: - return True - - return False - - def add_package(self, package, dtype): - """ - Add a package at the self.missing() dictionary. - Doesn't update missing_pkg. - """ - is_optional = DepManager.is_optional(dtype) - self.missing[package] = dtype - if is_optional: - self.optional += 1 - else: - self.need += 1 - - def del_package(self, package): - """ - Remove a package at the self.missing() dictionary. - Doesn't update missing_pkg. - """ - if package in self.missing: - del self.missing[package] - - def clear_deps(self): - """ - Clear dependencies without changing needed/optional. - - This is an ackward way to have a separate section to recommend - a package after system main dependencies. - - TODO: rework the logic to prevent needing it. - """ - - self.missing = {} - self.missing_pkg = {} - - def check_missing(self, progs): - """ - Update self.missing_pkg, using progs dict to convert from the - agnostic package name to distro-specific one. - - Returns an string with the packages to be installed, sorted and - with eventual duplicates removed. - """ - - self.missing_pkg = {} - - for prog, dtype in sorted(self.missing.items()): - # At least on some LTS distros like CentOS 7, texlive doesn't - # provide all packages we need. When such distros are - # detected, we have to disable PDF output. - # - # So, we need to ignore the packages that distros would - # need for LaTeX to work - if DepManager.is_pdf(dtype) and not self.pdf: - self.optional -= 1 - continue - - if not dtype in self.missing_pkg: - self.missing_pkg[dtype] = [] - - self.missing_pkg[dtype].append(progs.get(prog, prog)) - - install = [] - for dtype, pkgs in self.missing_pkg.items(): - install += pkgs - - return " ".join(sorted(set(install))) - - def warn_install(self): - """ - Emit warnings/errors related to missing packages. - """ - - output_msg = "" - - for dtype in sorted(self.missing_pkg.keys()): - progs = " ".join(sorted(set(self.missing_pkg[dtype]))) - - try: - name = DepManager.name(dtype) - output_msg += f'{name}:\t{progs}\n' - except KeyError: - raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}") - - if output_msg: - print(f"\n{output_msg}") - -class AncillaryMethods: - """ - Ancillary methods that checks for missing dependencies for different - types of types, like binaries, python modules, rpm deps, etc. - """ - - @staticmethod - def which(prog): - """ - Our own implementation of which(). We could instead use - shutil.which(), but this function is simple enough. - Probably faster to use this implementation than to import shutil. - """ - for path in os.environ.get("PATH", "").split(":"): - full_path = os.path.join(path, prog) - if os.access(full_path, os.X_OK): - return full_path - - return None - - @staticmethod - def get_python_version(cmd): - """ - Get python version from a Python binary. As we need to detect if - are out there newer python binaries, we can't rely on sys.release here. - """ - - result = SphinxDependencyChecker.run([cmd, "--version"], - capture_output=True, text=True) - version = result.stdout.strip() - - match = re.search(r"(\d+\.\d+\.\d+)", version) - if match: - return parse_version(match.group(1)) - - print(f"Can't parse version {version}") - return (0, 0, 0) - - @staticmethod - def find_python(): - """ - Detect if are out there any python 3.xy version newer than the - current one. - - Note: this routine is limited to up to 2 digits for python3. We - may need to update it one day, hopefully on a distant future. - """ - patterns = [ - "python3.[0-9]", - "python3.[0-9][0-9]", - ] - - # Seek for a python binary newer than MIN_PYTHON_VERSION - for path in os.getenv("PATH", "").split(":"): - for pattern in patterns: - for cmd in glob(os.path.join(path, pattern)): - if os.path.isfile(cmd) and os.access(cmd, os.X_OK): - version = SphinxDependencyChecker.get_python_version(cmd) - if version >= MIN_PYTHON_VERSION: - return cmd - - @staticmethod - def check_python(): - """ - Check if the current python binary satisfies our minimal requirement - for Sphinx build. If not, re-run with a newer version if found. - """ - cur_ver = sys.version_info[:3] - if cur_ver >= MIN_PYTHON_VERSION: - ver = ver_str(cur_ver) - print(f"Python version: {ver}") - - # This could be useful for debugging purposes - if SphinxDependencyChecker.which("docutils"): - result = SphinxDependencyChecker.run(["docutils", "--version"], - capture_output=True, text=True) - ver = result.stdout.strip() - match = re.search(r"(\d+\.\d+\.\d+)", ver) - if match: - ver = match.group(1) - - print(f"Docutils version: {ver}") - - return - - python_ver = ver_str(cur_ver) - - new_python_cmd = SphinxDependencyChecker.find_python() - if not new_python_cmd: - print(f"ERROR: Python version {python_ver} is not spported anymore\n") - print(" Can't find a new version. This script may fail") - return - - # Restart script using the newer version - script_path = os.path.abspath(sys.argv[0]) - args = [new_python_cmd, script_path] + sys.argv[1:] - - print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") - - try: - os.execv(new_python_cmd, args) - except OSError as e: - sys.exit(f"Failed to restart with {new_python_cmd}: {e}") - - @staticmethod - def run(*args, **kwargs): - """ - Excecute a command, hiding its output by default. - Preserve comatibility with older Python versions. - """ - - capture_output = kwargs.pop('capture_output', False) - - if capture_output: - if 'stdout' not in kwargs: - kwargs['stdout'] = subprocess.PIPE - if 'stderr' not in kwargs: - kwargs['stderr'] = subprocess.PIPE - else: - if 'stdout' not in kwargs: - kwargs['stdout'] = subprocess.DEVNULL - if 'stderr' not in kwargs: - kwargs['stderr'] = subprocess.DEVNULL - - # Don't break with older Python versions - if 'text' in kwargs and sys.version_info < (3, 7): - kwargs['universal_newlines'] = kwargs.pop('text') - - return subprocess.run(*args, **kwargs) - -class MissingCheckers(AncillaryMethods): - """ - Contains some ancillary checkers for different types of binaries and - package managers. - """ - - def __init__(self, args, texlive): - """ - Initialize its internal variables - """ - self.pdf = args.pdf - self.virtualenv = args.virtualenv - self.version_check = args.version_check - self.texlive = texlive - - self.min_version = (0, 0, 0) - self.cur_version = (0, 0, 0) - - self.deps = DepManager(self.pdf) - - self.need_symlink = 0 - self.need_sphinx = 0 - - self.verbose_warn_install = 1 - - self.virtenv_dir = "" - self.install = "" - self.python_cmd = "" - - self.virtenv_prefix = ["sphinx_", "Sphinx_" ] - - def check_missing_file(self, files, package, dtype): - """ - Does the file exists? If not, add it to missing dependencies. - """ - for f in files: - if os.path.exists(f): - return - self.deps.add_package(package, dtype) - - def check_program(self, prog, dtype): - """ - Does the program exists and it is at the PATH? - If not, add it to missing dependencies. - """ - found = self.which(prog) - if found: - return found - - self.deps.add_package(prog, dtype) - - return None - - def check_perl_module(self, prog, dtype): - """ - Does perl have a dependency? Is it available? - If not, add it to missing dependencies. - - Right now, we still need Perl for doc build, as it is required - by some tools called at docs or kernel build time, like: - - scripts/documentation-file-ref-check - - Also, checkpatch is on Perl. - """ - - # While testing with lxc download template, one of the - # distros (Oracle) didn't have perl - nor even an option to install - # before installing oraclelinux-release-el9 package. - # - # Check it before running an error. If perl is not there, - # add it as a mandatory package, as some parts of the doc builder - # needs it. - if not self.which("perl"): - self.deps.add_package("perl", DepManager.SYSTEM_MANDATORY) - self.deps.add_package(prog, dtype) - return - - try: - self.run(["perl", f"-M{prog}", "-e", "1"], check=True) - except subprocess.CalledProcessError: - self.deps.add_package(prog, dtype) - - def check_python_module(self, module, is_optional=False): - """ - Does a python module exists outside venv? If not, add it to missing - dependencies. - """ - if is_optional: - dtype = DepManager.PYTHON_OPTIONAL - else: - dtype = DepManager.PYTHON_MANDATORY - - try: - self.run([self.python_cmd, "-c", f"import {module}"], check=True) - except subprocess.CalledProcessError: - self.deps.add_package(module, dtype) - - def check_rpm_missing(self, pkgs, dtype): - """ - Does a rpm package exists? If not, add it to missing dependencies. - """ - for prog in pkgs: - try: - self.run(["rpm", "-q", prog], check=True) - except subprocess.CalledProcessError: - self.deps.add_package(prog, dtype) - - def check_pacman_missing(self, pkgs, dtype): - """ - Does a pacman package exists? If not, add it to missing dependencies. - """ - for prog in pkgs: - try: - self.run(["pacman", "-Q", prog], check=True) - except subprocess.CalledProcessError: - self.deps.add_package(prog, dtype) - - def check_missing_tex(self, is_optional=False): - """ - Does a LaTeX package exists? If not, add it to missing dependencies. - """ - if is_optional: - dtype = DepManager.PDF_OPTIONAL - else: - dtype = DepManager.PDF_MANDATORY - - kpsewhich = self.which("kpsewhich") - for prog, package in self.texlive.items(): - - # If kpsewhich is not there, just add it to deps - if not kpsewhich: - self.deps.add_package(package, dtype) - continue - - # Check if the package is needed - try: - result = self.run( - [kpsewhich, prog], stdout=subprocess.PIPE, text=True, check=True - ) - - # Didn't find. Add it - if not result.stdout.strip(): - self.deps.add_package(package, dtype) - - except subprocess.CalledProcessError: - # kpsewhich returned an error. Add it, just in case - self.deps.add_package(package, dtype) - - def get_sphinx_fname(self): - """ - Gets the binary filename for sphinx-build. - """ - if "SPHINXBUILD" in os.environ: - return os.environ["SPHINXBUILD"] - - fname = "sphinx-build" - if self.which(fname): - return fname - - fname = "sphinx-build-3" - if self.which(fname): - self.need_symlink = 1 - return fname - - return "" - - def get_sphinx_version(self, cmd): - """ - Gets sphinx-build version. - """ - try: - result = self.run([cmd, "--version"], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, check=True) - except (subprocess.CalledProcessError, FileNotFoundError): - return None - - for line in result.stdout.split("\n"): - match = re.match(r"^sphinx-build\s+([\d\.]+)(?:\+(?:/[\da-f]+)|b\d+)?\s*$", line) - if match: - return parse_version(match.group(1)) - - match = re.match(r"^Sphinx.*\s+([\d\.]+)\s*$", line) - if match: - return parse_version(match.group(1)) - - def check_sphinx(self, conf): - """ - Checks Sphinx minimal requirements - """ - try: - with open(conf, "r", encoding="utf-8") as f: - for line in f: - match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line) - if match: - self.min_version = parse_version(match.group(1)) - break - except IOError: - sys.exit(f"Can't open {conf}") - - if not self.min_version: - sys.exit(f"Can't get needs_sphinx version from {conf}") - - self.virtenv_dir = self.virtenv_prefix[0] + "latest" - - sphinx = self.get_sphinx_fname() - if not sphinx: - self.need_sphinx = 1 - return - - self.cur_version = self.get_sphinx_version(sphinx) - if not self.cur_version: - sys.exit(f"{sphinx} didn't return its version") - - if self.cur_version < self.min_version: - curver = ver_str(self.cur_version) - minver = ver_str(self.min_version) - - print(f"ERROR: Sphinx version is {curver}. It should be >= {minver}") - self.need_sphinx = 1 - return - - # On version check mode, just assume Sphinx has all mandatory deps - if self.version_check and self.cur_version >= RECOMMENDED_VERSION: - sys.exit(0) - - def catcheck(self, filename): - """ - Reads a file if it exists, returning as string. - If not found, returns an empty string. - """ - if os.path.exists(filename): - with open(filename, "r", encoding="utf-8") as f: - return f.read().strip() - return "" - - def get_system_release(self): - """ - Determine the system type. There's no unique way that would work - with all distros with a minimal package install. So, several - methods are used here. - - By default, it will use lsb_release function. If not available, it will - fail back to reading the known different places where the distro name - is stored. - - Several modern distros now have /etc/os-release, which usually have - a decent coverage. - """ - - system_release = "" - - if self.which("lsb_release"): - result = self.run(["lsb_release", "-d"], capture_output=True, text=True) - system_release = result.stdout.replace("Description:", "").strip() - - release_files = [ - "/etc/system-release", - "/etc/redhat-release", - "/etc/lsb-release", - "/etc/gentoo-release", - ] - - if not system_release: - for f in release_files: - system_release = self.catcheck(f) - if system_release: - break - - # This seems more common than LSB these days - if not system_release: - os_var = {} - try: - with open("/etc/os-release", "r", encoding="utf-8") as f: - for line in f: - match = re.match(r"^([\w\d\_]+)=\"?([^\"]*)\"?\n", line) - if match: - os_var[match.group(1)] = match.group(2) - - system_release = os_var.get("NAME", "") - if "VERSION_ID" in os_var: - system_release += " " + os_var["VERSION_ID"] - elif "VERSION" in os_var: - system_release += " " + os_var["VERSION"] - except IOError: - pass - - if not system_release: - system_release = self.catcheck("/etc/issue") - - system_release = system_release.strip() - - return system_release - -class SphinxDependencyChecker(MissingCheckers): - """ - Main class for checking Sphinx documentation build dependencies. - - - Check for missing system packages; - - Check for missing Python modules; - - Check for missing LaTeX packages needed by PDF generation; - - Propose Sphinx install via Python Virtual environment; - - Propose Sphinx install via distro-specific package install. - """ - def __init__(self, args): - """Initialize checker variables""" - - # List of required texlive packages on Fedora and OpenSuse - texlive = { - "amsfonts.sty": "texlive-amsfonts", - "amsmath.sty": "texlive-amsmath", - "amssymb.sty": "texlive-amsfonts", - "amsthm.sty": "texlive-amscls", - "anyfontsize.sty": "texlive-anyfontsize", - "atbegshi.sty": "texlive-oberdiek", - "bm.sty": "texlive-tools", - "capt-of.sty": "texlive-capt-of", - "cmap.sty": "texlive-cmap", - "ctexhook.sty": "texlive-ctex", - "ecrm1000.tfm": "texlive-ec", - "eqparbox.sty": "texlive-eqparbox", - "eu1enc.def": "texlive-euenc", - "fancybox.sty": "texlive-fancybox", - "fancyvrb.sty": "texlive-fancyvrb", - "float.sty": "texlive-float", - "fncychap.sty": "texlive-fncychap", - "footnote.sty": "texlive-mdwtools", - "framed.sty": "texlive-framed", - "luatex85.sty": "texlive-luatex85", - "multirow.sty": "texlive-multirow", - "needspace.sty": "texlive-needspace", - "palatino.sty": "texlive-psnfss", - "parskip.sty": "texlive-parskip", - "polyglossia.sty": "texlive-polyglossia", - "tabulary.sty": "texlive-tabulary", - "threeparttable.sty": "texlive-threeparttable", - "titlesec.sty": "texlive-titlesec", - "ucs.sty": "texlive-ucs", - "upquote.sty": "texlive-upquote", - "wrapfig.sty": "texlive-wrapfig", - } - - super().__init__(args, texlive) - - self.need_pip = 0 - self.rec_sphinx_upgrade = 0 - - self.system_release = self.get_system_release() - self.activate_cmd = "" - - # Some distros may not have a Sphinx shipped package compatible with - # our minimal requirements - self.package_supported = True - - # Recommend a new python version - self.recommend_python = None - - # Certain hints are meant to be shown only once - self.distro_msg = None - - self.latest_avail_ver = (0, 0, 0) - self.venv_ver = (0, 0, 0) - - prefix = os.environ.get("srctree", ".") + "/" - - self.conf = prefix + "Documentation/conf.py" - self.requirement_file = prefix + "Documentation/sphinx/requirements.txt" - - def get_install_progs(self, progs, cmd, extra=None): - """ - Check for missing dependencies using the provided program mapping. - - The actual distro-specific programs are mapped via progs argument. - """ - install = self.deps.check_missing(progs) - - if self.verbose_warn_install: - self.deps.warn_install() - - if not install: - return - - if cmd: - if self.verbose_warn_install: - msg = "You should run:" - else: - msg = "" - - if extra: - msg += "\n\t" + extra.replace("\n", "\n\t") - - return(msg + "\n\tsudo " + cmd + " " + install) - - return None - - # - # Distro-specific hints methods - # - - def give_debian_hints(self): - """ - Provide package installation hints for Debian-based distros. - """ - progs = { - "Pod::Usage": "perl-modules", - "convert": "imagemagick", - "dot": "graphviz", - "ensurepip": "python3-venv", - "python-sphinx": "python3-sphinx", - "rsvg-convert": "librsvg2-bin", - "virtualenv": "virtualenv", - "xelatex": "texlive-xetex", - "yaml": "python3-yaml", - } - - if self.pdf: - pdf_pkgs = { - "texlive-lang-chinese": [ - "/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty", - ], - "fonts-dejavu": [ - "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", - ], - "fonts-noto-cjk": [ - "/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc", - ], - } - - for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, DepManager.PDF_MANDATORY) - - self.check_program("dvipng", DepManager.PDF_MANDATORY) - - return self.get_install_progs(progs, "apt-get install") - - def give_redhat_hints(self): - """ - Provide package installation hints for RedHat-based distros - (Fedora, RHEL and RHEL-based variants). - """ - progs = { - "Pod::Usage": "perl-Pod-Usage", - "convert": "ImageMagick", - "dot": "graphviz", - "python-sphinx": "python3-sphinx", - "rsvg-convert": "librsvg2-tools", - "virtualenv": "python3-virtualenv", - "xelatex": "texlive-xetex-bin", - "yaml": "python3-pyyaml", - } - - fedora_tex_pkgs = [ - "dejavu-sans-fonts", - "dejavu-sans-mono-fonts", - "dejavu-serif-fonts", - "texlive-collection-fontsrecommended", - "texlive-collection-latex", - "texlive-xecjk", - ] - - fedora = False - rel = None - - match = re.search(r"(release|Linux)\s+(\d+)", self.system_release) - if match: - rel = int(match.group(2)) - - if not rel: - print("Couldn't identify release number") - noto_sans_redhat = None - self.pdf = False - elif re.search("Fedora", self.system_release): - # Fedora 38 and upper use this CJK font - - noto_sans_redhat = "google-noto-sans-cjk-fonts" - fedora = True - else: - # Almalinux, CentOS, RHEL, ... - - # at least up to version 9 (and Fedora < 38), that's the CJK font - noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts" - - progs["virtualenv"] = "python-virtualenv" - - if not rel or rel < 8: - print("ERROR: Distro not supported. Too old?") - return - - # RHEL 8 uses Python 3.6, which is not compatible with - # the build system anymore. Suggest Python 3.11 - if rel == 8: - self.deps.add_package("python39", DepManager.SYSTEM_MANDATORY) - self.recommend_python = True - - if not self.distro_msg: - self.distro_msg = \ - "Note: RHEL-based distros typically require extra repositories.\n" \ - "For most, enabling epel and crb are enough:\n" \ - "\tsudo dnf install -y epel-release\n" \ - "\tsudo dnf config-manager --set-enabled crb\n" \ - "Yet, some may have other required repositories. Those commands could be useful:\n" \ - "\tsudo dnf repolist all\n" \ - "\tsudo dnf repoquery --available --info \n" \ - "\tsudo dnf config-manager --set-enabled '*' # enable all - probably not what you want" - - if self.pdf: - pdf_pkgs = [ - "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc", - ] - - self.check_missing_file(pdf_pkgs, noto_sans_redhat, DepManager.PDF_MANDATORY) - - self.check_rpm_missing(fedora_tex_pkgs, DepManager.PDF_MANDATORY) - - self.check_missing_tex(DepManager.PDF_MANDATORY) - - # There's no texlive-ctex on RHEL 8 repositories. This will - # likely affect CJK pdf build only. - if not fedora and rel == 8: - self.deps.del_package("texlive-ctex") - - return self.get_install_progs(progs, "dnf install") - - def give_opensuse_hints(self): - """ - Provide package installation hints for openSUSE-based distros - (Leap and Tumbleweed). - """ - progs = { - "Pod::Usage": "perl-Pod-Usage", - "convert": "ImageMagick", - "dot": "graphviz", - "python-sphinx": "python3-sphinx", - "virtualenv": "python3-virtualenv", - "xelatex": "texlive-xetex-bin", - "yaml": "python3-pyyaml", - } - - suse_tex_pkgs = [ - "texlive-babel-english", - "texlive-caption", - "texlive-colortbl", - "texlive-courier", - "texlive-dvips", - "texlive-helvetic", - "texlive-makeindex", - "texlive-metafont", - "texlive-metapost", - "texlive-palatino", - "texlive-preview", - "texlive-times", - "texlive-zapfchan", - "texlive-zapfding", - ] - - progs["latexmk"] = "texlive-latexmk-bin" - - match = re.search(r"(Leap)\s+(\d+).(\d)", self.system_release) - if match: - rel = int(match.group(2)) - - # Leap 15.x uses Python 3.6, which is not compatible with - # the build system anymore. Suggest Python 3.11 - if rel == 15: - if not self.which(self.python_cmd): - self.recommend_python = True - self.deps.add_package(self.python_cmd, DepManager.SYSTEM_MANDATORY) - - progs.update({ - "python-sphinx": "python311-Sphinx", - "virtualenv": "python311-virtualenv", - "yaml": "python311-PyYAML", - }) - else: - # Tumbleweed defaults to Python 3.11 - - progs.update({ - "python-sphinx": "python313-Sphinx", - "virtualenv": "python313-virtualenv", - "yaml": "python313-PyYAML", - }) - - # FIXME: add support for installing CJK fonts - # - # I tried hard, but was unable to find a way to install - # "Noto Sans CJK SC" on openSUSE - - if self.pdf: - self.check_rpm_missing(suse_tex_pkgs, DepManager.PDF_MANDATORY) - if self.pdf: - self.check_missing_tex() - - return self.get_install_progs(progs, "zypper install --no-recommends") - - def give_mageia_hints(self): - """ - Provide package installation hints for Mageia and OpenMandriva. - """ - progs = { - "Pod::Usage": "perl-Pod-Usage", - "convert": "ImageMagick", - "dot": "graphviz", - "python-sphinx": "python3-sphinx", - "rsvg-convert": "librsvg2", - "virtualenv": "python3-virtualenv", - "xelatex": "texlive", - "yaml": "python3-yaml", - } - - tex_pkgs = [ - "texlive-fontsextra", - ] - - if re.search(r"OpenMandriva", self.system_release): - packager_cmd = "dnf install" - noto_sans = "noto-sans-cjk-fonts" - tex_pkgs = ["texlive-collection-fontsextra"] - - # Tested on OpenMandriva Lx 4.3 - progs["convert"] = "imagemagick" - progs["yaml"] = "python-pyyaml" - - else: - packager_cmd = "urpmi" - noto_sans = "google-noto-sans-cjk-ttc-fonts" - - progs["latexmk"] = "texlive-collection-basic" - - if self.pdf: - pdf_pkgs = [ - "/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/TTF/NotoSans-Regular.ttf", - ] - - self.check_missing_file(pdf_pkgs, noto_sans, DepManager.PDF_MANDATORY) - self.check_rpm_missing(tex_pkgs, DepManager.PDF_MANDATORY) - - return self.get_install_progs(progs, packager_cmd) - - def give_arch_linux_hints(self): - """ - Provide package installation hints for ArchLinux. - """ - progs = { - "convert": "imagemagick", - "dot": "graphviz", - "latexmk": "texlive-core", - "rsvg-convert": "extra/librsvg", - "virtualenv": "python-virtualenv", - "xelatex": "texlive-xetex", - "yaml": "python-yaml", - } - - archlinux_tex_pkgs = [ - "texlive-core", - "texlive-latexextra", - "ttf-dejavu", - ] - - if self.pdf: - self.check_pacman_missing(archlinux_tex_pkgs, - DepManager.PDF_MANDATORY) - - self.check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], - "noto-fonts-cjk", - DepManager.PDF_MANDATORY) - - - return self.get_install_progs(progs, "pacman -S") - - def give_gentoo_hints(self): - """ - Provide package installation hints for Gentoo. - """ - progs = { - "convert": "media-gfx/imagemagick", - "dot": "media-gfx/graphviz", - "rsvg-convert": "gnome-base/librsvg", - "virtualenv": "dev-python/virtualenv", - "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", - "yaml": "dev-python/pyyaml", - "python-sphinx": "dev-python/sphinx", - } - - if self.pdf: - pdf_pkgs = { - "media-fonts/dejavu": [ - "/usr/share/fonts/dejavu/DejaVuSans.ttf", - ], - "media-fonts/noto-cjk": [ - "/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", - "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc", - ], - } - for package, files in pdf_pkgs.items(): - self.check_missing_file(files, package, DepManager.PDF_MANDATORY) - - # Handling dependencies is a nightmare, as Gentoo refuses to emerge - # some packages if there's no package.use file describing them. - # To make it worse, compilation flags shall also be present there - # for some packages. If USE is not perfect, error/warning messages - # like those are shown: - # - # !!! The following binary packages have been ignored due to non matching USE: - # - # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_13 qt6 svg - # =media-gfx/graphviz-12.2.1-r1 X pdf python_single_target_python3_12 -python_single_target_python3_13 qt6 svg - # =media-gfx/graphviz-12.2.1-r1 X pdf qt6 svg - # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 qt6 svg - # =media-gfx/graphviz-12.2.1-r1 X pdf -python_single_target_python3_10 python_single_target_python3_12 -python_single_target_python3_13 qt6 svg - # =media-fonts/noto-cjk-20190416 X - # =app-text/texlive-core-2024-r1 X cjk -xetex - # =app-text/texlive-core-2024-r1 X -xetex - # =app-text/texlive-core-2024-r1 -xetex - # =dev-libs/zziplib-0.13.79-r1 sdl - # - # And will ignore such packages, installing the remaining ones. That - # affects mostly the image extension and PDF generation. - - # Package dependencies and the minimal needed args: - portages = { - "graphviz": "media-gfx/graphviz", - "imagemagick": "media-gfx/imagemagick", - "media-libs": "media-libs/harfbuzz icu", - "media-fonts": "media-fonts/noto-cjk", - "texlive": "app-text/texlive-core xetex", - "zziblib": "dev-libs/zziplib sdl", - } - - extra_cmds = "" - if not self.distro_msg: - self.distro_msg = "Note: Gentoo requires package.use to be adjusted before emerging packages" - - use_base = "/etc/portage/package.use" - files = glob(f"{use_base}/*") - - for fname, portage in portages.items(): - install = False - - while install is False: - if not files: - # No files under package.usage. Install all - install = True - break - - args = portage.split(" ") - - name = args.pop(0) - - cmd = ["grep", "-l", "-E", rf"^{name}\b" ] + files - result = self.run(cmd, stdout=subprocess.PIPE, text=True) - if result.returncode or not result.stdout.strip(): - # File containing portage name not found - install = True - break - - # Ensure that needed USE flags are present - if args: - match_fname = result.stdout.strip() - with open(match_fname, 'r', encoding='utf8', - errors='backslashreplace') as fp: - for line in fp: - for arg in args: - if arg.startswith("-"): - continue - - if not re.search(rf"\s*{arg}\b", line): - # Needed file argument not found - install = True - break - - # Everything looks ok, don't install - break - - # emit a code to setup missing USE - if install: - extra_cmds += (f"sudo su -c 'echo \"{portage}\" > {use_base}/{fname}'\n") - - # Now, we can use emerge and let it respect USE - return self.get_install_progs(progs, - "emerge --ask --changed-use --binpkg-respect-use=y", - extra_cmds) - - def get_install(self): - """ - OS-specific hints logic. Seeks for a hinter. If found, use it to - provide package-manager specific install commands. - - Otherwise, outputs install instructions for the meta-packages. - - Returns a string with the command to be executed to install the - the needed packages, if distro found. Otherwise, return just a - list of packages that require installation. - """ - os_hints = { - re.compile("Red Hat Enterprise Linux"): self.give_redhat_hints, - re.compile("Fedora"): self.give_redhat_hints, - re.compile("AlmaLinux"): self.give_redhat_hints, - re.compile("Amazon Linux"): self.give_redhat_hints, - re.compile("CentOS"): self.give_redhat_hints, - re.compile("openEuler"): self.give_redhat_hints, - re.compile("Oracle Linux Server"): self.give_redhat_hints, - re.compile("Rocky Linux"): self.give_redhat_hints, - re.compile("Springdale Open Enterprise"): self.give_redhat_hints, - - re.compile("Ubuntu"): self.give_debian_hints, - re.compile("Debian"): self.give_debian_hints, - re.compile("Devuan"): self.give_debian_hints, - re.compile("Kali"): self.give_debian_hints, - re.compile("Mint"): self.give_debian_hints, - - re.compile("openSUSE"): self.give_opensuse_hints, - - re.compile("Mageia"): self.give_mageia_hints, - re.compile("OpenMandriva"): self.give_mageia_hints, - - re.compile("Arch Linux"): self.give_arch_linux_hints, - re.compile("Gentoo"): self.give_gentoo_hints, - } - - # If the OS is detected, use per-OS hint logic - for regex, os_hint in os_hints.items(): - if regex.search(self.system_release): - return os_hint() - - # - # Fall-back to generic hint code for other distros - # That's far from ideal, specially for LaTeX dependencies. - # - progs = {"sphinx-build": "sphinx"} - if self.pdf: - self.check_missing_tex() - - self.distro_msg = \ - f"I don't know distro {self.system_release}.\n" \ - "So, I can't provide you a hint with the install procedure.\n" \ - "There are likely missing dependencies.\n" - - return self.get_install_progs(progs, None) - - # - # Common dependencies - # - def deactivate_help(self): - """ - Print a helper message to disable a virtual environment. - """ - - print("\n If you want to exit the virtualenv, you can use:") - print("\tdeactivate") - - def get_virtenv(self): - """ - Give a hint about how to activate an already-existing virtual - environment containing sphinx-build. - - Returns a tuble with (activate_cmd_path, sphinx_version) with - the newest available virtual env. - """ - - cwd = os.getcwd() - - activates = [] - - # Add all sphinx prefixes with possible version numbers - for p in self.virtenv_prefix: - activates += glob(f"{cwd}/{p}[0-9]*/bin/activate") - - activates.sort(reverse=True, key=str.lower) - - # Place sphinx_latest first, if it exists - for p in self.virtenv_prefix: - activates = glob(f"{cwd}/{p}*latest/bin/activate") + activates - - ver = (0, 0, 0) - for f in activates: - # Discard too old Sphinx virtual environments - match = re.search(r"(\d+)\.(\d+)\.(\d+)", f) - if match: - ver = (int(match.group(1)), int(match.group(2)), int(match.group(3))) - - if ver < self.min_version: - continue - - sphinx_cmd = f.replace("activate", "sphinx-build") - if not os.path.isfile(sphinx_cmd): - continue - - ver = self.get_sphinx_version(sphinx_cmd) - - if not ver: - venv_dir = f.replace("/bin/activate", "") - print(f"Warning: virtual environment {venv_dir} is not working.\n" \ - "Python version upgrade? Remove it with:\n\n" \ - "\trm -rf {venv_dir}\n\n") - else: - if self.need_sphinx and ver >= self.min_version: - return (f, ver) - elif parse_version(ver) > self.cur_version: - return (f, ver) - - return ("", ver) - - def recommend_sphinx_upgrade(self): - """ - Check if Sphinx needs to be upgraded. - - Returns a tuple with the higest available Sphinx version if found. - Otherwise, returns None to indicate either that no upgrade is needed - or no venv was found. - """ - - # Avoid running sphinx-builds from venv if cur_version is good - if self.cur_version and self.cur_version >= RECOMMENDED_VERSION: - self.latest_avail_ver = self.cur_version - return None - - # Get the highest version from sphinx_*/bin/sphinx-build and the - # corresponding command to activate the venv/virtenv - self.activate_cmd, self.venv_ver = self.get_virtenv() - - # Store the highest version from Sphinx existing virtualenvs - if self.activate_cmd and self.venv_ver > self.cur_version: - self.latest_avail_ver = self.venv_ver - else: - if self.cur_version: - self.latest_avail_ver = self.cur_version - else: - self.latest_avail_ver = (0, 0, 0) - - # As we don't know package version of Sphinx, and there's no - # virtual environments, don't check if upgrades are needed - if not self.virtualenv: - if not self.latest_avail_ver: - return None - - return self.latest_avail_ver - - # Either there are already a virtual env or a new one should be created - self.need_pip = 1 - - if not self.latest_avail_ver: - return None - - # Return if the reason is due to an upgrade or not - if self.latest_avail_ver != (0, 0, 0): - if self.latest_avail_ver < RECOMMENDED_VERSION: - self.rec_sphinx_upgrade = 1 - - return self.latest_avail_ver - - def recommend_package(self): - """ - Recommend installing Sphinx as a distro-specific package. - """ - - print("\n2) As a package with:") - - old_need = self.deps.need - old_optional = self.deps.optional - - self.pdf = False - self.deps.optional = 0 - old_verbose = self.verbose_warn_install - self.verbose_warn_install = 0 - - self.deps.clear_deps() - - self.deps.add_package("python-sphinx", DepManager.PYTHON_MANDATORY) - - cmd = self.get_install() - if cmd: - print(cmd) - - self.deps.need = old_need - self.deps.optional = old_optional - self.verbose_warn_install = old_verbose - - def recommend_sphinx_version(self, virtualenv_cmd): - """ - Provide recommendations for installing or upgrading Sphinx based - on current version. - - The logic here is complex, as it have to deal with different versions: - - - minimal supported version; - - minimal PDF version; - - recommended version. - - It also needs to work fine with both distro's package and - venv/virtualenv - """ - - if self.recommend_python: - print("\nPython version is incompatible with doc build.\n" \ - "Please upgrade it and re-run.\n") - return - - - # Version is OK. Nothing to do. - if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: - return - - if self.latest_avail_ver: - latest_avail_ver = ver_str(self.latest_avail_ver) - - if not self.need_sphinx: - # sphinx-build is present and its version is >= $min_version - - # only recommend enabling a newer virtenv version if makes sense. - if self.latest_avail_ver and self.latest_avail_ver > self.cur_version: - print(f"\nYou may also use the newer Sphinx version {latest_avail_ver} with:") - if f"{self.virtenv_prefix}" in os.getcwd(): - print("\tdeactivate") - print(f"\t. {self.activate_cmd}") - self.deactivate_help() - return - - if self.latest_avail_ver and self.latest_avail_ver >= RECOMMENDED_VERSION: - return - - if not self.virtualenv: - # No sphinx either via package or via virtenv. As we can't - # Compare the versions here, just return, recommending the - # user to install it from the package distro. - if not self.latest_avail_ver or self.latest_avail_ver == (0, 0, 0): - return - - # User doesn't want a virtenv recommendation, but he already - # installed one via virtenv with a newer version. - # So, print commands to enable it - if self.latest_avail_ver > self.cur_version: - print(f"\nYou may also use the Sphinx virtualenv version {latest_avail_ver} with:") - if f"{self.virtenv_prefix}" in os.getcwd(): - print("\tdeactivate") - print(f"\t. {self.activate_cmd}") - self.deactivate_help() - return - print("\n") - else: - if self.need_sphinx: - self.deps.need += 1 - - # Suggest newer versions if current ones are too old - if self.latest_avail_ver and self.latest_avail_ver >= self.min_version: - if self.latest_avail_ver >= RECOMMENDED_VERSION: - print(f"\nNeed to activate Sphinx (version {latest_avail_ver}) on virtualenv with:") - print(f"\t. {self.activate_cmd}") - self.deactivate_help() - return - - # Version is above the minimal required one, but may be - # below the recommended one. So, print warnings/notes - if self.latest_avail_ver < RECOMMENDED_VERSION: - print(f"Warning: It is recommended at least Sphinx version {RECOMMENDED_VERSION}.") - - # At this point, either it needs Sphinx or upgrade is recommended, - # both via pip - - if self.rec_sphinx_upgrade: - if not self.virtualenv: - print("Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n") - else: - print("To upgrade Sphinx, use:\n\n") - else: - print("\nSphinx needs to be installed either:\n1) via pip/pypi with:\n") - - if not virtualenv_cmd: - print(" Currently not possible.\n") - print(" Please upgrade Python to a newer version and run this script again") - else: - print(f"\t{virtualenv_cmd} {self.virtenv_dir}") - print(f"\t. {self.virtenv_dir}/bin/activate") - print(f"\tpip install -r {self.requirement_file}") - self.deactivate_help() - - if self.package_supported: - self.recommend_package() - - print("\n" \ - " Please note that Sphinx currentlys produce false-positive\n" \ - " warnings when the same name is used for more than one type (functions,\n" \ - " structs, enums,...). This is known Sphinx bug. For more details, see:\n" \ - "\thttps://github.com/sphinx-doc/sphinx/pull/8313") - - def check_needs(self): - """ - Main method that checks needed dependencies and provides - recommendations. - """ - self.python_cmd = sys.executable - - # Check if Sphinx is already accessible from current environment - self.check_sphinx(self.conf) - - if self.system_release: - print(f"Detected OS: {self.system_release}.") - else: - print("Unknown OS") - if self.cur_version != (0, 0, 0): - ver = ver_str(self.cur_version) - print(f"Sphinx version: {ver}\n") - - # Check the type of virtual env, depending on Python version - virtualenv_cmd = None - - if sys.version_info < MIN_PYTHON_VERSION: - min_ver = ver_str(MIN_PYTHON_VERSION) - print(f"ERROR: at least python {min_ver} is required to build the kernel docs") - self.need_sphinx = 1 - - self.venv_ver = self.recommend_sphinx_upgrade() - - if self.need_pip: - if sys.version_info < MIN_PYTHON_VERSION: - self.need_pip = False - print("Warning: python version is not supported.") - - else: - virtualenv_cmd = f"{self.python_cmd} -m venv" - self.check_python_module("ensurepip") - - # Check for needed programs/tools - self.check_perl_module("Pod::Usage", DepManager.SYSTEM_MANDATORY) - - self.check_program("make", DepManager.SYSTEM_MANDATORY) - self.check_program("gcc", DepManager.SYSTEM_MANDATORY) - - self.check_program("dot", DepManager.SYSTEM_OPTIONAL) - self.check_program("convert", DepManager.SYSTEM_OPTIONAL) - - self.check_python_module("yaml") - - if self.pdf: - self.check_program("xelatex", DepManager.PDF_MANDATORY) - self.check_program("rsvg-convert", DepManager.PDF_MANDATORY) - self.check_program("latexmk", DepManager.PDF_MANDATORY) - - # Do distro-specific checks and output distro-install commands - cmd = self.get_install() - if cmd: - print(cmd) - - # If distro requires some special instructions, print here. - # Please notice that get_install() needs to be called first. - if self.distro_msg: - print("\n" + self.distro_msg) - - if not self.python_cmd: - if self.need == 1: - sys.exit("Can't build as 1 mandatory dependency is missing") - elif self.need: - sys.exit(f"Can't build as {self.need} mandatory dependencies are missing") - - # Check if sphinx-build is called sphinx-build-3 - if self.need_symlink: - sphinx_path = self.which("sphinx-build-3") - if sphinx_path: - print(f"\tsudo ln -sf {sphinx_path} /usr/bin/sphinx-build\n") - - self.recommend_sphinx_version(virtualenv_cmd) - print("") - - if not self.deps.optional: - print("All optional dependencies are met.") - - if self.deps.need == 1: - sys.exit("Can't build as 1 mandatory dependency is missing") - elif self.deps.need: - sys.exit(f"Can't build as {self.deps.need} mandatory dependencies are missing") - - print("Needed package dependencies are met.") - -DESCRIPTION = """ -Process some flags related to Sphinx installation and documentation build. -""" - - -def main(): - """Main function""" - parser = argparse.ArgumentParser(description=DESCRIPTION) - - parser.add_argument( - "--no-virtualenv", - action="store_false", - dest="virtualenv", - help="Recommend installing Sphinx instead of using a virtualenv", - ) - - parser.add_argument( - "--no-pdf", - action="store_false", - dest="pdf", - help="Don't check for dependencies required to build PDF docs", - ) - - parser.add_argument( - "--version-check", - action="store_true", - dest="version_check", - help="If version is compatible, don't check for missing dependencies", - ) - - args = parser.parse_args() - - checker = SphinxDependencyChecker(args) - - checker.check_python() - checker.check_needs() - -# Call main if not used as module -if __name__ == "__main__": - main() -- cgit From 29e71d96837dd49ab2f35f3524a1ceda0ff21ea7 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:52 +0200 Subject: scripts: sphinx-pre-install.pl: get rid of the old script All features were ported to the Python version. Plus, it supports more variants and contain fixes. So, drop the old version. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/6900872e6b89b7ff304e70f5d1c23cbb3c757d28.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install.pl | 1056 ----------------------------------------- 1 file changed, 1056 deletions(-) delete mode 100755 scripts/sphinx-pre-install.pl (limited to 'scripts') diff --git a/scripts/sphinx-pre-install.pl b/scripts/sphinx-pre-install.pl deleted file mode 100755 index 07234d482fa8..000000000000 --- a/scripts/sphinx-pre-install.pl +++ /dev/null @@ -1,1056 +0,0 @@ -#!/usr/bin/env perl -# SPDX-License-Identifier: GPL-2.0-or-later -use strict; - -# Copyright (c) 2017-2020 Mauro Carvalho Chehab -# - -my $prefix = "./"; -$prefix = "$ENV{'srctree'}/" if ($ENV{'srctree'}); - -my $conf = $prefix . "Documentation/conf.py"; -my $requirement_file = $prefix . "Documentation/sphinx/requirements.txt"; -my $virtenv_prefix = "sphinx_"; - -# -# Static vars -# - -my %missing; -my $system_release; -my $need = 0; -my $optional = 0; -my $need_symlink = 0; -my $need_sphinx = 0; -my $need_pip = 0; -my $need_virtualenv = 0; -my $rec_sphinx_upgrade = 0; -my $verbose_warn_install = 1; -my $install = ""; -my $virtenv_dir = ""; -my $python_cmd = ""; -my $activate_cmd; -my $min_version; -my $cur_version; -my $rec_version = "3.4.3"; -my $latest_avail_ver; - -# -# Command line arguments -# - -my $pdf = 1; -my $virtualenv = 1; -my $version_check = 0; - -# -# List of required texlive packages on Fedora and OpenSuse -# - -my %texlive = ( - 'amsfonts.sty' => 'texlive-amsfonts', - 'amsmath.sty' => 'texlive-amsmath', - 'amssymb.sty' => 'texlive-amsfonts', - 'amsthm.sty' => 'texlive-amscls', - 'anyfontsize.sty' => 'texlive-anyfontsize', - 'atbegshi.sty' => 'texlive-oberdiek', - 'bm.sty' => 'texlive-tools', - 'capt-of.sty' => 'texlive-capt-of', - 'cmap.sty' => 'texlive-cmap', - 'ecrm1000.tfm' => 'texlive-ec', - 'eqparbox.sty' => 'texlive-eqparbox', - 'eu1enc.def' => 'texlive-euenc', - 'fancybox.sty' => 'texlive-fancybox', - 'fancyvrb.sty' => 'texlive-fancyvrb', - 'float.sty' => 'texlive-float', - 'fncychap.sty' => 'texlive-fncychap', - 'footnote.sty' => 'texlive-mdwtools', - 'framed.sty' => 'texlive-framed', - 'luatex85.sty' => 'texlive-luatex85', - 'multirow.sty' => 'texlive-multirow', - 'needspace.sty' => 'texlive-needspace', - 'palatino.sty' => 'texlive-psnfss', - 'parskip.sty' => 'texlive-parskip', - 'polyglossia.sty' => 'texlive-polyglossia', - 'tabulary.sty' => 'texlive-tabulary', - 'threeparttable.sty' => 'texlive-threeparttable', - 'titlesec.sty' => 'texlive-titlesec', - 'ucs.sty' => 'texlive-ucs', - 'upquote.sty' => 'texlive-upquote', - 'wrapfig.sty' => 'texlive-wrapfig', - 'ctexhook.sty' => 'texlive-ctex', -); - -# -# Subroutines that checks if a feature exists -# - -sub check_missing(%) -{ - my %map = %{$_[0]}; - - foreach my $prog (sort keys %missing) { - my $is_optional = $missing{$prog}; - - # At least on some LTS distros like CentOS 7, texlive doesn't - # provide all packages we need. When such distros are - # detected, we have to disable PDF output. - # - # So, we need to ignore the packages that distros would - # need for LaTeX to work - if ($is_optional == 2 && !$pdf) { - $optional--; - next; - } - - if ($verbose_warn_install) { - if ($is_optional) { - print "Warning: better to also install \"$prog\".\n"; - } else { - print "ERROR: please install \"$prog\", otherwise, build won't work.\n"; - } - } - if (defined($map{$prog})) { - $install .= " " . $map{$prog}; - } else { - $install .= " " . $prog; - } - } - - $install =~ s/^\s//; -} - -sub add_package($$) -{ - my $package = shift; - my $is_optional = shift; - - $missing{$package} = $is_optional; - if ($is_optional) { - $optional++; - } else { - $need++; - } -} - -sub check_missing_file($$$) -{ - my $files = shift; - my $package = shift; - my $is_optional = shift; - - for (@$files) { - return if(-e $_); - } - - add_package($package, $is_optional); -} - -sub findprog($) -{ - foreach(split(/:/, $ENV{PATH})) { - return "$_/$_[0]" if(-x "$_/$_[0]"); - } -} - -sub find_python_no_venv() -{ - my $prog = shift; - - my $cur_dir = qx(pwd); - $cur_dir =~ s/\s+$//; - - foreach my $dir (split(/:/, $ENV{PATH})) { - next if ($dir =~ m,($cur_dir)/sphinx,); - return "$dir/python3" if(-x "$dir/python3"); - } - foreach my $dir (split(/:/, $ENV{PATH})) { - next if ($dir =~ m,($cur_dir)/sphinx,); - return "$dir/python" if(-x "$dir/python"); - } - return "python"; -} - -sub check_program($$) -{ - my $prog = shift; - my $is_optional = shift; - - return $prog if findprog($prog); - - add_package($prog, $is_optional); -} - -sub check_perl_module($$) -{ - my $prog = shift; - my $is_optional = shift; - - my $err = system("perl -M$prog -e 1 2>/dev/null /dev/null"); - return if ($err == 0); - - add_package($prog, $is_optional); -} - -sub check_python_module($$) -{ - my $prog = shift; - my $is_optional = shift; - - return if (!$python_cmd); - - my $err = system("$python_cmd -c 'import $prog' 2>/dev/null /dev/null"); - return if ($err == 0); - - add_package($prog, $is_optional); -} - -sub check_rpm_missing($$) -{ - my @pkgs = @{$_[0]}; - my $is_optional = $_[1]; - - foreach my $prog(@pkgs) { - my $err = system("rpm -q '$prog' 2>/dev/null >/dev/null"); - add_package($prog, $is_optional) if ($err); - } -} - -sub check_pacman_missing($$) -{ - my @pkgs = @{$_[0]}; - my $is_optional = $_[1]; - - foreach my $prog(@pkgs) { - my $err = system("pacman -Q '$prog' 2>/dev/null >/dev/null"); - add_package($prog, $is_optional) if ($err); - } -} - -sub check_missing_tex($) -{ - my $is_optional = shift; - my $kpsewhich = findprog("kpsewhich"); - - foreach my $prog(keys %texlive) { - my $package = $texlive{$prog}; - if (!$kpsewhich) { - add_package($package, $is_optional); - next; - } - my $file = qx($kpsewhich $prog); - add_package($package, $is_optional) if ($file =~ /^\s*$/); - } -} - -sub get_sphinx_fname() -{ - if ($ENV{'SPHINXBUILD'}) { - return $ENV{'SPHINXBUILD'}; - } - - my $fname = "sphinx-build"; - return $fname if findprog($fname); - - $fname = "sphinx-build-3"; - if (findprog($fname)) { - $need_symlink = 1; - return $fname; - } - - return ""; -} - -sub get_sphinx_version($) -{ - my $cmd = shift; - my $ver; - - open IN, "$cmd --version 2>&1 |"; - while () { - if (m/^\s*sphinx-build\s+([\d\.]+)((\+\/[\da-f]+)|(b\d+))?$/) { - $ver=$1; - last; - } - # Sphinx 1.2.x uses a different format - if (m/^\s*Sphinx.*\s+([\d\.]+)$/) { - $ver=$1; - last; - } - } - close IN; - return $ver; -} - -sub check_sphinx() -{ - open IN, $conf or die "Can't open $conf"; - while () { - if (m/^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]/) { - $min_version=$1; - last; - } - } - close IN; - - die "Can't get needs_sphinx version from $conf" if (!$min_version); - - $virtenv_dir = $virtenv_prefix . "latest"; - - my $sphinx = get_sphinx_fname(); - if ($sphinx eq "") { - $need_sphinx = 1; - return; - } - - $cur_version = get_sphinx_version($sphinx); - die "$sphinx didn't return its version" if (!$cur_version); - - if ($cur_version lt $min_version) { - printf "ERROR: Sphinx version is %s. It should be >= %s\n", - $cur_version, $min_version; - $need_sphinx = 1; - return; - } - - return if ($cur_version lt $rec_version); - - # On version check mode, just assume Sphinx has all mandatory deps - exit (0) if ($version_check); -} - -# -# Ancillary subroutines -# - -sub catcheck($) -{ - my $res = ""; - $res = qx(cat $_[0]) if (-r $_[0]); - return $res; -} - -sub which($) -{ - my $file = shift; - my @path = split ":", $ENV{PATH}; - - foreach my $dir(@path) { - my $name = $dir.'/'.$file; - return $name if (-x $name ); - } - return undef; -} - -# -# Subroutines that check distro-specific hints -# - -sub give_debian_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-yaml", - "ensurepip" => "python3-venv", - "virtualenv" => "virtualenv", - "dot" => "graphviz", - "convert" => "imagemagick", - "Pod::Usage" => "perl-modules", - "xelatex" => "texlive-xetex", - "rsvg-convert" => "librsvg2-bin", - ); - - if ($pdf) { - check_missing_file(["/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty"], - "texlive-lang-chinese", 2); - - check_missing_file(["/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"], - "fonts-dejavu", 2); - - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc"], - "fonts-noto-cjk", 2); - } - - check_program("dvipng", 2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo apt-get install $install\n"); -} - -sub give_redhat_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-pyyaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive-xetex-bin", - "rsvg-convert" => "librsvg2-tools", - ); - - my @fedora26_opt_pkgs = ( - "graphviz-gd", # Fedora 26: needed for PDF support - ); - - my @fedora_tex_pkgs = ( - "texlive-collection-fontsrecommended", - "texlive-collection-latex", - "texlive-xecjk", - "dejavu-sans-fonts", - "dejavu-serif-fonts", - "dejavu-sans-mono-fonts", - ); - - # - # Checks valid for RHEL/CentOS version 7.x. - # - my $old = 0; - my $rel; - my $noto_sans_redhat = "google-noto-sans-cjk-ttc-fonts"; - $rel = $2 if ($system_release =~ /(release|Linux)\s+(\d+)/); - - if (!($system_release =~ /Fedora/)) { - $map{"virtualenv"} = "python-virtualenv"; - - if ($rel && $rel < 8) { - $old = 1; - $pdf = 0; - - printf("Note: texlive packages on RHEL/CENTOS <= 7 are incomplete. Can't support PDF output\n"); - printf("If you want to build PDF, please read:\n"); - printf("\thttps://www.systutorials.com/241660/how-to-install-tex-live-on-centos-7-linux/\n"); - } - } else { - if ($rel && $rel < 26) { - $old = 1; - } - if ($rel && $rel >= 38) { - $noto_sans_redhat = "google-noto-sans-cjk-fonts"; - } - } - if (!$rel) { - printf("Couldn't identify release number\n"); - $old = 1; - $pdf = 0; - } - - if ($pdf) { - check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/google-noto-sans-cjk-fonts/NotoSansCJK-Regular.ttc"], - $noto_sans_redhat, 2); - } - - check_rpm_missing(\@fedora26_opt_pkgs, 2) if ($pdf && !$old); - check_rpm_missing(\@fedora_tex_pkgs, 2) if ($pdf); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - - if (!$old) { - # dnf, for Fedora 18+ - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo dnf install -y $install\n"); - } else { - # yum, for RHEL (and clones) or Fedora version < 18 - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo yum install -y $install\n"); - } -} - -sub give_opensuse_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-pyyaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive-xetex-bin", - ); - - # On Tumbleweed, this package is also named rsvg-convert - $map{"rsvg-convert"} = "rsvg-view" if (!($system_release =~ /Tumbleweed/)); - - my @suse_tex_pkgs = ( - "texlive-babel-english", - "texlive-caption", - "texlive-colortbl", - "texlive-courier", - "texlive-dvips", - "texlive-helvetic", - "texlive-makeindex", - "texlive-metafont", - "texlive-metapost", - "texlive-palatino", - "texlive-preview", - "texlive-times", - "texlive-zapfchan", - "texlive-zapfding", - ); - - $map{"latexmk"} = "texlive-latexmk-bin"; - - # FIXME: add support for installing CJK fonts - # - # I tried hard, but was unable to find a way to install - # "Noto Sans CJK SC" on openSUSE - - check_rpm_missing(\@suse_tex_pkgs, 2) if ($pdf); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo zypper install --no-recommends $install\n"); -} - -sub give_mageia_hints() -{ - my %map = ( - "python-sphinx" => "python3-sphinx", - "yaml" => "python3-yaml", - "virtualenv" => "python3-virtualenv", - "dot" => "graphviz", - "convert" => "ImageMagick", - "Pod::Usage" => "perl-Pod-Usage", - "xelatex" => "texlive", - "rsvg-convert" => "librsvg2", - ); - - my @tex_pkgs = ( - "texlive-fontsextra", - ); - - $map{"latexmk"} = "texlive-collection-basic"; - - my $packager_cmd; - my $noto_sans; - if ($system_release =~ /OpenMandriva/) { - $packager_cmd = "dnf install"; - $noto_sans = "noto-sans-cjk-fonts"; - @tex_pkgs = ( "texlive-collection-fontsextra" ); - } else { - $packager_cmd = "urpmi"; - $noto_sans = "google-noto-sans-cjk-ttc-fonts"; - } - - - if ($pdf) { - check_missing_file(["/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", - "/usr/share/fonts/TTF/NotoSans-Regular.ttf"], - $noto_sans, 2); - } - - check_rpm_missing(\@tex_pkgs, 2) if ($pdf); - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo $packager_cmd $install\n"); -} - -sub give_arch_linux_hints() -{ - my %map = ( - "yaml" => "python-yaml", - "virtualenv" => "python-virtualenv", - "dot" => "graphviz", - "convert" => "imagemagick", - "xelatex" => "texlive-xetex", - "latexmk" => "texlive-core", - "rsvg-convert" => "extra/librsvg", - ); - - my @archlinux_tex_pkgs = ( - "texlive-core", - "texlive-latexextra", - "ttf-dejavu", - ); - check_pacman_missing(\@archlinux_tex_pkgs, 2) if ($pdf); - - if ($pdf) { - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc"], - "noto-fonts-cjk", 2); - } - - check_missing(\%map); - - return if (!$need && !$optional); - printf("You should run:\n") if ($verbose_warn_install); - printf("\n\tsudo pacman -S $install\n"); -} - -sub give_gentoo_hints() -{ - my %map = ( - "yaml" => "dev-python/pyyaml", - "virtualenv" => "dev-python/virtualenv", - "dot" => "media-gfx/graphviz", - "convert" => "media-gfx/imagemagick", - "xelatex" => "dev-texlive/texlive-xetex media-fonts/dejavu", - "rsvg-convert" => "gnome-base/librsvg", - ); - - check_missing_file(["/usr/share/fonts/dejavu/DejaVuSans.ttf"], - "media-fonts/dejavu", 2) if ($pdf); - - if ($pdf) { - check_missing_file(["/usr/share/fonts/noto-cjk/NotoSansCJKsc-Regular.otf", - "/usr/share/fonts/noto-cjk/NotoSerifCJK-Regular.ttc"], - "media-fonts/noto-cjk", 2); - } - - check_missing(\%map); - - return if (!$need && !$optional); - - printf("You should run:\n") if ($verbose_warn_install); - printf("\n"); - - my $imagemagick = "media-gfx/imagemagick svg png"; - my $cairo = "media-gfx/graphviz cairo pdf"; - my $portage_imagemagick = "/etc/portage/package.use/imagemagick"; - my $portage_cairo = "/etc/portage/package.use/graphviz"; - - if (qx(grep imagemagick $portage_imagemagick 2>/dev/null) eq "") { - printf("\tsudo su -c 'echo \"$imagemagick\" > $portage_imagemagick'\n") - } - if (qx(grep graphviz $portage_cairo 2>/dev/null) eq "") { - printf("\tsudo su -c 'echo \"$cairo\" > $portage_cairo'\n"); - } - - printf("\tsudo emerge --ask $install\n"); - -} - -sub check_distros() -{ - # Distro-specific hints - if ($system_release =~ /Red Hat Enterprise Linux/) { - give_redhat_hints; - return; - } - if ($system_release =~ /CentOS/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Scientific Linux/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Oracle Linux Server/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Fedora/) { - give_redhat_hints; - return; - } - if ($system_release =~ /Ubuntu/) { - give_debian_hints; - return; - } - if ($system_release =~ /Debian/) { - give_debian_hints; - return; - } - if ($system_release =~ /openSUSE/) { - give_opensuse_hints; - return; - } - if ($system_release =~ /Mageia/) { - give_mageia_hints; - return; - } - if ($system_release =~ /OpenMandriva/) { - give_mageia_hints; - return; - } - if ($system_release =~ /Arch Linux/) { - give_arch_linux_hints; - return; - } - if ($system_release =~ /Gentoo/) { - give_gentoo_hints; - return; - } - - # - # Fall-back to generic hint code for other distros - # That's far from ideal, specially for LaTeX dependencies. - # - my %map = ( - "sphinx-build" => "sphinx" - ); - check_missing_tex(2) if ($pdf); - check_missing(\%map); - print "I don't know distro $system_release.\n"; - print "So, I can't provide you a hint with the install procedure.\n"; - print "There are likely missing dependencies.\n"; -} - -# -# Common dependencies -# - -sub deactivate_help() -{ - printf "\n If you want to exit the virtualenv, you can use:\n"; - printf "\tdeactivate\n"; -} - -sub get_virtenv() -{ - my $ver; - my $min_activate = "$ENV{'PWD'}/${virtenv_prefix}${min_version}/bin/activate"; - my @activates = glob "$ENV{'PWD'}/${virtenv_prefix}*/bin/activate"; - - @activates = sort {$b cmp $a} @activates; - - foreach my $f (@activates) { - next if ($f lt $min_activate); - - my $sphinx_cmd = $f; - $sphinx_cmd =~ s/activate/sphinx-build/; - next if (! -f $sphinx_cmd); - - my $ver = get_sphinx_version($sphinx_cmd); - - if (!$ver) { - $f =~ s#/bin/activate##; - print("Warning: virtual environment $f is not working.\nPython version upgrade? Remove it with:\n\n\trm -rf $f\n\n"); - } - - if ($need_sphinx && ($ver ge $min_version)) { - return ($f, $ver); - } elsif ($ver gt $cur_version) { - return ($f, $ver); - } - } - return ("", ""); -} - -sub recommend_sphinx_upgrade() -{ - my $venv_ver; - - # Avoid running sphinx-builds from venv if $cur_version is good - if ($cur_version && ($cur_version ge $rec_version)) { - $latest_avail_ver = $cur_version; - return; - } - - # Get the highest version from sphinx_*/bin/sphinx-build and the - # corresponding command to activate the venv/virtenv - ($activate_cmd, $venv_ver) = get_virtenv(); - - # Store the highest version from Sphinx existing virtualenvs - if (($activate_cmd ne "") && ($venv_ver gt $cur_version)) { - $latest_avail_ver = $venv_ver; - } else { - $latest_avail_ver = $cur_version if ($cur_version); - } - - # As we don't know package version of Sphinx, and there's no - # virtual environments, don't check if upgrades are needed - if (!$virtualenv) { - return if (!$latest_avail_ver); - } - - # Either there are already a virtual env or a new one should be created - $need_pip = 1; - - return if (!$latest_avail_ver); - - # Return if the reason is due to an upgrade or not - if ($latest_avail_ver lt $rec_version) { - $rec_sphinx_upgrade = 1; - } - - return $latest_avail_ver; -} - -# -# The logic here is complex, as it have to deal with different versions: -# - minimal supported version; -# - minimal PDF version; -# - recommended version. -# It also needs to work fine with both distro's package and venv/virtualenv -sub recommend_sphinx_version($) -{ - my $virtualenv_cmd = shift; - - # Version is OK. Nothing to do. - if ($cur_version && ($cur_version ge $rec_version)) { - return; - }; - - if (!$need_sphinx) { - # sphinx-build is present and its version is >= $min_version - - #only recommend enabling a newer virtenv version if makes sense. - if ($latest_avail_ver gt $cur_version) { - printf "\nYou may also use the newer Sphinx version $latest_avail_ver with:\n"; - printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - return if ($latest_avail_ver ge $rec_version); - } - - if (!$virtualenv) { - # No sphinx either via package or via virtenv. As we can't - # Compare the versions here, just return, recommending the - # user to install it from the package distro. - return if (!$latest_avail_ver); - - # User doesn't want a virtenv recommendation, but he already - # installed one via virtenv with a newer version. - # So, print commands to enable it - if ($latest_avail_ver gt $cur_version) { - printf "\nYou may also use the Sphinx virtualenv version $latest_avail_ver with:\n"; - printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - print "\n"; - } else { - $need++ if ($need_sphinx); - } - - # Suggest newer versions if current ones are too old - if ($latest_avail_ver && $latest_avail_ver ge $min_version) { - # If there's a good enough version, ask the user to enable it - if ($latest_avail_ver ge $rec_version) { - printf "\nNeed to activate Sphinx (version $latest_avail_ver) on virtualenv with:\n"; - printf "\t. $activate_cmd\n"; - deactivate_help(); - - return; - } - - # Version is above the minimal required one, but may be - # below the recommended one. So, print warnings/notes - - if ($latest_avail_ver lt $rec_version) { - print "Warning: It is recommended at least Sphinx version $rec_version.\n"; - } - } - - # At this point, either it needs Sphinx or upgrade is recommended, - # both via pip - - if ($rec_sphinx_upgrade) { - if (!$virtualenv) { - print "Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n"; - } else { - print "To upgrade Sphinx, use:\n\n"; - } - } else { - print "\nSphinx needs to be installed either:\n1) via pip/pypi with:\n\n"; - } - - $python_cmd = find_python_no_venv(); - - printf "\t$virtualenv_cmd $virtenv_dir\n"; - - printf "\t. $virtenv_dir/bin/activate\n"; - printf "\tpip install -r $requirement_file\n"; - deactivate_help(); - - printf "\n2) As a package with:\n"; - - my $old_need = $need; - my $old_optional = $optional; - %missing = (); - $pdf = 0; - $optional = 0; - $install = ""; - $verbose_warn_install = 0; - - add_package("python-sphinx", 0); - - check_distros(); - - $need = $old_need; - $optional = $old_optional; - - printf "\n Please note that Sphinx >= 3.0 will currently produce false-positive\n"; - printf " warning when the same name is used for more than one type (functions,\n"; - printf " structs, enums,...). This is known Sphinx bug. For more details, see:\n"; - printf "\thttps://github.com/sphinx-doc/sphinx/pull/8313\n"; -} - -sub check_needs() -{ - # Check if Sphinx is already accessible from current environment - check_sphinx(); - - if ($system_release) { - print "Detected OS: $system_release.\n"; - } else { - print "Unknown OS\n"; - } - printf "Sphinx version: %s\n\n", $cur_version if ($cur_version); - - # Check python command line, trying first python3 - $python_cmd = findprog("python3"); - $python_cmd = check_program("python", 0) if (!$python_cmd); - - # Check the type of virtual env, depending on Python version - if ($python_cmd) { - if ($virtualenv) { - my $tmp = qx($python_cmd --version 2>&1); - if ($tmp =~ m/(\d+\.)(\d+\.)/) { - if ($1 < 3) { - # Fail if it finds python2 (or worse) - die "Python 3 is required to build the kernel docs\n"; - } - if ($1 == 3 && $2 < 3) { - # Need Python 3.3 or upper for venv - $need_virtualenv = 1; - } - } else { - die "Warning: couldn't identify $python_cmd version!"; - } - } else { - add_package("python-sphinx", 0); - } - } - - my $venv_ver = recommend_sphinx_upgrade(); - - my $virtualenv_cmd; - - if ($need_pip) { - # Set virtualenv command line, if python < 3.3 - if ($need_virtualenv) { - $virtualenv_cmd = findprog("virtualenv-3"); - $virtualenv_cmd = findprog("virtualenv-3.5") if (!$virtualenv_cmd); - if (!$virtualenv_cmd) { - check_program("virtualenv", 0); - $virtualenv_cmd = "virtualenv"; - } - } else { - $virtualenv_cmd = "$python_cmd -m venv"; - check_python_module("ensurepip", 0); - } - } - - # Check for needed programs/tools - check_perl_module("Pod::Usage", 0); - check_python_module("yaml", 0); - check_program("make", 0); - check_program("gcc", 0); - check_program("dot", 1); - check_program("convert", 1); - - # Extra PDF files - should use 2 for is_optional - check_program("xelatex", 2) if ($pdf); - check_program("rsvg-convert", 2) if ($pdf); - check_program("latexmk", 2) if ($pdf); - - # Do distro-specific checks and output distro-install commands - check_distros(); - - if (!$python_cmd) { - if ($need == 1) { - die "Can't build as $need mandatory dependency is missing"; - } elsif ($need) { - die "Can't build as $need mandatory dependencies are missing"; - } - } - - # Check if sphinx-build is called sphinx-build-3 - if ($need_symlink) { - printf "\tsudo ln -sf %s /usr/bin/sphinx-build\n\n", - which("sphinx-build-3"); - } - - recommend_sphinx_version($virtualenv_cmd); - printf "\n"; - - print "All optional dependencies are met.\n" if (!$optional); - - if ($need == 1) { - die "Can't build as $need mandatory dependency is missing"; - } elsif ($need) { - die "Can't build as $need mandatory dependencies are missing"; - } - - print "Needed package dependencies are met.\n"; -} - -# -# Main -# - -while (@ARGV) { - my $arg = shift(@ARGV); - - if ($arg eq "--no-virtualenv") { - $virtualenv = 0; - } elsif ($arg eq "--no-pdf"){ - $pdf = 0; - } elsif ($arg eq "--version-check"){ - $version_check = 1; - } else { - print "Usage:\n\t$0 <--no-virtualenv> <--no-pdf> <--version-check>\n\n"; - print "Where:\n"; - print "\t--no-virtualenv\t- Recommend installing Sphinx instead of using a virtualenv\n"; - print "\t--version-check\t- if version is compatible, don't check for missing dependencies\n"; - print "\t--no-pdf\t- don't check for dependencies required to build PDF docs\n\n"; - exit -1; - } -} - -# -# Determine the system type. There's no standard unique way that would -# work with all distros with a minimal package install. So, several -# methods are used here. -# -# By default, it will use lsb_release function. If not available, it will -# fail back to reading the known different places where the distro name -# is stored -# - -$system_release = qx(lsb_release -d) if which("lsb_release"); -$system_release =~ s/Description:\s*// if ($system_release); -$system_release = catcheck("/etc/system-release") if !$system_release; -$system_release = catcheck("/etc/redhat-release") if !$system_release; -$system_release = catcheck("/etc/lsb-release") if !$system_release; -$system_release = catcheck("/etc/gentoo-release") if !$system_release; - -# This seems more common than LSB these days -if (!$system_release) { - my %os_var; - if (open IN, "cat /etc/os-release|") { - while () { - if (m/^([\w\d\_]+)=\"?([^\"]*)\"?\n/) { - $os_var{$1}=$2; - } - } - $system_release = $os_var{"NAME"}; - if (defined($os_var{"VERSION_ID"})) { - $system_release .= " " . $os_var{"VERSION_ID"} if (defined($os_var{"VERSION_ID"})); - } else { - $system_release .= " " . $os_var{"VERSION"}; - } - } -} -$system_release = catcheck("/etc/issue") if !$system_release; -$system_release =~ s/\s+$//; - -check_needs; -- cgit From df4d2f966354297c94ba5e198533cc31548a12a8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:53 +0200 Subject: scripts: sphinx-pre-install: update mandatory system deps To build docs, gcc is not needed. Also, Kernel can be built nowadays with clang. So, drop it. On the other hand, which is needed. Add a system dependency for it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/4ec979e4692c9e4acd6c31424c0e2f4bf5b80e71.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 7dfe5c2a6cc2..fc9dc45054d7 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -1476,7 +1476,7 @@ class SphinxDependencyChecker(MissingCheckers): self.check_perl_module("Pod::Usage", DepManager.SYSTEM_MANDATORY) self.check_program("make", DepManager.SYSTEM_MANDATORY) - self.check_program("gcc", DepManager.SYSTEM_MANDATORY) + self.check_program("which", DepManager.SYSTEM_MANDATORY) self.check_program("dot", DepManager.SYSTEM_OPTIONAL) self.check_program("convert", DepManager.SYSTEM_OPTIONAL) -- cgit From 9f51a1d6966760dadf62ff1f62c614a342fb7e62 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:54 +0200 Subject: scripts: sphinx-pre-install: add support for RHEL8-based distros On RHEL8, only installing with a venv is supported, as there's no Sphinx package using Python 3.7 or upper. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/bcdde20edab07be6bf447eac18eecdd88c7f947c.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index fc9dc45054d7..324baa98a395 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -691,7 +691,7 @@ class SphinxDependencyChecker(MissingCheckers): super().__init__(args, texlive) - self.need_pip = 0 + self.need_pip = False self.rec_sphinx_upgrade = 0 self.system_release = self.get_system_release() @@ -840,9 +840,15 @@ class SphinxDependencyChecker(MissingCheckers): # RHEL 8 uses Python 3.6, which is not compatible with # the build system anymore. Suggest Python 3.11 if rel == 8: - self.deps.add_package("python39", DepManager.SYSTEM_MANDATORY) + self.check_program("python3.9", DepManager.SYSTEM_MANDATORY) + progs["python3.9"] = "python39" + progs["yaml"] = "python39-pyyaml" + self.recommend_python = True + # There's no python39-sphinx package. Only pip is supported + self.package_supported = False + if not self.distro_msg: self.distro_msg = \ "Note: RHEL-based distros typically require extra repositories.\n" \ @@ -915,8 +921,9 @@ class SphinxDependencyChecker(MissingCheckers): # the build system anymore. Suggest Python 3.11 if rel == 15: if not self.which(self.python_cmd): + self.check_program("python3.11", DepManager.SYSTEM_MANDATORY) + progs["python3.11"] = "python311" self.recommend_python = True - self.deps.add_package(self.python_cmd, DepManager.SYSTEM_MANDATORY) progs.update({ "python-sphinx": "python311-Sphinx", @@ -1289,7 +1296,7 @@ class SphinxDependencyChecker(MissingCheckers): return self.latest_avail_ver # Either there are already a virtual env or a new one should be created - self.need_pip = 1 + self.need_pip = True if not self.latest_avail_ver: return None @@ -1344,10 +1351,11 @@ class SphinxDependencyChecker(MissingCheckers): """ if self.recommend_python: - print("\nPython version is incompatible with doc build.\n" \ - "Please upgrade it and re-run.\n") - return - + cur_ver = sys.version_info[:3] + if cur_ver < MIN_PYTHON_VERSION: + print(f"\nPython version {cur_ver} is incompatible with doc build.\n" \ + "Please upgrade it and re-run.\n") + return # Version is OK. Nothing to do. if self.cur_version != (0, 0, 0) and self.cur_version >= RECOMMENDED_VERSION: @@ -1467,7 +1475,6 @@ class SphinxDependencyChecker(MissingCheckers): if sys.version_info < MIN_PYTHON_VERSION: self.need_pip = False print("Warning: python version is not supported.") - else: virtualenv_cmd = f"{self.python_cmd} -m venv" self.check_python_module("ensurepip") -- cgit From 491a99511eaf333855caf28170cfb3f6f4460b54 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:55 +0200 Subject: scripts: sphinx-pre-install: add a warning for Debian-based distros On Some Debian-based distros, ImageMagick package has a broken policy that causes LaTeX to fail while building docs. Add a note about that. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/628d01784e8c24e3d93c69c436f12398e00165b3.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 324baa98a395..09a337509b23 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -782,6 +782,11 @@ class SphinxDependencyChecker(MissingCheckers): self.check_program("dvipng", DepManager.PDF_MANDATORY) + if not self.distro_msg: + self.distro_msg = \ + "Note: ImageMagick is broken on some distros, affecting PDF output. For more details:\n" \ + "\thttps://askubuntu.com/questions/1158894/imagemagick-still-broken-using-with-usr-bin-convert" + return self.get_install_progs(progs, "apt-get install") def give_redhat_hints(self): @@ -1193,7 +1198,7 @@ class SphinxDependencyChecker(MissingCheckers): self.distro_msg = \ f"I don't know distro {self.system_release}.\n" \ "So, I can't provide you a hint with the install procedure.\n" \ - "There are likely missing dependencies.\n" + "There are likely missing dependencies." return self.get_install_progs(progs, None) -- cgit From 6170b1eacac881af5732af78f33b76bd560c441b Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 12 Aug 2025 17:52:56 +0200 Subject: scripts: sphinx-pre-install: some adjustments related to venv While nothing was really needed for virtualenv to work on most distros, we had an issue with OpenMandriva. While checking for it, it was noticed that there was no check if python-virtualenv was installed. This didn't solve the issues we faced there: at least with the half-broken OpenMandriva Lx 4.0 docker container we used, ensurepip was not available anywhere, causing venv to fail. Add a distro-specific note about that. Note: at least at the time we did our tests, OpenMandriva Lx 4.0 docker was shipped with wrong dnf repositories. Also, there was no repos available for it anymore. So, we had to do some hacks to upgrade to 4.3 before being able to run any tests. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/e3a0e5eccd50eb506846e3e8487a2d9124ef83e2.1754992972.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 09a337509b23..b8474848df4e 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -983,6 +983,22 @@ class SphinxDependencyChecker(MissingCheckers): # Tested on OpenMandriva Lx 4.3 progs["convert"] = "imagemagick" progs["yaml"] = "python-pyyaml" + progs["python-virtualenv"] = "python-virtualenv" + progs["python-sphinx"] = "python-sphinx" + + self.check_program("python-virtualenv", DepManager.PYTHON_MANDATORY) + + # On my tests with openMandriva LX 4.0 docker image, upgraded + # to 4.3, python-virtualenv package is broken: it is missing + # ensurepip. Without it, the alternative would be to run: + # python3 -m venv --without-pip ~/sphinx_latest, but running + # pip there won't install sphinx at venv. + # + # Add a note about that. + + if not self.distro_msg: + self.distro_msg = \ + "Note: for venv, ensurepip could be broken, preventing its install method." else: packager_cmd = "urpmi" -- cgit From f852ce052a8b78d6fade371bd1fad583541e78fa Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Thu, 14 Aug 2025 12:14:41 +0200 Subject: kbuild: align W=e with CONFIG_WERROR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CONFIG_WERROR sets KBUILD_CPPFLAGS while W=e would only set KBUILD_CFLAGS. As a preparation to unify the two mechanism, align their effects. While at it, add some alignment whitespace to prepare for later additions to the list of changed variables. Signed-off-by: Thomas Weißschuh Reviewed-by: Nicolas Schier Link: https://lore.kernel.org/r/20250814-kbuild-werror-v2-1-c01e596309d2@linutronix.de Signed-off-by: Nathan Chancellor --- scripts/Makefile.extrawarn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn index dca175fffcab..4b4e8e136ce6 100644 --- a/scripts/Makefile.extrawarn +++ b/scripts/Makefile.extrawarn @@ -218,6 +218,6 @@ endif # ifneq ($(findstring e, $(KBUILD_EXTRA_WARN)),) -KBUILD_CFLAGS += -Werror +KBUILD_CPPFLAGS += -Werror endif -- cgit From e7a10929c574cf30981a8e19ef39bc35e63e8b46 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Thu, 14 Aug 2025 12:14:42 +0200 Subject: kbuild: unify W=e and CONFIG_WERROR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two mechanisms have the same effect, unify their implementation. Also avoid spurious rebuilds when switching between the two. Signed-off-by: Thomas Weißschuh Reviewed-by: Nicolas Schier Link: https://lore.kernel.org/r/20250814-kbuild-werror-v2-2-c01e596309d2@linutronix.de Signed-off-by: Nathan Chancellor --- scripts/Makefile.extrawarn | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn index 4b4e8e136ce6..1ffc7beca43b 100644 --- a/scripts/Makefile.extrawarn +++ b/scripts/Makefile.extrawarn @@ -25,8 +25,6 @@ ifneq ($(CONFIG_FRAME_WARN),0) KBUILD_CFLAGS += -Wframe-larger-than=$(CONFIG_FRAME_WARN) endif -KBUILD_CPPFLAGS-$(CONFIG_WERROR) += -Werror -KBUILD_CPPFLAGS += $(KBUILD_CPPFLAGS-y) KBUILD_CFLAGS-$(CONFIG_CC_NO_ARRAY_BOUNDS) += -Wno-array-bounds ifdef CONFIG_CC_IS_CLANG @@ -214,9 +212,9 @@ KBUILD_CFLAGS += -Wno-unused-parameter endif # -# W=e - error out on warnings +# W=e and CONFIG_WERROR - error out on warnings # -ifneq ($(findstring e, $(KBUILD_EXTRA_WARN)),) +ifneq ($(findstring e, $(KBUILD_EXTRA_WARN))$(CONFIG_WERROR),) KBUILD_CPPFLAGS += -Werror -- cgit From 592b571f20c5b905c47a1370210456f9f90ce04f Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Thu, 14 Aug 2025 12:14:43 +0200 Subject: kbuild: rust: move `-Dwarnings` handling to `Makefile.extrawarn` Following commit e88ca24319e4 ("kbuild: consolidate warning flags in scripts/Makefile.extrawarn"), move `-Dwarnings` handling into `Makefile.extrawarn` like C's `-Werror`. No functional change intended. Signed-off-by: Miguel Ojeda Reviewed-by: Nicolas Schier Link: https://lore.kernel.org/r/20250814-kbuild-werror-v2-3-c01e596309d2@linutronix.de Signed-off-by: Nathan Chancellor --- scripts/Makefile.extrawarn | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn index 1ffc7beca43b..b04b3062e0e4 100644 --- a/scripts/Makefile.extrawarn +++ b/scripts/Makefile.extrawarn @@ -217,5 +217,6 @@ endif ifneq ($(findstring e, $(KBUILD_EXTRA_WARN))$(CONFIG_WERROR),) KBUILD_CPPFLAGS += -Werror +KBUILD_RUSTFLAGS += -Dwarnings endif -- cgit From ec4a3992bc0b5b659eceb44a9f8582b26f2c8489 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Thu, 14 Aug 2025 12:14:44 +0200 Subject: kbuild: respect CONFIG_WERROR for linker and assembler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The linker and assembler do not share the compiler flags. Make sure they also fail on warnings with CONFIG_WERROR and W=e. Signed-off-by: Thomas Weißschuh Reviewed-by: Nicolas Schier Link: https://lore.kernel.org/r/20250814-kbuild-werror-v2-4-c01e596309d2@linutronix.de Signed-off-by: Nathan Chancellor --- scripts/Makefile.extrawarn | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn index b04b3062e0e4..6efb9e5eeeed 100644 --- a/scripts/Makefile.extrawarn +++ b/scripts/Makefile.extrawarn @@ -217,6 +217,8 @@ endif ifneq ($(findstring e, $(KBUILD_EXTRA_WARN))$(CONFIG_WERROR),) KBUILD_CPPFLAGS += -Werror +KBUILD_AFLAGS += -Wa,--fatal-warnings +KBUILD_LDFLAGS += --fatal-warnings KBUILD_RUSTFLAGS += -Dwarnings endif -- cgit From 3f0ff4cc6ffb43ea250fdf69d9f7ae8e5a05f1f9 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Thu, 14 Aug 2025 12:14:45 +0200 Subject: kbuild: respect CONFIG_WERROR for userprogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The userprogs compiler and linker do not share the regular compiler flags. Make sure they also fail on warnings with CONFIG_WERROR and W=e. Signed-off-by: Thomas Weißschuh Reviewed-by: Nicolas Schier Link: https://lore.kernel.org/r/20250814-kbuild-werror-v2-5-c01e596309d2@linutronix.de Signed-off-by: Nathan Chancellor --- scripts/Makefile.extrawarn | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn index 6efb9e5eeeed..96ff3f5582d6 100644 --- a/scripts/Makefile.extrawarn +++ b/scripts/Makefile.extrawarn @@ -219,6 +219,8 @@ ifneq ($(findstring e, $(KBUILD_EXTRA_WARN))$(CONFIG_WERROR),) KBUILD_CPPFLAGS += -Werror KBUILD_AFLAGS += -Wa,--fatal-warnings KBUILD_LDFLAGS += --fatal-warnings +KBUILD_USERCFLAGS += -Werror +KBUILD_USERLDFLAGS += -Wl,--fatal-warnings KBUILD_RUSTFLAGS += -Dwarnings endif -- cgit From 670ec7333a2c4823ac777b70f045cf731525ae5e Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:29 -0600 Subject: docs: kdoc: remove dead code create_parameter_list() tests an argument against the same regex twice, in two different locations; remove the pointless extra tests and the never-executed error cases that go with them. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-2-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 9e65948f8254..96e3fe4ec431 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -564,28 +564,18 @@ class KernelDoc: args.insert(0, first_arg.pop()) dtype = ' '.join(first_arg) + bitfield_re = KernRe(r'(.*?):(\w+)') for param in args: - if KernRe(r'^(\*+)\s*(.*)').match(param): - r = KernRe(r'^(\*+)\s*(.*)') - if not r.match(param): - self.emit_msg(ln, f"Invalid param: {param}") - continue - - param = r.group(1) - + r = KernRe(r'^(\*+)\s*(.*)') + if r.match(param): self.push_parameter(ln, decl_type, r.group(2), f"{dtype} {r.group(1)}", arg, declaration_name) - elif KernRe(r'(.*?):(\w+)').search(param): - r = KernRe(r'(.*?):(\w+)') - if not r.match(param): - self.emit_msg(ln, f"Invalid param: {param}") - continue - + elif bitfield_re.search(param): if dtype != "": # Skip unnamed bit-fields - self.push_parameter(ln, decl_type, r.group(1), - f"{dtype}:{r.group(2)}", + self.push_parameter(ln, decl_type, bitfield_re.group(1), + f"{dtype}:{bitfield_re.group(2)}", arg, declaration_name) else: self.push_parameter(ln, decl_type, param, dtype, -- cgit From f51b42b99e1d35698e0277337fde2c15ccc29a2b Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:30 -0600 Subject: docs: kdoc: tidy up space removal in create_parameter_list() Remove a redundant test and add a comment describing what the space removal is doing. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-3-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 96e3fe4ec431..53051ce831ba 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -545,12 +545,14 @@ class KernelDoc: arg, declaration_name) elif arg: + # + # Clean up extraneous spaces and split the string at commas; the first + # element of the resulting list will also include the type information. + # arg = KernRe(r'\s*:\s*').sub(":", arg) arg = KernRe(r'\s*\[').sub('[', arg) - args = KernRe(r'\s*,\s*').split(arg) - if args[0] and '*' in args[0]: - args[0] = re.sub(r'(\*+)\s*', r' \1', args[0]) + args[0] = re.sub(r'(\*+)\s*', r' \1', args[0]) first_arg = [] r = KernRe(r'^(.*\s+)(.*?\[.*\].*)$') -- cgit From 05d72fe07242a8e4535aa52e0858f9198e668a41 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:31 -0600 Subject: docs: kdoc: clean up the create_parameter_list() "first arg" logic The logic for finding the name of the first in a series of variable names is somewhat convoluted and, in the use of .extend(), actively buggy. Document what is happening and simplify the logic. Acked-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-4-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 53051ce831ba..07234ce04409 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -553,18 +553,18 @@ class KernelDoc: arg = KernRe(r'\s*\[').sub('[', arg) args = KernRe(r'\s*,\s*').split(arg) args[0] = re.sub(r'(\*+)\s*', r' \1', args[0]) - - first_arg = [] - r = KernRe(r'^(.*\s+)(.*?\[.*\].*)$') - if args[0] and r.match(args[0]): - args.pop(0) - first_arg.extend(r.group(1)) - first_arg.append(r.group(2)) + # + # args[0] has a string of "type a". If "a" includes an [array] + # declaration, we want to not be fooled by any white space inside + # the brackets, so detect and handle that case specially. + # + r = KernRe(r'^([^[\]]*\s+)(.*)$') + if r.match(args[0]): + args[0] = r.group(2) + dtype = r.group(1) else: - first_arg = KernRe(r'\s+').split(args.pop(0)) - - args.insert(0, first_arg.pop()) - dtype = ' '.join(first_arg) + # No space in args[0]; this seems wrong but preserves previous behavior + dtype = '' bitfield_re = KernRe(r'(.*?):(\w+)') for param in args: -- cgit From 8f05fbc5afb86f0d4dcae33f3cb0cda561d4d93e Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:32 -0600 Subject: docs: kdoc: add a couple more comments in create_parameter_list() Make what the final code is doing a bit more clear to slow readers like me. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-5-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 07234ce04409..29881757bf1c 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -568,12 +568,18 @@ class KernelDoc: bitfield_re = KernRe(r'(.*?):(\w+)') for param in args: + # + # For pointers, shift the star(s) from the variable name to the + # type declaration. + # r = KernRe(r'^(\*+)\s*(.*)') if r.match(param): self.push_parameter(ln, decl_type, r.group(2), f"{dtype} {r.group(1)}", arg, declaration_name) - + # + # Perform a similar shift for bitfields. + # elif bitfield_re.search(param): if dtype != "": # Skip unnamed bit-fields self.push_parameter(ln, decl_type, bitfield_re.group(1), -- cgit From bf6b310d1b7e31a1cd6951eb75608a1d2876c04a Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:33 -0600 Subject: docs: kdoc: tighten up the array-of-pointers case Simplify one gnarly regex and remove another altogether; add a comment describing what is going on. There will be no #-substituted commas in this case, so don't bother trying to put them back. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-6-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 29881757bf1c..7f4d95dd47d4 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -527,23 +527,21 @@ class KernelDoc: dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg) self.push_parameter(ln, decl_type, param, dtype, arg, declaration_name) - + # + # The array-of-pointers case. Dig the parameter name out from the middle + # of the declaration. + # elif KernRe(r'\(.+\)\s*\[').search(arg): - # Array-of-pointers - - arg = arg.replace('#', ',') - r = KernRe(r'[^\(]+\(\s*\*\s*([\w\[\].]*?)\s*(\s*\[\s*[\w]+\s*\]\s*)*\)') + r = KernRe(r'[^\(]+\(\s*\*\s*' # Up to "(" and maybe "*" + r'([\w.]*?)' # The actual pointer name + r'\s*(\[\s*\w+\s*\]\s*)*\)') # The [array portion] if r.match(arg): param = r.group(1) else: self.emit_msg(ln, f"Invalid param: {arg}") param = arg - - dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg) - - self.push_parameter(ln, decl_type, param, dtype, - arg, declaration_name) - + dtype = arg.replace(param, '') + self.push_parameter(ln, decl_type, param, dtype, arg, declaration_name) elif arg: # # Clean up extraneous spaces and split the string at commas; the first -- cgit From e5d91662fcbac251dd17f04dbacf4d997939316e Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:34 -0600 Subject: docs: kdoc: tighten up the pointer-to-function case Tighten up the code and remove an unneeded regex operation. Reviewed-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-7-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 7f4d95dd47d4..998b1ece932a 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -511,22 +511,21 @@ class KernelDoc: # Treat preprocessor directive as a typeless variable self.push_parameter(ln, decl_type, arg, "", "", declaration_name) - + # + # The pointer-to-function case. + # elif KernRe(r'\(.+\)\s*\(').search(arg): - # Pointer-to-function - arg = arg.replace('#', ',') - - r = KernRe(r'[^\(]+\(\*?\s*([\w\[\].]*)\s*\)') + r = KernRe(r'[^\(]+\(\*?\s*' # Everything up to "(*" + r'([\w\[\].]*)' # Capture the name and possible [array] + r'\s*\)') # Make sure the trailing ")" is there if r.match(arg): param = r.group(1) else: self.emit_msg(ln, f"Invalid param: {arg}") param = arg - - dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg) - self.push_parameter(ln, decl_type, param, dtype, - arg, declaration_name) + dtype = arg.replace(param, '') + self.push_parameter(ln, decl_type, param, dtype, arg, declaration_name) # # The array-of-pointers case. Dig the parameter name out from the middle # of the declaration. -- cgit From 1d8125e27323d8a378cb38f88a6c5a0d7fdb2f6c Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 14 Aug 2025 09:40:35 -0600 Subject: docs: kdoc: remove redundant comment stripping By the time stuff gets to create_parameter_list(), comments have long since been stripped out, so we do not need to do it again here. Acked-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250814154035.328769-8-corbet@lwn.net --- scripts/lib/kdoc/kdoc_parser.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 998b1ece932a..a560546c1867 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -493,9 +493,6 @@ class KernelDoc: args = arg_expr.sub(r"\1#", args) for arg in args.split(splitter): - # Strip comments - arg = KernRe(r'/\*.*\*/').sub('', arg) - # Ignore argument attributes arg = KernRe(r'\sPOS0?\s').sub(' ', arg) -- cgit From 37c52167b007d9d0bb8c5ed53dd6efc4969a1356 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 13 Aug 2025 12:00:52 +0200 Subject: docs: Remove remainders of reiserfs Reiserfs has been removed in 6.13, there are still some mentions in the documentation about it and the tools. Remove those that don't seem relevant anymore but keep references to reiserfs' r5 hash used by some code. There's one change in a script scripts/selinux/install_policy.sh but it does not seem to be relevant either. Signed-off-by: David Sterba Acked-by: Paul Moore Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250813100053.1291961-1-dsterba@suse.com --- scripts/selinux/install_policy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/selinux/install_policy.sh b/scripts/selinux/install_policy.sh index db40237e60ce..77368a73f111 100755 --- a/scripts/selinux/install_policy.sh +++ b/scripts/selinux/install_policy.sh @@ -74,7 +74,7 @@ cd /etc/selinux/dummy/contexts/files $SF -F file_contexts / mounts=`cat /proc/$$/mounts | \ - grep -E "ext[234]|jfs|xfs|reiserfs|jffs2|gfs2|btrfs|f2fs|ocfs2" | \ + grep -E "ext[234]|jfs|xfs|jffs2|gfs2|btrfs|f2fs|ocfs2" | \ awk '{ print $2 '}` $SF -F file_contexts $mounts -- cgit From 27758d8c2583d10472b745a43ff86fef96c11ef7 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Thu, 14 Aug 2025 12:14:46 +0200 Subject: kbuild: enable -Werror for hostprogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hostprog compilers and linkers do not share the regular compiler flags, so they are not affected by CONFIG_WERROR or W=e. As hostprogs are used during the bootstrap of the build, they can't depend on kconfig options. Enable -Werror unconditionally. Signed-off-by: Thomas Weißschuh Link: https://lore.kernel.org/rust-for-linux/CANiq72k-PdSH2BNgbq=X+FhpyEErifSCKfO5ObXz6bu9_J8+fA@mail.gmail.com/ Link: https://lore.kernel.org/r/20250814-kbuild-werror-v2-6-c01e596309d2@linutronix.de Signed-off-by: Nathan Chancellor --- scripts/Makefile.extrawarn | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'scripts') diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn index 96ff3f5582d6..1434cb6208cb 100644 --- a/scripts/Makefile.extrawarn +++ b/scripts/Makefile.extrawarn @@ -224,3 +224,8 @@ KBUILD_USERLDFLAGS += -Wl,--fatal-warnings KBUILD_RUSTFLAGS += -Dwarnings endif + +# Hostprog flags are used during build bootstrapping and can not rely on CONFIG_ symbols. +KBUILD_HOSTCFLAGS += -Werror +KBUILD_HOSTLDFLAGS += -Wl,--fatal-warnings +KBUILD_HOSTRUSTFLAGS += -Dwarnings -- cgit From 35883b030c5f19f12c7fe53319c93cb740e476be Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Mon, 11 Aug 2025 18:16:48 +0200 Subject: kconfig: nconf: Format and print 'line' without a temporary copy Use "%.*s" as the format specifier and supply the 'line' length 'len' to mvwprintw() to format and print each line without making a temporary copy. Remove the temporary buffer. Signed-off-by: Thorsten Blum Reviewed-by: Nicolas Schier Link: https://lore.kernel.org/r/20250811161650.37428-2-thorsten.blum@linux.dev Signed-off-by: Nathan Chancellor --- scripts/kconfig/nconf.gui.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/kconfig/nconf.gui.c b/scripts/kconfig/nconf.gui.c index 7206437e784a..2d097bc7ef1a 100644 --- a/scripts/kconfig/nconf.gui.c +++ b/scripts/kconfig/nconf.gui.c @@ -173,12 +173,10 @@ void fill_window(WINDOW *win, const char *text) /* do not go over end of line */ total_lines = min(total_lines, y); for (i = 0; i < total_lines; i++) { - char tmp[x+10]; const char *line = get_line(text, i); - int len = get_line_length(line); - strncpy(tmp, line, min(len, x)); - tmp[len] = '\0'; - mvwprintw(win, i, 0, "%s", tmp); + int len = min(get_line_length(line), x); + + mvwprintw(win, i, 0, "%.*s", len, line); } } -- cgit From 75a6b4595daa569bbc2899eef40372fc013b2d73 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 12 Aug 2025 15:35:02 -0700 Subject: kconfig: qconf/xconfig: show the OptionsMode radio button setting at startup When qconf (xconfig) exits, it saves the current Option settings for Show Name, Show Debug Info, and Show {Normal|All|Prompt} Options. When it is next run, it loads these Option settings from its config file. It correctly shows the flag settings for Show Name and Show Debug Info, but it does not show which of the 3 Show...Options is set. This can lead to confusing output, e.g., if the user thinks that xconfig is in Show All Options mode but kconfig options which have an unmet dependency are still being listed. Add code to show the radio button for the current Show...Options mode during startup so that it will reflect the current config setting. Signed-off-by: Randy Dunlap Tested-by: Nicolas Schier Acked-by: Nicolas Schier Link: https://lore.kernel.org/r/20250812223502.1356426-1-rdunlap@infradead.org Signed-off-by: Nathan Chancellor --- scripts/kconfig/qconf.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'scripts') diff --git a/scripts/kconfig/qconf.cc b/scripts/kconfig/qconf.cc index f8992db1870a..b84c9f2485d1 100644 --- a/scripts/kconfig/qconf.cc +++ b/scripts/kconfig/qconf.cc @@ -1377,6 +1377,19 @@ ConfigMainWindow::ConfigMainWindow(void) ConfigList::showPromptAction = new QAction("Show Prompt Options", optGroup); ConfigList::showPromptAction->setCheckable(true); + switch (configList->optMode) { + case allOpt: + ConfigList::showAllAction->setChecked(true); + break; + case promptOpt: + ConfigList::showPromptAction->setChecked(true); + break; + case normalOpt: + default: + ConfigList::showNormalAction->setChecked(true); + break; + } + QAction *showDebugAction = new QAction("Show Debug Info", this); showDebugAction->setCheckable(true); connect(showDebugAction, &QAction::toggled, -- cgit From 4e9a563f0774fdd6f1bd68235314b4b8298fc0a9 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:44 +0200 Subject: scripts: sphinx-pre-install: fix PDF build issues on Ubuntu PDF output with current Debian-based distros require other packages for it to work. The main one is pzdr.tfm. Without that, \sphinxhyphen{} won't work, affecting multiple docs. For CJK, tex-gyre is required. Add the missing packages to the list. After the change, all PDF files build on latest Ubuntu: Ubuntu 25.04: ------------- PASSED: OS detection: Ubuntu 25.04 SKIPPED (Sphinx Sphinx 8.1.3): System packages SKIPPED (Sphinx already installed either as venv or as native package): Sphinx on venv SKIPPED (Sphinx already installed either as venv or as native package): Sphinx package PASSED: Clean documentation: Build time: 0:00, return code: 0 PASSED: Build HTML documentation: Build time: 3:28, return code: 0 PASSED: Build PDF documentation: Build time: 11:08, return code: 0 PDF docs: --------- PASSED: dev-tools: pdf/dev-tools.pdf PASSED: tools: pdf/tools.pdf PASSED: filesystems: pdf/filesystems.pdf PASSED: w1: pdf/w1.pdf PASSED: maintainer: pdf/maintainer.pdf PASSED: process: pdf/process.pdf PASSED: isdn: pdf/isdn.pdf PASSED: fault-injection: pdf/fault-injection.pdf PASSED: iio: pdf/iio.pdf PASSED: scheduler: pdf/scheduler.pdf PASSED: staging: pdf/staging.pdf PASSED: fpga: pdf/fpga.pdf PASSED: power: pdf/power.pdf PASSED: leds: pdf/leds.pdf PASSED: edac: pdf/edac.pdf PASSED: PCI: pdf/PCI.pdf PASSED: firmware-guide: pdf/firmware-guide.pdf PASSED: cpu-freq: pdf/cpu-freq.pdf PASSED: mhi: pdf/mhi.pdf PASSED: wmi: pdf/wmi.pdf PASSED: timers: pdf/timers.pdf PASSED: accel: pdf/accel.pdf PASSED: hid: pdf/hid.pdf PASSED: userspace-api: pdf/userspace-api.pdf PASSED: spi: pdf/spi.pdf PASSED: networking: pdf/networking.pdf PASSED: virt: pdf/virt.pdf PASSED: nvme: pdf/nvme.pdf PASSED: translations: pdf/translations.pdf PASSED: input: pdf/input.pdf PASSED: tee: pdf/tee.pdf PASSED: doc-guide: pdf/doc-guide.pdf PASSED: cdrom: pdf/cdrom.pdf PASSED: gpu: pdf/gpu.pdf PASSED: i2c: pdf/i2c.pdf PASSED: RCU: pdf/RCU.pdf PASSED: watchdog: pdf/watchdog.pdf PASSED: usb: pdf/usb.pdf PASSED: rust: pdf/rust.pdf PASSED: crypto: pdf/crypto.pdf PASSED: kbuild: pdf/kbuild.pdf PASSED: livepatch: pdf/livepatch.pdf PASSED: mm: pdf/mm.pdf PASSED: locking: pdf/locking.pdf PASSED: infiniband: pdf/infiniband.pdf PASSED: driver-api: pdf/driver-api.pdf PASSED: bpf: pdf/bpf.pdf PASSED: devicetree: pdf/devicetree.pdf PASSED: block: pdf/block.pdf PASSED: target: pdf/target.pdf PASSED: arch: pdf/arch.pdf PASSED: pcmcia: pdf/pcmcia.pdf PASSED: scsi: pdf/scsi.pdf PASSED: netlabel: pdf/netlabel.pdf PASSED: sound: pdf/sound.pdf PASSED: security: pdf/security.pdf PASSED: accounting: pdf/accounting.pdf PASSED: admin-guide: pdf/admin-guide.pdf PASSED: core-api: pdf/core-api.pdf PASSED: fb: pdf/fb.pdf PASSED: peci: pdf/peci.pdf PASSED: trace: pdf/trace.pdf PASSED: misc-devices: pdf/misc-devices.pdf PASSED: kernel-hacking: pdf/kernel-hacking.pdf PASSED: hwmon: pdf/hwmon.pdf Reported-by: Akira Yokosawa Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/b5e2e0df68b377b148fdbdd721f6c1cbefe6f861.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index b8474848df4e..b24a6f91ec0a 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -764,9 +764,6 @@ class SphinxDependencyChecker(MissingCheckers): if self.pdf: pdf_pkgs = { - "texlive-lang-chinese": [ - "/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty", - ], "fonts-dejavu": [ "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", ], @@ -775,6 +772,15 @@ class SphinxDependencyChecker(MissingCheckers): "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc", ], + "tex-gyre": [ + "/usr/share/texmf/tex/latex/tex-gyre/tgtermes.sty" + ], + "texlive-fonts-recommended": [ + "/usr/share/texlive/texmf-dist/fonts/tfm/adobe/zapfding/pzdr.tfm", + ], + "texlive-lang-chinese": [ + "/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty", + ], } for package, files in pdf_pkgs.items(): -- cgit From 9ff5c2f51da284906c2f89e7d594f31be69e7abe Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:45 +0200 Subject: scripts: sphinx-pre-install: add missing gentoo pdf dependencies There are two packages that are required to build PDF at gentoo: dev-texlive/texlive-latexextra media-fonts/lm Place latex_dependencies on a list to make it easier to maintain and add the missing ones. With that, most PDF documents now build on Gentoo: Gentoo Base System release 2.17: -------------------------------- PASSED: OS detection: Gentoo Base System release 2.17 SKIPPED (Sphinx Sphinx 8.2.3): System packages SKIPPED (Sphinx already installed either as venv or as native package): Sphinx on venv SKIPPED (Sphinx already installed either as venv or as native package): Sphinx package PASSED: Clean documentation: Build time: 0:00, return code: 0 PASSED: Build HTML documentation: Build time: 5:28, return code: 0 PARTIAL: Build PDF documentation: Test failed (Build time: 9:19, return code: 2) PDF docs: --------- PASSED: dev-tools: pdf/dev-tools.pdf PASSED: tools: pdf/tools.pdf PASSED: filesystems: pdf/filesystems.pdf PASSED: w1: pdf/w1.pdf PASSED: maintainer: pdf/maintainer.pdf PASSED: process: pdf/process.pdf PASSED: isdn: pdf/isdn.pdf PASSED: fault-injection: pdf/fault-injection.pdf PASSED: iio: pdf/iio.pdf PASSED: scheduler: pdf/scheduler.pdf PASSED: staging: pdf/staging.pdf PASSED: fpga: pdf/fpga.pdf PASSED: power: pdf/power.pdf PASSED: leds: pdf/leds.pdf PASSED: edac: pdf/edac.pdf PASSED: PCI: pdf/PCI.pdf PASSED: firmware-guide: pdf/firmware-guide.pdf PASSED: cpu-freq: pdf/cpu-freq.pdf PASSED: mhi: pdf/mhi.pdf PASSED: wmi: pdf/wmi.pdf PASSED: timers: pdf/timers.pdf PASSED: accel: pdf/accel.pdf PASSED: hid: pdf/hid.pdf FAILED: userspace-api: Build failed (FAILED) PASSED: spi: pdf/spi.pdf PASSED: networking: pdf/networking.pdf PASSED: virt: pdf/virt.pdf PASSED: nvme: pdf/nvme.pdf FAILED: translations: Build failed (FAILED) PASSED: input: pdf/input.pdf PASSED: tee: pdf/tee.pdf PASSED: doc-guide: pdf/doc-guide.pdf PASSED: cdrom: pdf/cdrom.pdf FAILED: gpu: Build failed (FAILED) FAILED: i2c: Build failed (FAILED) FAILED: RCU: Build failed (FAILED) PASSED: watchdog: pdf/watchdog.pdf PASSED: usb: pdf/usb.pdf PASSED: rust: pdf/rust.pdf PASSED: crypto: pdf/crypto.pdf PASSED: kbuild: pdf/kbuild.pdf PASSED: livepatch: pdf/livepatch.pdf PASSED: mm: pdf/mm.pdf PASSED: locking: pdf/locking.pdf PASSED: infiniband: pdf/infiniband.pdf PASSED: driver-api: pdf/driver-api.pdf PASSED: bpf: pdf/bpf.pdf PASSED: devicetree: pdf/devicetree.pdf PASSED: block: pdf/block.pdf PASSED: target: pdf/target.pdf FAILED: arch: Build failed (FAILED) PASSED: pcmcia: pdf/pcmcia.pdf PASSED: scsi: pdf/scsi.pdf PASSED: netlabel: pdf/netlabel.pdf PASSED: sound: pdf/sound.pdf PASSED: security: pdf/security.pdf PASSED: accounting: pdf/accounting.pdf PASSED: admin-guide: pdf/admin-guide.pdf FAILED: core-api: Build failed (FAILED) PASSED: fb: pdf/fb.pdf PASSED: peci: pdf/peci.pdf PASSED: trace: pdf/trace.pdf PASSED: misc-devices: pdf/misc-devices.pdf PASSED: kernel-hacking: pdf/kernel-hacking.pdf PASSED: hwmon: pdf/hwmon.pdf Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0ac8d6b7484aaf930917c8edde53742d425e7e8f.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index b24a6f91ec0a..f987abfec802 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -1058,12 +1058,19 @@ class SphinxDependencyChecker(MissingCheckers): """ Provide package installation hints for Gentoo. """ + texlive_deps = [ + "dev-texlive/texlive-latexextra", + "dev-texlive/texlive-xetex", + "media-fonts/dejavu", + "media-fonts/lm", + ] + progs = { "convert": "media-gfx/imagemagick", "dot": "media-gfx/graphviz", "rsvg-convert": "gnome-base/librsvg", "virtualenv": "dev-python/virtualenv", - "xelatex": "dev-texlive/texlive-xetex media-fonts/dejavu", + "xelatex": " ".join(texlive_deps), "yaml": "dev-python/pyyaml", "python-sphinx": "dev-python/sphinx", } -- cgit From b2d5d61c1371dc6e0ddda69a0a2c921e3ceb928e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:46 +0200 Subject: scripts: sphinx-pre-install: fix PDF dependencies for openSuse The dependencies are outdated: both versions need texlive-dejavu fonts. Also, for PDF generation, python311-Sphinx-latex is required. With that, all PDF files are now tuilt on both: openSUSE Leap 15.6: ------------------- PASSED: OS detection: openSUSE Leap 15.6 SKIPPED (Sphinx Sphinx 7.2.6): System packages SKIPPED (Sphinx already installed either as venv or as native package): Sphinx on venv SKIPPED (Sphinx already installed either as venv or as native package): Sphinx package PASSED: Clean documentation: Build time: 0:00, return code: 0 PASSED: Build HTML documentation: Build time: 5:29, return code: 0 PASSED: Build PDF documentation: Build time: 13:45, return code: 0 openSUSE Tumbleweed: -------------------- PASSED: OS detection: openSUSE Tumbleweed SKIPPED (Sphinx Sphinx 8.2.3): System packages SKIPPED (Sphinx already installed either as venv or as native package): Sphinx on venv SKIPPED (Sphinx already installed either as venv or as native package): Sphinx package PASSED: Clean documentation: Build time: 0:00, return code: 0 PASSED: Build HTML documentation: Build time: 4:33, return code: 0 PASSED: Build PDF documentation: Build time: 13:18, return code: 0 Summary ======= PASSED - openSUSE Leap 15.6 (7 tests) PASSED - openSUSE Tumbleweed (7 tests) Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/d78457376f9dfd24cb7ac3a32895c654412715f3.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index f987abfec802..86f129c76ecd 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -901,7 +901,7 @@ class SphinxDependencyChecker(MissingCheckers): "dot": "graphviz", "python-sphinx": "python3-sphinx", "virtualenv": "python3-virtualenv", - "xelatex": "texlive-xetex-bin", + "xelatex": "texlive-xetex-bin texlive-dejavu", "yaml": "python3-pyyaml", } @@ -937,7 +937,7 @@ class SphinxDependencyChecker(MissingCheckers): self.recommend_python = True progs.update({ - "python-sphinx": "python311-Sphinx", + "python-sphinx": "python311-Sphinx python311-Sphinx-latex", "virtualenv": "python311-virtualenv", "yaml": "python311-PyYAML", }) @@ -945,7 +945,7 @@ class SphinxDependencyChecker(MissingCheckers): # Tumbleweed defaults to Python 3.11 progs.update({ - "python-sphinx": "python313-Sphinx", + "python-sphinx": "python313-Sphinx python313-Sphinx-latex", "virtualenv": "python313-virtualenv", "yaml": "python313-PyYAML", }) -- cgit From b51f8c12d16bcf29496ebaf1d7cf3587ca28ba0a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:47 +0200 Subject: scripts: sphinx-pre-install: fix dependencies for OpenMandriva The dependeny list for OpenMandriva is wrong. Update it. Yet, on my tests with OpenMandriva LX 4.3, the texlive packages are broken: xelatex can't build anything there, as it lacks xelatex.sfm. Yet, this could be a problem at the way I created the container. Just in case, add a note about that. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/669e759ba366328e5c8d5b14a591ba45a1f58176.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 86f129c76ecd..224db3af17db 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -984,13 +984,19 @@ class SphinxDependencyChecker(MissingCheckers): if re.search(r"OpenMandriva", self.system_release): packager_cmd = "dnf install" noto_sans = "noto-sans-cjk-fonts" - tex_pkgs = ["texlive-collection-fontsextra"] + tex_pkgs = [ + "texlive-collection-basic", + "texlive-collection-langcjk", + "texlive-collection-fontsextra", + "texlive-collection-fontsrecommended" + ] # Tested on OpenMandriva Lx 4.3 progs["convert"] = "imagemagick" progs["yaml"] = "python-pyyaml" progs["python-virtualenv"] = "python-virtualenv" progs["python-sphinx"] = "python-sphinx" + progs["xelatex"] = "texlive" self.check_program("python-virtualenv", DepManager.PYTHON_MANDATORY) @@ -1004,7 +1010,9 @@ class SphinxDependencyChecker(MissingCheckers): if not self.distro_msg: self.distro_msg = \ - "Note: for venv, ensurepip could be broken, preventing its install method." + "Notes:\n"\ + "1. for venv, ensurepip could be broken, preventing its install method.\n" \ + "2. at least on OpenMandriva LX 4.3, texlive packages seem broken" else: packager_cmd = "urpmi" -- cgit From c71c5d6dcb34423c0f15c146240b25a0e78f65aa Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:48 +0200 Subject: scripts: sphinx-pre-install: fix pdf dependencies for Mageia 9 On Mageia 9, two packages are missing. Add them. With that, all PDF packages now build: Mageia 9: --------- PASSED: OS detection: Mageia 9 PASSED: System packages: Packages installed PASSED: Sphinx on venv: Sphinx Sphinx 8.1.3 PASSED: Sphinx package: Sphinx Sphinx 6.1.3 PASSED: Clean documentation: Build time: 0:00, return code: 0 PASSED: Build HTML documentation: Build time: 5:17, return code: 0 PASSED: Build PDF documentation: Build time: 14:28, return code: 0 PDF docs: --------- PASSED: dev-tools: pdf/dev-tools.pdf PASSED: tools: pdf/tools.pdf PASSED: filesystems: pdf/filesystems.pdf PASSED: w1: pdf/w1.pdf PASSED: maintainer: pdf/maintainer.pdf PASSED: process: pdf/process.pdf PASSED: isdn: pdf/isdn.pdf PASSED: fault-injection: pdf/fault-injection.pdf PASSED: iio: pdf/iio.pdf PASSED: scheduler: pdf/scheduler.pdf PASSED: staging: pdf/staging.pdf PASSED: fpga: pdf/fpga.pdf PASSED: power: pdf/power.pdf PASSED: leds: pdf/leds.pdf PASSED: edac: pdf/edac.pdf PASSED: PCI: pdf/PCI.pdf PASSED: firmware-guide: pdf/firmware-guide.pdf PASSED: cpu-freq: pdf/cpu-freq.pdf PASSED: mhi: pdf/mhi.pdf PASSED: wmi: pdf/wmi.pdf PASSED: timers: pdf/timers.pdf PASSED: accel: pdf/accel.pdf PASSED: hid: pdf/hid.pdf PASSED: userspace-api: pdf/userspace-api.pdf PASSED: spi: pdf/spi.pdf PASSED: networking: pdf/networking.pdf PASSED: virt: pdf/virt.pdf PASSED: nvme: pdf/nvme.pdf PASSED: translations: pdf/translations.pdf PASSED: input: pdf/input.pdf PASSED: tee: pdf/tee.pdf PASSED: doc-guide: pdf/doc-guide.pdf PASSED: cdrom: pdf/cdrom.pdf PASSED: gpu: pdf/gpu.pdf PASSED: i2c: pdf/i2c.pdf PASSED: RCU: pdf/RCU.pdf PASSED: watchdog: pdf/watchdog.pdf PASSED: usb: pdf/usb.pdf PASSED: rust: pdf/rust.pdf PASSED: crypto: pdf/crypto.pdf PASSED: kbuild: pdf/kbuild.pdf PASSED: livepatch: pdf/livepatch.pdf PASSED: mm: pdf/mm.pdf PASSED: locking: pdf/locking.pdf PASSED: infiniband: pdf/infiniband.pdf PASSED: driver-api: pdf/driver-api.pdf PASSED: bpf: pdf/bpf.pdf PASSED: devicetree: pdf/devicetree.pdf PASSED: block: pdf/block.pdf PASSED: target: pdf/target.pdf PASSED: arch: pdf/arch.pdf PASSED: pcmcia: pdf/pcmcia.pdf PASSED: scsi: pdf/scsi.pdf PASSED: netlabel: pdf/netlabel.pdf PASSED: sound: pdf/sound.pdf PASSED: security: pdf/security.pdf PASSED: accounting: pdf/accounting.pdf PASSED: admin-guide: pdf/admin-guide.pdf PASSED: core-api: pdf/core-api.pdf PASSED: fb: pdf/fb.pdf PASSED: peci: pdf/peci.pdf PASSED: trace: pdf/trace.pdf PASSED: misc-devices: pdf/misc-devices.pdf PASSED: kernel-hacking: pdf/kernel-hacking.pdf PASSED: hwmon: pdf/hwmon.pdf Summary ======= PASSED - Mageia 9 (7 tests) Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/bd6e03c79b890ad0168493cdb4cdaf610bbc8c45.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 224db3af17db..758a84ae6347 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -979,6 +979,8 @@ class SphinxDependencyChecker(MissingCheckers): tex_pkgs = [ "texlive-fontsextra", + "texlive-fonts-asian", + "fonts-ttf-dejavu", ] if re.search(r"OpenMandriva", self.system_release): -- cgit From 4509d36ceea288a78bb256416d9ca38d3298c7cb Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:49 +0200 Subject: scripts: sphinx-pre-install: fix PDF dependencies for gentoo Package fonts are wrong. Fix it. With that, most PDF files now builds. PDF docs: --------- PASSED: dev-tools: pdf/dev-tools.pdf PASSED: tools: pdf/tools.pdf PASSED: filesystems: pdf/filesystems.pdf PASSED: w1: pdf/w1.pdf PASSED: maintainer: pdf/maintainer.pdf PASSED: process: pdf/process.pdf PASSED: isdn: pdf/isdn.pdf PASSED: fault-injection: pdf/fault-injection.pdf PASSED: iio: pdf/iio.pdf PASSED: scheduler: pdf/scheduler.pdf PASSED: staging: pdf/staging.pdf PASSED: fpga: pdf/fpga.pdf PASSED: power: pdf/power.pdf PASSED: leds: pdf/leds.pdf PASSED: edac: pdf/edac.pdf PASSED: PCI: pdf/PCI.pdf PASSED: firmware-guide: pdf/firmware-guide.pdf PASSED: cpu-freq: pdf/cpu-freq.pdf PASSED: mhi: pdf/mhi.pdf PASSED: wmi: pdf/wmi.pdf PASSED: timers: pdf/timers.pdf PASSED: accel: pdf/accel.pdf PASSED: hid: pdf/hid.pdf FAILED: userspace-api: Build failed (FAILED) PASSED: spi: pdf/spi.pdf PASSED: networking: pdf/networking.pdf PASSED: virt: pdf/virt.pdf PASSED: nvme: pdf/nvme.pdf FAILED: translations: Build failed (FAILED) PASSED: input: pdf/input.pdf PASSED: tee: pdf/tee.pdf PASSED: doc-guide: pdf/doc-guide.pdf PASSED: cdrom: pdf/cdrom.pdf FAILED: gpu: Build failed (FAILED) FAILED: i2c: Build failed (FAILED) FAILED: RCU: Build failed (FAILED) PASSED: watchdog: pdf/watchdog.pdf PASSED: usb: pdf/usb.pdf PASSED: rust: pdf/rust.pdf PASSED: crypto: pdf/crypto.pdf PASSED: kbuild: pdf/kbuild.pdf PASSED: livepatch: pdf/livepatch.pdf PASSED: mm: pdf/mm.pdf PASSED: locking: pdf/locking.pdf PASSED: infiniband: pdf/infiniband.pdf PASSED: driver-api: pdf/driver-api.pdf PASSED: bpf: pdf/bpf.pdf PASSED: devicetree: pdf/devicetree.pdf PASSED: block: pdf/block.pdf PASSED: target: pdf/target.pdf FAILED: arch: Build failed (FAILED) PASSED: pcmcia: pdf/pcmcia.pdf PASSED: scsi: pdf/scsi.pdf PASSED: netlabel: pdf/netlabel.pdf PASSED: sound: pdf/sound.pdf PASSED: security: pdf/security.pdf PASSED: accounting: pdf/accounting.pdf PASSED: admin-guide: pdf/admin-guide.pdf FAILED: core-api: Build failed (FAILED) PASSED: fb: pdf/fb.pdf PASSED: peci: pdf/peci.pdf PASSED: trace: pdf/trace.pdf PASSED: misc-devices: pdf/misc-devices.pdf PASSED: kernel-hacking: pdf/kernel-hacking.pdf PASSED: hwmon: pdf/hwmon.pdf Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/1ccbac9fd1f4e598dda82e775b64768ec3696248.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 758a84ae6347..c46d7b76f93c 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -1069,10 +1069,10 @@ class SphinxDependencyChecker(MissingCheckers): Provide package installation hints for Gentoo. """ texlive_deps = [ + "dev-texlive/texlive-fontsrecommended", "dev-texlive/texlive-latexextra", "dev-texlive/texlive-xetex", "media-fonts/dejavu", - "media-fonts/lm", ] progs = { -- cgit From c6e23912855d4848883080200e09551b6dcbc7df Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Aug 2025 10:16:50 +0200 Subject: scripts/sphinx-pre-install: fix Archlinux PDF dependencies There are some missing packages causing PDF build to fail on Archlinux and add latexmk (from texlive-binextra package). Yet, at least today, PDF builds are failing on a very late stage, when trying to run xdvipdfmx: $ xdvipdfmx -E -o "peci.pdf" "peci.xdv" xdvipdfmx:fatal: Unrecognized paper format: # Simply write the paper name. See man 1 paper and "paper --no-size --all" for possible values Despite its message, even using a very simple document like: \def\sphinxdocclass{report} \documentclass[a4paper,11pt,english]{sphinxmanual} \begin{document} Test \end{document} or even: \def\sphinxdocclass{report} \documentclass{sphinxmanual} \begin{document} Test \end{document} Is causing xdvipdfmx to complain about geometry. As Archlinux is a rolling release distro, maybe I got it on a bad day. So, let's fix it in the hope that soon enough someone would fix the issues there. Such broken scenario happens with those packages installed: texlive-basic 2025.2-1 texlive-bin 2025.2-1 texlive-binextra 2025.2-1 texlive-fontsrecommended 2025.2-1 texlive-langchinese 2025.2-1 texlive-langcjk 2025.2-1 texlive-latex 2025.2-1 texlive-latexextra 2025.2-1 texlive-latexrecommended 2025.2-1 texlive-pictures 2025.2-1 texlive-xetex 2025.2-1 python-docutils 1:0.21.2-3 python-sphinx 8.2.3-1 python-sphinx-alabaster-theme 1.0.0-4 python-sphinxcontrib-applehelp 2.0.0-3 python-sphinxcontrib-devhelp 2.0.0-4 python-sphinxcontrib-htmlhelp 2.1.0-3 python-sphinxcontrib-jsmath 1.0.1-19 python-sphinxcontrib-qthelp 2.0.0-3 python-sphinxcontrib-serializinghtml 2.0.0-3 Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/574d902f7691861e18339217f42409850ee58791.1755763127.git.mchehab+huawei@kernel.org --- scripts/sphinx-pre-install | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'scripts') diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index c46d7b76f93c..954ed3dc0645 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -1048,7 +1048,12 @@ class SphinxDependencyChecker(MissingCheckers): } archlinux_tex_pkgs = [ + "texlive-basic", + "texlive-binextra", "texlive-core", + "texlive-fontsrecommended", + "texlive-langchinese", + "texlive-langcjk", "texlive-latexextra", "ttf-dejavu", ] -- cgit From 0354e81b7bd629f9c3379c9524e988ebc504fa25 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Mon, 25 Aug 2025 17:00:37 +0200 Subject: scripts/misc-check: update export checks for EXPORT_SYMBOL_FOR_MODULES() The module export checks are looking for EXPORT_SYMBOL_GPL_FOR_MODULES() which was renamed to EXPORT_SYMBOL_FOR_MODULES(). Update the checks. Fixes: 6d3c3ca4c77e ("module: Rename EXPORT_SYMBOL_GPL_FOR_MODULES to EXPORT_SYMBOL_FOR_MODULES") Signed-off-by: Vlastimil Babka Reviewed-by: Daniel Gomez Reviewed-by: Nicolas Schier Link: https://lore.kernel.org/r/20250825-export_modules_fix-v1-1-5c331e949538@suse.cz Signed-off-by: Nathan Chancellor --- scripts/misc-check | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/misc-check b/scripts/misc-check index 84f08da17b2c..40e5a4b01ff4 100755 --- a/scripts/misc-check +++ b/scripts/misc-check @@ -45,7 +45,7 @@ check_tracked_ignored_files () { # does not automatically fix it. check_missing_include_linux_export_h () { - git -C "${srctree:-.}" grep --files-with-matches -E 'EXPORT_SYMBOL((_NS)?(_GPL)?|_GPL_FOR_MODULES)\(.*\)' \ + git -C "${srctree:-.}" grep --files-with-matches -E 'EXPORT_SYMBOL((_NS)?(_GPL)?|_FOR_MODULES)\(.*\)' \ -- '*.[ch]' :^tools/ :^include/linux/export.h | xargs -r git -C "${srctree:-.}" grep --files-without-match '#include[[:space:]]*' | xargs -r printf "%s: warning: EXPORT_SYMBOL() is used, but #include is missing\n" >&2 @@ -58,7 +58,7 @@ check_unnecessary_include_linux_export_h () { git -C "${srctree:-.}" grep --files-with-matches '#include[[:space:]]*' \ -- '*.[c]' :^tools/ | - xargs -r git -C "${srctree:-.}" grep --files-without-match -E 'EXPORT_SYMBOL((_NS)?(_GPL)?|_GPL_FOR_MODULES)\(.*\)' | + xargs -r git -C "${srctree:-.}" grep --files-without-match -E 'EXPORT_SYMBOL((_NS)?(_GPL)?|_FOR_MODULES)\(.*\)' | xargs -r printf "%s: warning: EXPORT_SYMBOL() is not used, but #include is present\n" >&2 } -- cgit From 51337a9a3a404fde0f5337662ffc7699793dfeb5 Mon Sep 17 00:00:00 2001 From: Ada Couprie Diaz Date: Thu, 21 Aug 2025 13:07:35 +0100 Subject: kasan: fix GCC mem-intrinsic prefix with sw tags GCC doesn't support "hwasan-kernel-mem-intrinsic-prefix", only "asan-kernel-mem-intrinsic-prefix"[0], while LLVM supports both. This is already taken into account when checking "CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX", but not in the KASAN Makefile adding those parameters when "CONFIG_KASAN_SW_TAGS" is enabled. Replace the version check with "CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX", which already validates that mem-intrinsic prefix parameter can be used, and choose the correct name depending on compiler. GCC 13 and above trigger "CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX" which prevents `mem{cpy,move,set}()` being redefined in "mm/kasan/shadow.c" since commit 36be5cba99f6 ("kasan: treat meminstrinsic as builtins in uninstrumented files"), as we expect the compiler to prefix those calls with `__(hw)asan_` instead. But as the option passed to GCC has been incorrect, the compiler has not been emitting those prefixes, effectively never calling the instrumented versions of `mem{cpy,move,set}()` with "CONFIG_KASAN_SW_TAGS" enabled. If "CONFIG_FORTIFY_SOURCES" is enabled, this issue would be mitigated as it redefines `mem{cpy,move,set}()` and properly aliases the `__underlying_mem*()` that will be called to the instrumented versions. Link: https://lkml.kernel.org/r/20250821120735.156244-1-ada.coupriediaz@arm.com Link: https://gcc.gnu.org/onlinedocs/gcc-13.4.0/gcc/Optimize-Options.html [0] Signed-off-by: Ada Couprie Diaz Fixes: 36be5cba99f6 ("kasan: treat meminstrinsic as builtins in uninstrumented files") Reviewed-by: Yeoreum Yun Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Dmitriy Vyukov Cc: Marco Elver Cc: Marc Rutland Cc: Michael Ellerman Cc: Nathan Chancellor Cc: Vincenzo Frascino Cc: Signed-off-by: Andrew Morton --- scripts/Makefile.kasan | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.kasan b/scripts/Makefile.kasan index 693dbbebebba..0ba2aac3b8dc 100644 --- a/scripts/Makefile.kasan +++ b/scripts/Makefile.kasan @@ -86,10 +86,14 @@ kasan_params += hwasan-instrument-stack=$(stack_enable) \ hwasan-use-short-granules=0 \ hwasan-inline-all-checks=0 -# Instrument memcpy/memset/memmove calls by using instrumented __hwasan_mem*(). -ifeq ($(call clang-min-version, 150000)$(call gcc-min-version, 130000),y) - kasan_params += hwasan-kernel-mem-intrinsic-prefix=1 -endif +# Instrument memcpy/memset/memmove calls by using instrumented __(hw)asan_mem*(). +ifdef CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX + ifdef CONFIG_CC_IS_GCC + kasan_params += asan-kernel-mem-intrinsic-prefix=1 + else + kasan_params += hwasan-kernel-mem-intrinsic-prefix=1 + endif +endif # CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX endif # CONFIG_KASAN_SW_TAGS -- cgit From c2a756891bb428104fa8899998ba277042274cdb Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 25 Aug 2025 13:18:28 -0700 Subject: uapi: wrap compiler_types.h in an ifdef instead of the implicit strip The uAPI stddef header includes compiler_types.h, a kernel-only header, to make sure that kernel definitions of annotations like __counted_by() take precedence. There is a hack in scripts/headers_install.sh which strips includes of compiler.h and compiler_types.h when installing uAPI headers. While explicit handling makes sense for compiler.h, which is included all over the uAPI, compiler_types.h is only included by stddef.h (within the uAPI, obviously it's included in kernel code a lot). Remove the stripping from scripts/headers_install.sh and wrap the include of compiler_types.h in #ifdef __KERNEL__ instead. This should be equivalent functionally, but is easier to understand to a casual reader of the code. It also makes it easier to work with kernel headers directly from under tools/ Signed-off-by: Jakub Kicinski Reviewed-by: Simon Horman Link: https://patch.msgid.link/20250825201828.2370083-1-kuba@kernel.org Signed-off-by: Paolo Abeni --- scripts/headers_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/headers_install.sh b/scripts/headers_install.sh index 6bbccb43f7e7..4c20c62c4faf 100755 --- a/scripts/headers_install.sh +++ b/scripts/headers_install.sh @@ -32,7 +32,7 @@ fi sed -E -e ' s/([[:space:](])(__user|__force|__iomem)[[:space:]]/\1/g s/__attribute_const__([[:space:]]|$)/\1/g - s@^#include @@ + s@^#include @@ s/(^|[^a-zA-Z0-9])__packed([^a-zA-Z0-9_]|$)/\1__attribute__((packed))\2/g s/(^|[[:space:](])(inline|asm|volatile)([[:space:](]|$)/\1__\2__\3/g s@#(ifndef|define|endif[[:space:]]*/[*])[[:space:]]*_UAPI@#\1 @ -- cgit From 1e150869caf2fe540dd9c3367e124b59c23228c7 Mon Sep 17 00:00:00 2001 From: Maxime Thiebaut Date: Fri, 22 Aug 2025 10:40:03 +0200 Subject: extract-vmlinux: Output used decompression method When extract-vmlinux succeeds, it doesn't output which decompression method was found at which offset. Adding this additional output in check_vmlinux() helps troubleshooting and reverse-engineering images. The last check_vmlinux() call was also quoted to accept spaces. Signed-off-by: Maxime Thiebaut Reviewed-by: Nicolas Schier Link: https://lore.kernel.org/r/X6OQ4pHdpreJtlTnf0tFEb4Uxz8T8gFv_7Yw6tpBK4ZBgHYjJr_URwUwCVynpkb-H8Yjk7DdBF01zY-sfqu_7N5trZQfcd6s_4PtdGlHtlA=@thiebaut.dev Signed-off-by: Nathan Chancellor --- scripts/extract-vmlinux | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/extract-vmlinux b/scripts/extract-vmlinux index 189956b5a5c8..266df9bc7a48 100755 --- a/scripts/extract-vmlinux +++ b/scripts/extract-vmlinux @@ -10,12 +10,15 @@ # # ---------------------------------------------------------------------- +me=${0##*/} + check_vmlinux() { if file "$1" | grep -q 'Linux kernel.*boot executable' || readelf -h "$1" > /dev/null 2>&1 then cat "$1" + echo "$me: Extracted vmlinux using '$2' from offset $3" >&2 exit 0 fi } @@ -30,12 +33,11 @@ try_decompress() do pos=${pos%%:*} tail -c+$pos "$img" | $3 > $tmp 2> /dev/null - check_vmlinux $tmp + check_vmlinux $tmp "$3" $pos done } # Check invocation: -me=${0##*/} img=$1 if [ $# -ne 1 -o ! -s "$img" ] then @@ -57,7 +59,7 @@ try_decompress '\002!L\030' xxx 'lz4 -d' try_decompress '(\265/\375' xxx unzstd # Finally check for uncompressed images or objects: -check_vmlinux $img +check_vmlinux "$img" cat 0 # Bail out: echo "$me: Cannot find vmlinux." >&2 -- cgit From 20c0989283564d662dcc4c77252a60b1812b32ca Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Thu, 21 Aug 2025 14:15:38 -0700 Subject: kbuild: Bump minimum version of LLVM for building the kernel to 15.0.0 s390 and x86 have required LLVM 15 since 30d17fac6aae ("scripts/min-tool-version.sh: raise minimum clang version to 15.0.0 for s390") 7861640aac52 ("x86/build: Raise the minimum LLVM version to 15.0.0") respectively but most other architectures allow LLVM 13.0.1 or newer. In accordance with the recent minimum supported version of GCC bump that happened in 118c40b7b503 ("kbuild: require gcc-8 and binutils-2.30") do the same for LLVM to 15.0.0. Of the supported releases of Arch Linux, Debian, Fedora, and OpenSUSE surveyed in evaluating this bump, this only leaves behind Debian Bookworm (14.0.6) and Ubuntu Jammy (14.0.0). Debian Trixie has 19.1.7 and Ubuntu Noble has 18.1.3 (so there are viable upgrade paths) or users can use apt.llvm.org, which provides even newer packages for those distributions. Reviewed-by: Kees Cook Acked-by: Arnd Bergmann Acked-by: Nicolas Schier Link: https://lore.kernel.org/r/20250821-bump-min-llvm-ver-15-v2-1-635f3294e5f0@kernel.org Signed-off-by: Nathan Chancellor --- scripts/min-tool-version.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/min-tool-version.sh b/scripts/min-tool-version.sh index 0d223b4a9445..99b5575c1ef7 100755 --- a/scripts/min-tool-version.sh +++ b/scripts/min-tool-version.sh @@ -24,12 +24,10 @@ gcc) fi ;; llvm) - if [ "$SRCARCH" = s390 -o "$SRCARCH" = x86 ]; then - echo 15.0.0 - elif [ "$SRCARCH" = loongarch ]; then + if [ "$SRCARCH" = loongarch ]; then echo 18.0.0 else - echo 13.0.1 + echo 15.0.0 fi ;; rustc) -- cgit From 362f92286065d9f8282da5def89e173a12191568 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 27 Aug 2025 08:11:31 -0700 Subject: lib/crypto: tests: Add KUnit tests for BLAKE2s Add a KUnit test suite for BLAKE2s. Most of the core test logic is in the previously-added hash-test-template.h. This commit just adds the actual KUnit suite, commits the generated test vectors to the tree so that gen-hash-testvecs.py won't have to be run at build time, and adds a few BLAKE2s-specific test cases. This is the replacement for blake2s-selftest, which an earlier commit removed. Improvements over blake2s-selftest include integration with KUnit, more comprehensive test cases, and support for benchmarking. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250827151131.27733-13-ebiggers@kernel.org Signed-off-by: Eric Biggers --- scripts/crypto/gen-hash-testvecs.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/crypto/gen-hash-testvecs.py b/scripts/crypto/gen-hash-testvecs.py index 4ac927d40cf5..fc063f2ee95f 100755 --- a/scripts/crypto/gen-hash-testvecs.py +++ b/scripts/crypto/gen-hash-testvecs.py @@ -84,11 +84,16 @@ def print_c_struct_u8_array_field(name, value): print_bytes('\t\t\t', value, 8) print('\t\t},') +def alg_digest_size_const(alg): + if alg == 'blake2s': + return 'BLAKE2S_HASH_SIZE' + return f'{alg.upper()}_DIGEST_SIZE' + def gen_unkeyed_testvecs(alg): print('') print('static const struct {') print('\tsize_t data_len;') - print(f'\tu8 digest[{alg.upper()}_DIGEST_SIZE];') + print(f'\tu8 digest[{alg_digest_size_const(alg)}];') print('} hash_testvecs[] = {') for data_len in DATA_LENS: data = rand_bytes(data_len) @@ -103,7 +108,7 @@ def gen_unkeyed_testvecs(alg): for data_len in range(len(data) + 1): hash_update(ctx, compute_hash(alg, data[:data_len])) print_static_u8_array_definition( - f'hash_testvec_consolidated[{alg.upper()}_DIGEST_SIZE]', + f'hash_testvec_consolidated[{alg_digest_size_const(alg)}]', hash_final(ctx)) def gen_hmac_testvecs(alg): @@ -119,6 +124,20 @@ def gen_hmac_testvecs(alg): f'hmac_testvec_consolidated[{alg.upper()}_DIGEST_SIZE]', ctx.digest()) +BLAKE2S_KEY_SIZE = 32 +BLAKE2S_HASH_SIZE = 32 + +def gen_additional_blake2s_testvecs(): + hashes = b'' + for key_len in range(BLAKE2S_KEY_SIZE + 1): + for out_len in range(1, BLAKE2S_HASH_SIZE + 1): + h = hashlib.blake2s(digest_size=out_len, key=rand_bytes(key_len)) + h.update(rand_bytes(100)) + hashes += h.digest() + print_static_u8_array_definition( + 'blake2s_keyed_testvec_consolidated[BLAKE2S_HASH_SIZE]', + compute_hash('blake2s', hashes)) + def gen_additional_poly1305_testvecs(): key = b'\xff' * POLY1305_KEY_SIZE data = b'' @@ -141,7 +160,9 @@ alg = sys.argv[1] print('/* SPDX-License-Identifier: GPL-2.0-or-later */') print(f'/* This file was generated by: {sys.argv[0]} {" ".join(sys.argv[1:])} */') gen_unkeyed_testvecs(alg) -if alg == 'poly1305': +if alg == 'blake2s': + gen_additional_blake2s_testvecs() +elif alg == 'poly1305': gen_additional_poly1305_testvecs() else: gen_hmac_testvecs(alg) -- cgit From 8a298579cdfcfdbcb70bb7af5e30769387494f37 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:35 +0200 Subject: scripts: sphinx-build-wrapper: get rid of uapi/media Makefile Now that kernel-include directive supports parsing data structs directly, we can finally get rid of the horrible hack we added to support parsing media uAPI symbols. As a side effect, Documentation/output doesn't have anymore media auto-generated .rst files on it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/5dbb257a4b283697271c9c7b8f4713857e8191c8.1755872208.git.mchehab+huawei@kernel.org --- scripts/sphinx-build-wrapper | 719 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 719 insertions(+) create mode 100755 scripts/sphinx-build-wrapper (limited to 'scripts') diff --git a/scripts/sphinx-build-wrapper b/scripts/sphinx-build-wrapper new file mode 100755 index 000000000000..abe8c26ae137 --- /dev/null +++ b/scripts/sphinx-build-wrapper @@ -0,0 +1,719 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2025 Mauro Carvalho Chehab +# +# pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103 +# +# Converted from docs Makefile and parallel-wrapper.sh, both under +# GPLv2, copyrighted since 2008 by the following authors: +# +# Akira Yokosawa +# Arnd Bergmann +# Breno Leitao +# Carlos Bilbao +# Dave Young +# Donald Hunter +# Geert Uytterhoeven +# Jani Nikula +# Jan Stancek +# Jonathan Corbet +# Joshua Clayton +# Kees Cook +# Linus Torvalds +# Magnus Damm +# Masahiro Yamada +# Mauro Carvalho Chehab +# Maxim Cournoyer +# Peter Foley +# Randy Dunlap +# Rob Herring +# Shuah Khan +# Thorsten Blum +# Tomas Winkler + + +""" +Sphinx build wrapper that handles Kernel-specific business rules: + +- it gets the Kernel build environment vars; +- it determines what's the best parallelism; +- it handles SPHINXDIRS + +This tool ensures that MIN_PYTHON_VERSION is satisfied. If version is +below that, it seeks for a new Python version. If found, it re-runs using +the newer version. +""" + +import argparse +import locale +import os +import re +import shlex +import shutil +import subprocess +import sys + +from concurrent import futures +from glob import glob + +LIB_DIR = "lib" +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) + +sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) + +from jobserver import JobserverExec # pylint: disable=C0413 + + +def parse_version(version): + """Convert a major.minor.patch version into a tuple""" + return tuple(int(x) for x in version.split(".")) + +def ver_str(version): + """Returns a version tuple as major.minor.patch""" + + return ".".join([str(x) for x in version]) + +# Minimal supported Python version needed by Sphinx and its extensions +MIN_PYTHON_VERSION = parse_version("3.7") + +# Default value for --venv parameter +VENV_DEFAULT = "sphinx_latest" + +# List of make targets and its corresponding builder and output directory +TARGETS = { + "cleandocs": { + "builder": "clean", + }, + "htmldocs": { + "builder": "html", + }, + "epubdocs": { + "builder": "epub", + "out_dir": "epub", + }, + "texinfodocs": { + "builder": "texinfo", + "out_dir": "texinfo", + }, + "infodocs": { + "builder": "texinfo", + "out_dir": "texinfo", + }, + "latexdocs": { + "builder": "latex", + "out_dir": "latex", + }, + "pdfdocs": { + "builder": "latex", + "out_dir": "latex", + }, + "xmldocs": { + "builder": "xml", + "out_dir": "xml", + }, + "linkcheckdocs": { + "builder": "linkcheck" + }, +} + +# Paper sizes. An empty value will pick the default +PAPER = ["", "a4", "letter"] + +class SphinxBuilder: + """ + Handles a sphinx-build target, adding needed arguments to build + with the Kernel. + """ + + def is_rust_enabled(self): + """Check if rust is enabled at .config""" + config_path = os.path.join(self.srctree, ".config") + if os.path.isfile(config_path): + with open(config_path, "r", encoding="utf-8") as f: + return "CONFIG_RUST=y" in f.read() + return False + + def get_path(self, path, abs_path=False): + """ + Ancillary routine to handle patches the right way, as shell does. + + It first expands "~" and "~user". Then, if patch is not absolute, + join self.srctree. Finally, if requested, convert to abspath. + """ + + path = os.path.expanduser(path) + if not path.startswith("/"): + path = os.path.join(self.srctree, path) + + if abs_path: + return os.path.abspath(path) + + return path + + def __init__(self, venv=None, verbose=False, n_jobs=None, interactive=None): + """Initialize internal variables""" + self.venv = venv + self.verbose = None + + # Normal variables passed from Kernel's makefile + self.kernelversion = os.environ.get("KERNELVERSION", "unknown") + self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown") + self.pdflatex = os.environ.get("PDFLATEX", "xelatex") + + if not interactive: + self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape") + else: + self.latexopts = os.environ.get("LATEXOPTS", "") + + if not verbose: + verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "") + + # Handle SPHINXOPTS evironment + sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) + + # As we handle number of jobs and quiet in separate, we need to pick + # it the same way as sphinx-build would pick, so let's use argparse + # do to the right argument expansion + parser = argparse.ArgumentParser() + parser.add_argument('-j', '--jobs', type=int) + parser.add_argument('-q', '--quiet', type=int) + + # Other sphinx-build arguments go as-is, so place them + # at self.sphinxopts + sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) + if sphinx_args.quiet == True: + self.verbose = False + + if sphinx_args.jobs: + self.n_jobs = sphinx_args.jobs + + # Command line arguments was passed, override SPHINXOPTS + if verbose is not None: + self.verbose = verbose + + self.n_jobs = n_jobs + + # Source tree directory. This needs to be at os.environ, as + # Sphinx extensions and media uAPI makefile needs it + self.srctree = os.environ.get("srctree") + if not self.srctree: + self.srctree = "." + os.environ["srctree"] = self.srctree + + # Now that we can expand srctree, get other directories as well + self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build") + self.kerneldoc = self.get_path(os.environ.get("KERNELDOC", + "scripts/kernel-doc.py")) + self.obj = os.environ.get("obj", "Documentation") + self.builddir = self.get_path(os.path.join(self.obj, "output"), + abs_path=True) + + # Media uAPI needs it + os.environ["BUILDDIR"] = self.builddir + + # Detect if rust is enabled + self.config_rust = self.is_rust_enabled() + + # Get directory locations for LaTeX build toolchain + self.pdflatex_cmd = shutil.which(self.pdflatex) + self.latexmk_cmd = shutil.which("latexmk") + + self.env = os.environ.copy() + + # If venv parameter is specified, run Sphinx from venv + if venv: + bin_dir = os.path.join(venv, "bin") + if os.path.isfile(os.path.join(bin_dir, "activate")): + # "activate" virtual env + self.env["PATH"] = bin_dir + ":" + self.env["PATH"] + self.env["VIRTUAL_ENV"] = venv + if "PYTHONHOME" in self.env: + del self.env["PYTHONHOME"] + print(f"Setting venv to {venv}") + else: + sys.exit(f"Venv {venv} not found.") + + def run_sphinx(self, sphinx_build, build_args, *args, **pwargs): + """ + Executes sphinx-build using current python3 command and setting + -j parameter if possible to run the build in parallel. + """ + + with JobserverExec() as jobserver: + if jobserver.claim: + n_jobs = str(jobserver.claim) + else: + n_jobs = "auto" # Supported since Sphinx 1.7 + + cmd = [] + + if self.venv: + cmd.append("python") + else: + cmd.append(sys.executable) + + cmd.append(sphinx_build) + + # if present, SPHINXOPTS or command line --jobs overrides default + if self.n_jobs: + n_jobs = str(self.n_jobs) + + if n_jobs: + cmd += [f"-j{n_jobs}"] + + if not self.verbose: + cmd.append("-q") + + cmd += self.sphinxopts + + cmd += build_args + + if self.verbose: + print(" ".join(cmd)) + + rc = subprocess.call(cmd, *args, **pwargs) + + def handle_html(self, css, output_dir): + """ + Extra steps for HTML and epub output. + + For such targets, we need to ensure that CSS will be properly + copied to the output _static directory + """ + + if not css: + return + + css = os.path.expanduser(css) + if not css.startswith("/"): + css = os.path.join(self.srctree, css) + + static_dir = os.path.join(output_dir, "_static") + os.makedirs(static_dir, exist_ok=True) + + try: + shutil.copy2(css, static_dir) + except (OSError, IOError) as e: + print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) + + def build_pdf_file(self, latex_cmd, from_dir, path): + """Builds a single pdf file using latex_cmd""" + try: + subprocess.run(latex_cmd + [path], + cwd=from_dir, check=True) + + return True + except subprocess.CalledProcessError: + # LaTeX PDF error code is almost useless: it returns + # error codes even when build succeeds but has warnings. + # So, we'll ignore the results + return False + + def pdf_parallel_build(self, tex_suffix, latex_cmd, tex_files, n_jobs): + """Build PDF files in parallel if possible""" + builds = {} + build_failed = False + max_len = 0 + has_tex = False + + # Process files in parallel + with futures.ThreadPoolExecutor(max_workers=n_jobs) as executor: + jobs = {} + + for from_dir, pdf_dir, entry in tex_files: + name = entry.name + + if not name.endswith(tex_suffix): + continue + + name = name[:-len(tex_suffix)] + + max_len = max(max_len, len(name)) + + has_tex = True + + future = executor.submit(self.build_pdf_file, latex_cmd, + from_dir, entry.path) + jobs[future] = (from_dir, name, entry.path) + + for future in futures.as_completed(jobs): + from_dir, name, path = jobs[future] + + pdf_name = name + ".pdf" + pdf_from = os.path.join(from_dir, pdf_name) + + try: + success = future.result() + + if success and os.path.exists(pdf_from): + pdf_to = os.path.join(pdf_dir, pdf_name) + + os.rename(pdf_from, pdf_to) + builds[name] = os.path.relpath(pdf_to, self.builddir) + else: + builds[name] = "FAILED" + build_failed = True + except Exception as e: + builds[name] = f"FAILED ({str(e)})" + build_failed = True + + # Handle case where no .tex files were found + if not has_tex: + name = "Sphinx LaTeX builder" + max_len = max(max_len, len(name)) + builds[name] = "FAILED (no .tex file was generated)" + build_failed = True + + return builds, build_failed, max_len + + def handle_pdf(self, output_dirs): + """ + Extra steps for PDF output. + + As PDF is handled via a LaTeX output, after building the .tex file, + a new build is needed to create the PDF output from the latex + directory. + """ + builds = {} + max_len = 0 + tex_suffix = ".tex" + + # Get all tex files that will be used for PDF build + tex_files = [] + for from_dir in output_dirs: + pdf_dir = os.path.join(from_dir, "../pdf") + os.makedirs(pdf_dir, exist_ok=True) + + if self.latexmk_cmd: + latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"] + else: + latex_cmd = [self.pdflatex] + + latex_cmd.extend(shlex.split(self.latexopts)) + + # Get a list of tex files to process + with os.scandir(from_dir) as it: + for entry in it: + if entry.name.endswith(tex_suffix): + tex_files.append((from_dir, pdf_dir, entry)) + + # When using make, this won't be used, as the number of jobs comes + # from POSIX jobserver. So, this covers the case where build comes + # from command line. On such case, serialize by default, except if + # the user explicitly sets the number of jobs. + n_jobs = 1 + + # n_jobs is either an integer or "auto". Only use it if it is a number + if self.n_jobs: + try: + n_jobs = int(self.n_jobs) + except ValueError: + pass + + # When using make, jobserver.claim is the number of jobs that were + # used with "-j" and that aren't used by other make targets + with JobserverExec() as jobserver: + n_jobs = 1 + + # Handle the case when a parameter is passed via command line, + # using it as default, if jobserver doesn't claim anything + if self.n_jobs: + try: + n_jobs = int(self.n_jobs) + except ValueError: + pass + + if jobserver.claim: + n_jobs = jobserver.claim + + # Build files in parallel + builds, build_failed, max_len = self.pdf_parallel_build(tex_suffix, + latex_cmd, + tex_files, + n_jobs) + + msg = "Summary" + msg += "\n" + "=" * len(msg) + print() + print(msg) + + for pdf_name, pdf_file in builds.items(): + print(f"{pdf_name:<{max_len}}: {pdf_file}") + + print() + + # return an error if a PDF file is missing + + if build_failed: + sys.exit(f"PDF build failed: not all PDF files were created.") + else: + print("All PDF files were built.") + + def handle_info(self, output_dirs): + """ + Extra steps for Info output. + + For texinfo generation, an additional make is needed from the + texinfo directory. + """ + + for output_dir in output_dirs: + try: + subprocess.run(["make", "info"], cwd=output_dir, check=True) + except subprocess.CalledProcessError as e: + sys.exit(f"Error generating info docs: {e}") + + def cleandocs(self, builder): + + shutil.rmtree(self.builddir, ignore_errors=True) + + def build(self, target, sphinxdirs=None, conf="conf.py", + theme=None, css=None, paper=None): + """ + Build documentation using Sphinx. This is the core function of this + module. It prepares all arguments required by sphinx-build. + """ + + builder = TARGETS[target]["builder"] + out_dir = TARGETS[target].get("out_dir", "") + + # Cleandocs doesn't require sphinx-build + if target == "cleandocs": + self.cleandocs(builder) + return + + # Other targets require sphinx-build + sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) + if not sphinxbuild: + sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") + + if builder == "latex": + if not self.pdflatex_cmd and not self.latexmk_cmd: + sys.exit("Error: pdflatex or latexmk required for PDF generation") + + docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation")) + + # Prepare base arguments for Sphinx build + kerneldoc = self.kerneldoc + if kerneldoc.startswith(self.srctree): + kerneldoc = os.path.relpath(kerneldoc, self.srctree) + + # Prepare common Sphinx options + args = [ + "-b", builder, + "-c", docs_dir, + ] + + if builder == "latex": + if not paper: + paper = PAPER[1] + + args.extend(["-D", f"latex_elements.papersize={paper}paper"]) + + if self.config_rust: + args.extend(["-t", "rustdoc"]) + + if conf: + self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True) + + if not sphinxdirs: + sphinxdirs = os.environ.get("SPHINXDIRS", ".") + + # The sphinx-build tool has a bug: internally, it tries to set + # locale with locale.setlocale(locale.LC_ALL, ''). This causes a + # crash if language is not set. Detect and fix it. + try: + locale.setlocale(locale.LC_ALL, '') + except Exception: + self.env["LC_ALL"] = "C" + self.env["LANG"] = "C" + + # sphinxdirs can be a list or a whitespace-separated string + sphinxdirs_list = [] + for sphinxdir in sphinxdirs: + if isinstance(sphinxdir, list): + sphinxdirs_list += sphinxdir + else: + for name in sphinxdir.split(" "): + sphinxdirs_list.append(name) + + # Build each directory + output_dirs = [] + for sphinxdir in sphinxdirs_list: + src_dir = os.path.join(docs_dir, sphinxdir) + doctree_dir = os.path.join(self.builddir, ".doctrees") + output_dir = os.path.join(self.builddir, sphinxdir, out_dir) + + # Make directory names canonical + src_dir = os.path.normpath(src_dir) + doctree_dir = os.path.normpath(doctree_dir) + output_dir = os.path.normpath(output_dir) + + os.makedirs(doctree_dir, exist_ok=True) + os.makedirs(output_dir, exist_ok=True) + + output_dirs.append(output_dir) + + build_args = args + [ + "-d", doctree_dir, + "-D", f"kerneldoc_bin={kerneldoc}", + "-D", f"version={self.kernelversion}", + "-D", f"release={self.kernelrelease}", + "-D", f"kerneldoc_srctree={self.srctree}", + src_dir, + output_dir, + ] + + # Execute sphinx-build + try: + self.run_sphinx(sphinxbuild, build_args, env=self.env) + except Exception as e: + sys.exit(f"Build failed: {e}") + + # Ensure that html/epub will have needed static files + if target in ["htmldocs", "epubdocs"]: + self.handle_html(css, output_dir) + + # PDF and Info require a second build step + if target == "pdfdocs": + self.handle_pdf(output_dirs) + elif target == "infodocs": + self.handle_info(output_dirs) + + @staticmethod + def get_python_version(cmd): + """ + Get python version from a Python binary. As we need to detect if + are out there newer python binaries, we can't rely on sys.release here. + """ + + result = subprocess.run([cmd, "--version"], check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + version = result.stdout.strip() + + match = re.search(r"(\d+\.\d+\.\d+)", version) + if match: + return parse_version(match.group(1)) + + print(f"Can't parse version {version}") + return (0, 0, 0) + + @staticmethod + def find_python(): + """ + Detect if are out there any python 3.xy version newer than the + current one. + + Note: this routine is limited to up to 2 digits for python3. We + may need to update it one day, hopefully on a distant future. + """ + patterns = [ + "python3.[0-9]", + "python3.[0-9][0-9]", + ] + + # Seek for a python binary newer than MIN_PYTHON_VERSION + for path in os.getenv("PATH", "").split(":"): + for pattern in patterns: + for cmd in glob(os.path.join(path, pattern)): + if os.path.isfile(cmd) and os.access(cmd, os.X_OK): + version = SphinxBuilder.get_python_version(cmd) + if version >= MIN_PYTHON_VERSION: + return cmd + + return None + + @staticmethod + def check_python(): + """ + Check if the current python binary satisfies our minimal requirement + for Sphinx build. If not, re-run with a newer version if found. + """ + cur_ver = sys.version_info[:3] + if cur_ver >= MIN_PYTHON_VERSION: + return + + python_ver = ver_str(cur_ver) + + new_python_cmd = SphinxBuilder.find_python() + if not new_python_cmd: + sys.exit(f"Python version {python_ver} is not supported anymore.") + + # Restart script using the newer version + script_path = os.path.abspath(sys.argv[0]) + args = [new_python_cmd, script_path] + sys.argv[1:] + + print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") + + try: + os.execv(new_python_cmd, args) + except OSError as e: + sys.exit(f"Failed to restart with {new_python_cmd}: {e}") + +def jobs_type(value): + """ + Handle valid values for -j. Accepts Sphinx "-jauto", plus a number + equal or bigger than one. + """ + if value is None: + return None + + if value.lower() == 'auto': + return value.lower() + + try: + if int(value) >= 1: + return value + + raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}") + except ValueError: + raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") + +def main(): + """ + Main function. The only mandatory argument is the target. If not + specified, the other arguments will use default values if not + specified at os.environ. + """ + parser = argparse.ArgumentParser(description="Kernel documentation builder") + + parser.add_argument("target", choices=list(TARGETS.keys()), + help="Documentation target to build") + parser.add_argument("--sphinxdirs", nargs="+", + help="Specific directories to build") + parser.add_argument("--conf", default="conf.py", + help="Sphinx configuration file") + + parser.add_argument("--theme", help="Sphinx theme to use") + + parser.add_argument("--css", help="Custom CSS file for HTML/EPUB") + + parser.add_argument("--paper", choices=PAPER, default=PAPER[0], + help="Paper size for LaTeX/PDF output") + + parser.add_argument("-v", "--verbose", action='store_true', + help="place build in verbose mode") + + parser.add_argument('-j', '--jobs', type=jobs_type, + help="Sets number of jobs to use with sphinx-build") + + parser.add_argument('-i', '--interactive', action='store_true', + help="Change latex default to run in interactive mode") + + parser.add_argument("-V", "--venv", nargs='?', const=f'{VENV_DEFAULT}', + default=None, + help=f'If used, run Sphinx from a venv dir (default dir: {VENV_DEFAULT})') + + args = parser.parse_args() + + SphinxBuilder.check_python() + + builder = SphinxBuilder(venv=args.venv, verbose=args.verbose, + n_jobs=args.jobs, interactive=args.interactive) + + builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf, + theme=args.theme, css=args.css, paper=args.paper) + +if __name__ == "__main__": + main() -- cgit From 8851e27d2cb947ea8bbbe8e812068f7bf5cbd00b Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Fri, 29 Aug 2025 21:55:25 +0200 Subject: rust: support Rust >= 1.91.0 target spec Starting with Rust 1.91.0 (expected 2025-10-30), the target spec format has changed the type of the `target-pointer-width` key from string to integer [1]. Thus conditionally use one or the other depending on the version. Cc: Waffle Maybe Link: https://github.com/rust-lang/rust/pull/144443 [1] Link: https://lore.kernel.org/r/20250829195525.721664-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/generate_rust_target.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs index 39c82908ff3a..38b3416bb979 100644 --- a/scripts/generate_rust_target.rs +++ b/scripts/generate_rust_target.rs @@ -225,7 +225,11 @@ fn main() { ts.push("features", features); ts.push("llvm-target", "x86_64-linux-gnu"); ts.push("supported-sanitizers", ["kcfi", "kernel-address"]); - ts.push("target-pointer-width", "64"); + if cfg.rustc_version_atleast(1, 91, 0) { + ts.push("target-pointer-width", 64); + } else { + ts.push("target-pointer-width", "64"); + } } else if cfg.has("X86_32") { // This only works on UML, as i386 otherwise needs regparm support in rustc if !cfg.has("UML") { @@ -245,7 +249,11 @@ fn main() { } ts.push("features", features); ts.push("llvm-target", "i386-unknown-linux-gnu"); - ts.push("target-pointer-width", "32"); + if cfg.rustc_version_atleast(1, 91, 0) { + ts.push("target-pointer-width", 32); + } else { + ts.push("target-pointer-width", "32"); + } } else if cfg.has("LOONGARCH") { panic!("loongarch uses the builtin rustc loongarch64-unknown-none-softfloat target"); } else { -- cgit From 3c847e17225aa9481fc3f6c948f5c718dea526f1 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Thu, 14 Aug 2025 11:30:28 +0200 Subject: rust: add `pin-init` as a dependency to `bindings` and `uapi` This allows `bindings` and `uapi` to implement `Zeroable` and use other items from pin-init. Co-developed-by: Miguel Ojeda Signed-off-by: Miguel Ojeda Link: https://rust-for-linux.zulipchat.com/#narrow/channel/291565-Help/topic/Zeroable.20trait.20for.20C.20structs/near/510264158 Signed-off-by: Benno Lossin Reviewed-by: Alice Ryhl Signed-off-by: Miguel Ojeda --- scripts/generate_rust_analyzer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index 7c3ea2b55041..fc27f0cca752 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -139,8 +139,8 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs, core_edit "exclude_dirs": [], } - append_crate_with_generated("bindings", ["core", "ffi"]) - append_crate_with_generated("uapi", ["core", "ffi"]) + append_crate_with_generated("bindings", ["core", "ffi", "pin_init"]) + append_crate_with_generated("uapi", ["core", "ffi", "pin_init"]) append_crate_with_generated("kernel", ["core", "macros", "build_error", "pin_init", "ffi", "bindings", "uapi"]) def is_root_crate(build_file, target): -- cgit From 009eb5da29a91016e3ebb988e6401e79411be7a1 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Thu, 21 Aug 2025 15:28:15 +0200 Subject: hrtimer: Remove hrtimer_clock_base:: Get_time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The get_time() callbacks always need to match the bases clockid. Instead of maintaining that association twice in hrtimer_bases, use a helper. Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/all/20250821-hrtimer-cleanup-get_time-v2-8-3ae822e5bfbd@linutronix.de --- scripts/gdb/linux/timerlist.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'scripts') diff --git a/scripts/gdb/linux/timerlist.py b/scripts/gdb/linux/timerlist.py index 98445671fe83..ccc24d30de80 100644 --- a/scripts/gdb/linux/timerlist.py +++ b/scripts/gdb/linux/timerlist.py @@ -56,8 +56,6 @@ def print_base(base): text += " .index: {}\n".format(base['index']) text += " .resolution: {} nsecs\n".format(constants.LX_hrtimer_resolution) - - text += " .get_time: {}\n".format(base['get_time']) if constants.LX_CONFIG_HIGH_RES_TIMERS: text += " .offset: {} nsecs\n".format(base['offset']) text += "active timers:\n" -- cgit From a6cf527e66b8ea7b3119a354a873ebfa57ea225e Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Sat, 9 Aug 2025 21:31:10 -0700 Subject: checkpatch: allow http links of any length in commit logs Dave Gilbert noticed that checkpatch warns about URL links over 75 chars in length in commit logs. Fix that. Link: https://lkml.kernel.org/r/3529faaf84a5a9a96c5c0ec4183ae0ba6e97673c.camel@perches.com Signed-off-by: Joe Perches Cc: Dave Gilbert Signed-off-by: Andrew Morton --- scripts/checkpatch.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index e722dd6fa8ef..319cc5f85885 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -3294,7 +3294,7 @@ sub process { # file delta changes $line =~ /^\s*(?:[\w\.\-\+]*\/)++[\w\.\-\+]+:/ || # filename then : - $line =~ /^\s*(?:Fixes:|$link_tags_search|$signature_tags)/i || + $line =~ /^\s*(?:Fixes:|https?:|$link_tags_search|$signature_tags)/i || # A Fixes:, link or signature tag line $commit_log_possible_stack_dump)) { WARN("COMMIT_LOG_LONG_LINE", -- cgit From fdd7c7e0d2ab3987882c570612d4622f437292c7 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Thu, 4 Sep 2025 21:41:28 -0700 Subject: rust: Introduce atomic API helpers In order to support LKMM atomics in Rust, add rust_helper_* for atomic APIs. These helpers ensure the implementation of LKMM atomics in Rust is the same as in C. This could save the maintenance burden of having two similar atomic implementations in asm. Originally-by: Mark Rutland Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Alice Ryhl Link: https://lore.kernel.org/all/20250719030827.61357-2-boqun.feng@gmail.com/ --- scripts/atomic/gen-atomics.sh | 1 + scripts/atomic/gen-rust-atomic-helpers.sh | 67 +++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100755 scripts/atomic/gen-rust-atomic-helpers.sh (limited to 'scripts') diff --git a/scripts/atomic/gen-atomics.sh b/scripts/atomic/gen-atomics.sh index 5b98a8307693..02508d0d6fe4 100755 --- a/scripts/atomic/gen-atomics.sh +++ b/scripts/atomic/gen-atomics.sh @@ -11,6 +11,7 @@ cat < ${LINUXDIR}/include/${header} diff --git a/scripts/atomic/gen-rust-atomic-helpers.sh b/scripts/atomic/gen-rust-atomic-helpers.sh new file mode 100755 index 000000000000..45b1e100ed7c --- /dev/null +++ b/scripts/atomic/gen-rust-atomic-helpers.sh @@ -0,0 +1,67 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +ATOMICDIR=$(dirname $0) + +. ${ATOMICDIR}/atomic-tbl.sh + +#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...) +gen_proto_order_variant() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + local atomic="$1"; shift + local int="$1"; shift + + local atomicname="${atomic}_${pfx}${name}${sfx}${order}" + + local ret="$(gen_ret_type "${meta}" "${int}")" + local params="$(gen_params "${int}" "${atomic}" "$@")" + local args="$(gen_args "$@")" + local retstmt="$(gen_ret_stmt "${meta}")" + +cat < + +// TODO: Remove this after INLINE_HELPERS support is added. +#ifndef __rust_helper +#define __rust_helper +#endif + +EOF + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic" "int" ${args} +done + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} +done + +cat < Date: Wed, 13 Aug 2025 11:39:51 -0400 Subject: rust: kunit: use `kernel::{fmt,prelude::fmt!}` Reduce coupling to implementation details of the formatting machinery by avoiding direct use for `core`'s formatting traits and macros. Acked-by: Greg Kroah-Hartman Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Signed-off-by: Tamir Duberstein Signed-off-by: Miguel Ojeda --- scripts/rustdoc_test_gen.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/rustdoc_test_gen.rs b/scripts/rustdoc_test_gen.rs index abb34ada2508..c8f9dc2ab976 100644 --- a/scripts/rustdoc_test_gen.rs +++ b/scripts/rustdoc_test_gen.rs @@ -202,7 +202,7 @@ pub extern "C" fn {kunit_name}(__kunit_test: *mut ::kernel::bindings::kunit) {{ // This follows the syntax for declaring test metadata in the proposed KTAP v2 spec, which may // be used for the proposed KUnit test attributes API. Thus hopefully this will make migration // easier later on. - ::kernel::kunit::info(format_args!(" # {kunit_name}.location: {real_path}:{line}\n")); + ::kernel::kunit::info(fmt!(" # {kunit_name}.location: {real_path}:{line}\n")); /// The anchor where the test code body starts. #[allow(unused)] -- cgit From 8b00d6fe96960aaba1b923d4a8c1ddb173c9c1ff Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 4 Sep 2025 13:33:56 -0600 Subject: docs: kdoc: trim __cacheline_group_* with the other annotations The special case for __cacheline_group_begin/end() can be handled by just adding another pattern to the struct_prefixes, eliminating the need for a special case in push_parameter(). One change is that these annotations no longer appear in the rendered output, just like all the other annotations that we clean out. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index a560546c1867..a90f77d6b669 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -90,6 +90,7 @@ struct_prefixes = [ (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), (KernRe(r'\s*____cacheline_aligned', re.S), ' '), + (KernRe(r'\s*__cacheline_group_(begin|end)\([^\)]+\);'), ''), # # Unwrap struct_group macros based on this definition: # __struct_group(TAG, NAME, ATTRS, MEMBERS...) @@ -447,12 +448,6 @@ class KernelDoc: self.entry.parameterdescs[param] = "anonymous\n" self.entry.anon_struct_union = True - # Handle cache group enforcing variables: they do not need - # to be described in header files - elif "__cacheline_group" in param: - # Ignore __cacheline_group_begin and __cacheline_group_end - return - # Warn if parameter has no description # (but ignore ones starting with # as these are not parameters # but inline preprocessor statements) -- cgit From e214cca38f1f35d42e63e990c610c96f993343c4 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 4 Sep 2025 14:01:20 -0600 Subject: docs: kdoc: tighten up the push_parameter() no-type case The handling of untyped parameters involved a number of redundant tests; restructure the code to remove them and be more compact. No output changes. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 44 +++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 24 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index a90f77d6b669..2118c20b3056 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -423,30 +423,26 @@ class KernelDoc: param = KernRe(r'[\[\)].*').sub('', param, count=1) - if dtype == "" and param.endswith("..."): - if KernRe(r'\w\.\.\.$').search(param): - # For named variable parameters of the form `x...`, - # remove the dots - param = param[:-3] - else: - # Handles unnamed variable parameters - param = "..." - - if param not in self.entry.parameterdescs or \ - not self.entry.parameterdescs[param]: - - self.entry.parameterdescs[param] = "variable arguments" - - elif dtype == "" and (not param or param == "void"): - param = "void" - self.entry.parameterdescs[param] = "no arguments" - - elif dtype == "" and param in ["struct", "union"]: - # Handle unnamed (anonymous) union or struct - dtype = param - param = "{unnamed_" + param + "}" - self.entry.parameterdescs[param] = "anonymous\n" - self.entry.anon_struct_union = True + # + # Look at various "anonymous type" cases. + # + if dtype == '': + if param.endswith("..."): + if len(param) > 3: # there is a name provided, use that + param = param[:-3] + if not self.entry.parameterdescs.get(param): + self.entry.parameterdescs[param] = "variable arguments" + + elif (not param) or param == "void": + param = "void" + self.entry.parameterdescs[param] = "no arguments" + + elif param in ["struct", "union"]: + # Handle unnamed (anonymous) union or struct + dtype = param + param = "{unnamed_" + param + "}" + self.entry.parameterdescs[param] = "anonymous\n" + self.entry.anon_struct_union = True # Warn if parameter has no description # (but ignore ones starting with # as these are not parameters -- cgit From f853e83006ab39c3dafe085a488c14bb46906601 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 4 Sep 2025 14:27:00 -0600 Subject: docs: kdoc: remove a single-use variable struct_attribute is only used once, so just put its value there directly and drop the name. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 2118c20b3056..b25c8d80b965 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -76,13 +76,11 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' # Here begins a long set of transformations to turn structure member prefixes # and macro invocations into something we can parse and generate kdoc for. # -struct_attribute = KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", - flags=re.I | re.S, cache=False) struct_args_pattern = r'([^,)]+)' struct_prefixes = [ # Strip attributes - (struct_attribute, ' '), + (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), -- cgit From 4c232a81b0831e7bfa7518968e431d5db29b2cac Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 4 Sep 2025 16:40:49 -0600 Subject: docs: kdoc: move the function transform patterns out of dump_function() Move these definitions to file level, where they are executed once, and don't clutter the function itself. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 78 ++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 43 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index b25c8d80b965..37811cddd55c 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -161,6 +161,37 @@ struct_nested_prefixes = [ (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), ] +# +# Transforms for function prototypes +# +function_xforms = [ + (r"^static +", "", 0), + (r"^extern +", "", 0), + (r"^asmlinkage +", "", 0), + (r"^inline +", "", 0), + (r"^__inline__ +", "", 0), + (r"^__inline +", "", 0), + (r"^__always_inline +", "", 0), + (r"^noinline +", "", 0), + (r"^__FORTIFY_INLINE +", "", 0), + (r"__init +", "", 0), + (r"__init_or_module +", "", 0), + (r"__deprecated +", "", 0), + (r"__flatten +", "", 0), + (r"__meminit +", "", 0), + (r"__must_check +", "", 0), + (r"__weak +", "", 0), + (r"__sched +", "", 0), + (r"_noprof", "", 0), + (r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +", "", 0), + (r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +", "", 0), + (r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +", "", 0), + (r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)", r"\1, \2", 0), + (r"__attribute_const__ +", "", 0), + (r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+", "", 0), +] + + # # A little helper to get rid of excess white space @@ -894,49 +925,10 @@ class KernelDoc: return_type = '' decl_type = 'function' - # Prefixes that would be removed - sub_prefixes = [ - (r"^static +", "", 0), - (r"^extern +", "", 0), - (r"^asmlinkage +", "", 0), - (r"^inline +", "", 0), - (r"^__inline__ +", "", 0), - (r"^__inline +", "", 0), - (r"^__always_inline +", "", 0), - (r"^noinline +", "", 0), - (r"^__FORTIFY_INLINE +", "", 0), - (r"__init +", "", 0), - (r"__init_or_module +", "", 0), - (r"__deprecated +", "", 0), - (r"__flatten +", "", 0), - (r"__meminit +", "", 0), - (r"__must_check +", "", 0), - (r"__weak +", "", 0), - (r"__sched +", "", 0), - (r"_noprof", "", 0), - (r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +", "", 0), - (r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +", "", 0), - (r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +", "", 0), - (r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)", r"\1, \2", 0), - (r"__attribute_const__ +", "", 0), - - # It seems that Python support for re.X is broken: - # At least for me (Python 3.13), this didn't work -# (r""" -# __attribute__\s*\(\( -# (?: -# [\w\s]+ # attribute name -# (?:\([^)]*\))? # attribute arguments -# \s*,? # optional comma at the end -# )+ -# \)\)\s+ -# """, "", re.X), - - # So, remove whitespaces and comments from it - (r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+", "", 0), - ] - - for search, sub, flags in sub_prefixes: + # + # Apply the initial transformations. + # + for search, sub, flags in function_xforms: prototype = KernRe(search, flags).sub(sub, prototype) # Macros are a special case, as they change the prototype format -- cgit From a2752f8c631201e189f501fc4d320354efa3e72e Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 4 Sep 2025 16:49:52 -0600 Subject: doc: kdoc: unify transform handling Both functions and structs are passed through a set of regex-based transforms, but the two were structured differently, despite being the same thing. Create a utility function to apply transformations and use it in both cases. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 65 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 31 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 37811cddd55c..1a1558211acd 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -78,7 +78,7 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' # struct_args_pattern = r'([^,)]+)' -struct_prefixes = [ +struct_xforms = [ # Strip attributes (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), @@ -165,33 +165,39 @@ struct_nested_prefixes = [ # Transforms for function prototypes # function_xforms = [ - (r"^static +", "", 0), - (r"^extern +", "", 0), - (r"^asmlinkage +", "", 0), - (r"^inline +", "", 0), - (r"^__inline__ +", "", 0), - (r"^__inline +", "", 0), - (r"^__always_inline +", "", 0), - (r"^noinline +", "", 0), - (r"^__FORTIFY_INLINE +", "", 0), - (r"__init +", "", 0), - (r"__init_or_module +", "", 0), - (r"__deprecated +", "", 0), - (r"__flatten +", "", 0), - (r"__meminit +", "", 0), - (r"__must_check +", "", 0), - (r"__weak +", "", 0), - (r"__sched +", "", 0), - (r"_noprof", "", 0), - (r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +", "", 0), - (r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +", "", 0), - (r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +", "", 0), - (r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)", r"\1, \2", 0), - (r"__attribute_const__ +", "", 0), - (r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+", "", 0), + (KernRe(r"^static +"), ""), + (KernRe(r"^extern +"), ""), + (KernRe(r"^asmlinkage +"), ""), + (KernRe(r"^inline +"), ""), + (KernRe(r"^__inline__ +"), ""), + (KernRe(r"^__inline +"), ""), + (KernRe(r"^__always_inline +"), ""), + (KernRe(r"^noinline +"), ""), + (KernRe(r"^__FORTIFY_INLINE +"), ""), + (KernRe(r"__init +"), ""), + (KernRe(r"__init_or_module +"), ""), + (KernRe(r"__deprecated +"), ""), + (KernRe(r"__flatten +"), ""), + (KernRe(r"__meminit +"), ""), + (KernRe(r"__must_check +"), ""), + (KernRe(r"__weak +"), ""), + (KernRe(r"__sched +"), ""), + (KernRe(r"_noprof"), ""), + (KernRe(r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +"), ""), + (KernRe(r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +"), ""), + (KernRe(r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +"), ""), + (KernRe(r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)"), r"\1, \2"), + (KernRe(r"__attribute_const__ +"), ""), + (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""), ] - +# +# Apply a set of transforms to a block of text. +# +def apply_transforms(xforms, text): + for search, subst in xforms: + text = search.sub(subst, text) + return text # # A little helper to get rid of excess white space @@ -807,8 +813,7 @@ class KernelDoc: # Go through the list of members applying all of our transformations. # members = trim_private_members(members) - for search, sub in struct_prefixes: - members = search.sub(sub, members) + members = apply_transforms(struct_xforms, members) nested = NestedMatch() for search, sub in struct_nested_prefixes: @@ -924,12 +929,10 @@ class KernelDoc: func_macro = False return_type = '' decl_type = 'function' - # # Apply the initial transformations. # - for search, sub, flags in function_xforms: - prototype = KernRe(search, flags).sub(sub, prototype) + prototype = apply_transforms(function_xforms, prototype) # Macros are a special case, as they change the prototype format new_proto = KernRe(r"^#\s*define\s+").sub("", prototype) -- cgit From fee63c8f10c2fe77f618f9955c2f5521ff9cc622 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Fri, 5 Sep 2025 15:53:05 -0600 Subject: docs: kdoc: remove a couple of spurious regex characters The "name" regex in dump_function() includes both the tilde and colon characters, but neither has any place in function prototypes. Remove the characters, after which the regex simplifies to "\w+" No output changes. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 1a1558211acd..decd127df82e 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -958,7 +958,7 @@ class KernelDoc: # - atomic_set (macro) # - pci_match_device, __copy_to_user (long return type) - name = r'[a-zA-Z0-9_~:]+' + name = r'\w+' prototype_end1 = r'[^\(]*' prototype_end2 = r'[^\{]*' prototype_end = fr'\(({prototype_end1}|{prototype_end2})\)' -- cgit From 08b5228cf455d46a23bfb341766563d1a48e3c8f Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Fri, 5 Sep 2025 16:30:48 -0600 Subject: docs: kdoc: remove a useless empty capture group The is_define_proto case in dump_function() uses a regex with an empty capture group - () - that has no use; just take it out. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index decd127df82e..f9be5414244d 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -972,11 +972,11 @@ class KernelDoc: found = False if is_define_proto: - r = KernRe(r'^()(' + name + r')\s+') + r = KernRe(r'^(' + name + r')\s+') if r.search(prototype): return_type = '' - declaration_name = r.group(2) + declaration_name = r.group(1) func_macro = True found = True -- cgit From ff1f2af341b72bd5b6b5d432da55faf2f6d24cfe Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 13:22:42 -0600 Subject: docs: kdoc: Simplify the dump_function() prototype regexes The regexes for the parsing of function prototypes were more complicated than they needed to be and difficult to understand -- at least, I spent a fair amount of time bashing my head against them. Simplify them, and add some documentation comments as well. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index f9be5414244d..ec2e6e83df05 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -959,15 +959,15 @@ class KernelDoc: # - pci_match_device, __copy_to_user (long return type) name = r'\w+' - prototype_end1 = r'[^\(]*' - prototype_end2 = r'[^\{]*' - prototype_end = fr'\(({prototype_end1}|{prototype_end2})\)' - - # Besides compiling, Perl qr{[\w\s]+} works as a non-capturing group. - # So, this needs to be mapped in Python with (?:...)? or (?:...)+ - type1 = r'(?:[\w\s]+)?' type2 = r'(?:[\w\s]+\*+)+' + # + # Attempt to match first on (args) with no internal parentheses; this + # lets us easily filter out __acquires() and other post-args stuff. If + # that fails, just grab the rest of the line to the last closing + # parenthesis. + # + proto_args = r'\(([^\(]*|.*)\)' found = False @@ -983,9 +983,9 @@ class KernelDoc: if not found: patterns = [ - rf'^()({name})\s*{prototype_end}', - rf'^({type1})\s+({name})\s*{prototype_end}', - rf'^({type2})\s*({name})\s*{prototype_end}', + rf'^()({name})\s*{proto_args}', + rf'^({type1})\s+({name})\s*{proto_args}', + rf'^({type2})\s*({name})\s*{proto_args}', ] for p in patterns: -- cgit From 370f430527ecd35938ad94167e45fc784f6e4d95 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 16:01:15 -0600 Subject: docs: kdoc: consolidate some of the macro-processing logic The logic to handle macros is split in dump_function(); bring it all together into a single place and add a comment saying what's going on. Remove the unneeded is_define_proto variable, and tighten up the code a bit. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 43 +++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 23 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index ec2e6e83df05..27329ce9b5e9 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -926,21 +926,31 @@ class KernelDoc: Stores a function of function macro inside self.entries array. """ - func_macro = False + found = func_macro = False return_type = '' decl_type = 'function' # # Apply the initial transformations. # prototype = apply_transforms(function_xforms, prototype) - - # Macros are a special case, as they change the prototype format + # + # If we have a macro, remove the "#define" at the front. + # new_proto = KernRe(r"^#\s*define\s+").sub("", prototype) if new_proto != prototype: - is_define_proto = True prototype = new_proto - else: - is_define_proto = False + # + # Dispense with the simple "#define A B" case here; the key + # is the space after the name of the symbol being defined. + # NOTE that the seemingly misnamed "func_macro" indicates a + # macro *without* arguments. + # + r = KernRe(r'^(\w+)\s+') + if r.search(prototype): + return_type = '' + declaration_name = r.group(1) + func_macro = True + found = True # Yes, this truly is vile. We are looking for: # 1. Return type (may be nothing if we're looking at a macro) @@ -968,19 +978,10 @@ class KernelDoc: # parenthesis. # proto_args = r'\(([^\(]*|.*)\)' - - found = False - - if is_define_proto: - r = KernRe(r'^(' + name + r')\s+') - - if r.search(prototype): - return_type = '' - declaration_name = r.group(1) - func_macro = True - - found = True - + # + # (Except for the simple macro case) attempt to split up the prototype + # in the various ways we understand. + # if not found: patterns = [ rf'^()({name})\s*{proto_args}', @@ -990,16 +991,12 @@ class KernelDoc: for p in patterns: r = KernRe(p) - if r.match(prototype): - return_type = r.group(1) declaration_name = r.group(2) args = r.group(3) - self.create_parameter_list(ln, decl_type, args, ',', declaration_name) - found = True break if not found: -- cgit From 3dff54410e56ddee2dad8824ea77e60cd5b16d5b Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 16:21:28 -0600 Subject: docs: kdoc: final dump_function() cleanups Add some more comments to dump_function(), add some comments, and trim out an unneeded duplicate output_declaration() call. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 27329ce9b5e9..5e41acfef7b8 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -999,32 +999,28 @@ class KernelDoc: declaration_name) found = True break + # + # Parsing done; make sure that things are as we expect. + # if not found: self.emit_msg(ln, f"cannot understand function prototype: '{prototype}'") return - if self.entry.identifier != declaration_name: - self.emit_msg(ln, - f"expecting prototype for {self.entry.identifier}(). Prototype was for {declaration_name}() instead") + self.emit_msg(ln, f"expecting prototype for {self.entry.identifier}(). " + f"Prototype was for {declaration_name}() instead") return - self.check_sections(ln, declaration_name, "function") - self.check_return_section(ln, declaration_name, return_type) + # + # Store the result. + # + self.output_declaration(decl_type, declaration_name, + typedef=('typedef' in return_type), + functiontype=return_type, + purpose=self.entry.declaration_purpose, + func_macro=func_macro) - if 'typedef' in return_type: - self.output_declaration(decl_type, declaration_name, - typedef=True, - functiontype=return_type, - purpose=self.entry.declaration_purpose, - func_macro=func_macro) - else: - self.output_declaration(decl_type, declaration_name, - typedef=False, - functiontype=return_type, - purpose=self.entry.declaration_purpose, - func_macro=func_macro) def dump_typedef(self, ln, proto): """ -- cgit From 999a642d7e7d4241cc7dba942a13c67d0685284b Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 16:32:10 -0600 Subject: docs: kdoc: remove some dead code in dump_typedef() The regex in this block of code makes no sense, and a quick test shows that it never matches anything; simply delete the code. No output changes. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 5e41acfef7b8..7c739b495d58 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -1060,11 +1060,6 @@ class KernelDoc: purpose=self.entry.declaration_purpose) return - # Handle nested parentheses or brackets - r = KernRe(r'(\(*.\)\s*|\[*.\]\s*);$') - while r.search(proto): - proto = r.sub('', proto) - # Parse simple typedefs r = KernRe(r'typedef.*\s+(\w+)\s*;') if r.match(proto): -- cgit From 00fa9bc4e93cb336575a5f2c1da90d2443ff14c8 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 16:49:58 -0600 Subject: docs: kdoc: remove redundant comment stripping in dump_typedef() By the time we get here, comments have long since been stripped out; there is no need to do it again. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 7c739b495d58..ad9df0536bbf 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -1034,9 +1034,6 @@ class KernelDoc: typedef1 = KernRe(r'typedef' + typedef_type + r'\(' + typedef_ident + r'\)' + typedef_args) typedef2 = KernRe(r'typedef' + typedef_type + typedef_ident + typedef_args) - # Strip comments - proto = KernRe(r'/\*.*?\*/', flags=re.S).sub('', proto) - # Parse function typedef prototypes for r in [typedef1, typedef2]: if not r.match(proto): -- cgit From c01878437739f86851da76235f394346c6bd8ce3 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Sep 2025 17:12:14 -0600 Subject: docs: kdoc: a few more dump_typedef() tweaks Merge "typedef" into the typedef_type pattern rather than repeating it later, and add some comments. Signed-off-by: Jonathan Corbet --- scripts/lib/kdoc/kdoc_parser.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index ad9df0536bbf..2376f180b1fa 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -1026,13 +1026,15 @@ class KernelDoc: """ Stores a typedef inside self.entries array. """ - - typedef_type = r'((?:\s+[\w*]+\b){0,7}\s+(?:\w+\b|\*+))\s*' + # + # We start by looking for function typedefs. + # + typedef_type = r'typedef((?:\s+[\w*]+\b){0,7}\s+(?:\w+\b|\*+))\s*' typedef_ident = r'\*?\s*(\w\S+)\s*' typedef_args = r'\s*\((.*)\);' - typedef1 = KernRe(r'typedef' + typedef_type + r'\(' + typedef_ident + r'\)' + typedef_args) - typedef2 = KernRe(r'typedef' + typedef_type + typedef_ident + typedef_args) + typedef1 = KernRe(typedef_type + r'\(' + typedef_ident + r'\)' + typedef_args) + typedef2 = KernRe(typedef_type + typedef_ident + typedef_args) # Parse function typedef prototypes for r in [typedef1, typedef2]: @@ -1048,16 +1050,16 @@ class KernelDoc: f"expecting prototype for typedef {self.entry.identifier}. Prototype was for typedef {declaration_name} instead\n") return - decl_type = 'function' - self.create_parameter_list(ln, decl_type, args, ',', declaration_name) + self.create_parameter_list(ln, 'function', args, ',', declaration_name) - self.output_declaration(decl_type, declaration_name, + self.output_declaration('function', declaration_name, typedef=True, functiontype=return_type, purpose=self.entry.declaration_purpose) return - - # Parse simple typedefs + # + # Not a function, try to parse a simple typedef. + # r = KernRe(r'typedef.*\s+(\w+)\s*;') if r.match(proto): declaration_name = r.group(1) -- cgit From d322f6a24ee5964a58294f61bf96a1b6404c676d Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Mon, 8 Sep 2025 17:41:57 +0200 Subject: scripts/decode_stacktrace.sh: symbol: avoid trailing whitespaces A few patches slightly improving the output generated by decode_stacktrace.sh. This patch (of 3): Lines having a symbol to decode might not always have info after this symbol. It means ${info_str} might not be set, but it will always be printed after a space, causing trailing whitespaces. That's a detail, but when the output is opened with an editor marking these trailing whitespaces, that's a bit disturbing. It is easy to remove them by printing this variable with a space only if it is set. While at it, do the same with ${module} and print everything in one line. Link: https://lkml.kernel.org/r/20250908-decode_strace_indent-v1-0-28e5e4758080@kernel.org Link: https://lkml.kernel.org/r/20250908-decode_strace_indent-v1-1-28e5e4758080@kernel.org Signed-off-by: Matthieu Baerts (NGI0) Reviewed-by: Carlos Llamas Reviewed-by: Breno Leitao Reviewed-by: Luca Ceresoli Cc: Carlos Llamas Cc: Elliot Berman Cc: Stephen Boyd Signed-off-by: Andrew Morton --- scripts/decode_stacktrace.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/decode_stacktrace.sh b/scripts/decode_stacktrace.sh index 17abc4e7a985..c6b5c14412f0 100755 --- a/scripts/decode_stacktrace.sh +++ b/scripts/decode_stacktrace.sh @@ -323,12 +323,7 @@ handle_line() { parse_symbol # modifies $symbol # Add up the line number to the symbol - if [[ -z ${module} ]] - then - echo "${words[@]}" "$symbol ${info_str}" - else - echo "${words[@]}" "$symbol $module ${info_str}" - fi + echo "${words[@]}" "${symbol}${module:+ ${module}}${info_str:+ ${info_str}}" } while read line; do -- cgit From 4a2fc4897b5e0ca1e7a3cb4e32f44c7db3367dee Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Mon, 8 Sep 2025 17:41:58 +0200 Subject: scripts/decode_stacktrace.sh: symbol: preserve alignment With lines having a symbol to decode, the script was only trying to preserve the alignment for the timestamps, but not the rest, nor when the caller was set (CONFIG_PRINTK_CALLER=y). With this sample ... [ 52.080924] Call Trace: [ 52.080926] [ 52.080931] dump_stack_lvl+0x6f/0xb0 ... the script was producing the following output: [ 52.080924] Call Trace: [ 52.080926] [ 52.080931] dump_stack_lvl (arch/x86/include/asm/irqflags.h:19) (dump_stack_lvl is no longer aligned with : one missing space) With this other sample ... [ 52.080924][ T48] Call Trace: [ 52.080926][ T48] [ 52.080931][ T48] dump_stack_lvl+0x6f/0xb0 ... the script was producing the following output: [ 52.080924][ T48] Call Trace: [ 52.080926][ T48] [ 52.080931][ T48] dump_stack_lvl (arch/x86/include/asm/irqflags.h:19) (the misalignment is clearer here) That's because the script had a workaround for CONFIG_PRINTK_TIME=y only, see the previous comment called "Format timestamps with tabs". To always preserve spaces, they need to be recorded along the words. That is what is now done with the new 'spaces' array. Some notes: - 'extglob' is needed only for this operation, and that's why it is set in a dedicated subshell. - 'read' is used with '-r' not to treat a character in any special way, e.g. when followed by a space. - When a word is removed from the 'words' array, the corresponding space needs to be removed from the 'spaces' array as well. With the last sample, we now have: [ 52.080924][ T48] Call Trace: [ 52.080926][ T48] [ 52.080931][ T48] dump_stack_lvl (arch/x86/include/asm/irqflags.h:19) (the alignment is preserved) Link: https://lkml.kernel.org/r/20250908-decode_strace_indent-v1-2-28e5e4758080@kernel.org Signed-off-by: Matthieu Baerts (NGI0) Tested-by: Carlos Llamas Cc: Breno Leitao Cc: Elliot Berman Cc: Luca Ceresoli Cc: Stephen Boyd Signed-off-by: Andrew Morton --- scripts/decode_stacktrace.sh | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) (limited to 'scripts') diff --git a/scripts/decode_stacktrace.sh b/scripts/decode_stacktrace.sh index c6b5c14412f0..0c92d6a7f777 100755 --- a/scripts/decode_stacktrace.sh +++ b/scripts/decode_stacktrace.sh @@ -255,10 +255,11 @@ handle_line() { basepath=${basepath%/init/main.c:*)} fi - local words + local words spaces - # Tokenize - read -a words <<<"$1" + # Tokenize: words and spaces to preserve the alignment + read -ra words <<<"$1" + IFS='#' read -ra spaces <<<"$(shopt -s extglob; echo "${1//+([^[:space:]])/#}")" # Remove hex numbers. Do it ourselves until it happens in the # kernel @@ -270,19 +271,13 @@ handle_line() { for i in "${!words[@]}"; do # Remove the address if [[ ${words[$i]} =~ \[\<([^]]+)\>\] ]]; then - unset words[$i] - fi - - # Format timestamps with tabs - if [[ ${words[$i]} == \[ && ${words[$i+1]} == *\] ]]; then - unset words[$i] - words[$i+1]=$(printf "[%13s\n" "${words[$i+1]}") + unset words[$i] spaces[$i] fi done if [[ ${words[$last]} =~ ^[0-9a-f]+\] ]]; then words[$last-1]="${words[$last-1]} ${words[$last]}" - unset words[$last] + unset words[$last] spaces[$last] last=$(( $last - 1 )) fi @@ -294,7 +289,7 @@ handle_line() { local info_str="" if [[ ${words[$last]} =~ \([A-Z]*\) ]]; then info_str=${words[$last]} - unset words[$last] + unset words[$last] spaces[$last] last=$(( $last - 1 )) fi @@ -311,7 +306,7 @@ handle_line() { modbuildid= fi symbol=${words[$last-1]} - unset words[$last-1] + unset words[$last-1] spaces[$last-1] else # The symbol is the last element, process it symbol=${words[$last]} @@ -323,7 +318,10 @@ handle_line() { parse_symbol # modifies $symbol # Add up the line number to the symbol - echo "${words[@]}" "${symbol}${module:+ ${module}}${info_str:+ ${info_str}}" + for i in "${!words[@]}"; do + echo -n "${spaces[i]}${words[i]}" + done + echo "${spaces[$last]}${symbol}${module:+ ${module}}${info_str:+ ${info_str}}" } while read line; do -- cgit From e1831e8dd1c839088692c09c97031ce467db1a2a Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Mon, 8 Sep 2025 17:41:59 +0200 Subject: scripts/decode_stacktrace.sh: code: preserve alignment With lines having a code to decode, the alignment was not preserved for the first line. With this sample ... [ 52.238089][ T55] RIP: 0010:__ip_queue_xmit+0x127c/0x1820 [ 52.238401][ T55] Code: c1 83 e0 07 48 c1 e9 03 83 c0 03 (...) ... the script was producing the following output: [ 52.238089][ T55] RIP: 0010:__ip_queue_xmit (...) [ 52.238401][ T55] Code: c1 83 e0 07 48 c1 e9 03 83 c0 03 (...) That's because scripts/decodecode doesn't preserve the alignment. No need to modify it, it is enough to give only the "Code: (...)" part to this script, and print the prefix without modifications. With the same sample, we now have: [ 52.238089][ T55] RIP: 0010:__ip_queue_xmit (...) [ 52.238401][ T55] Code: c1 83 e0 07 48 c1 e9 03 83 c0 03 (...) Link: https://lkml.kernel.org/r/20250908-decode_strace_indent-v1-3-28e5e4758080@kernel.org Signed-off-by: Matthieu Baerts (NGI0) Tested-by: Carlos Llamas Cc: Breno Leitao Cc: Elliot Berman Cc: Luca Ceresoli Cc: Stephen Boyd Signed-off-by: Andrew Morton --- scripts/decode_stacktrace.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/decode_stacktrace.sh b/scripts/decode_stacktrace.sh index 0c92d6a7f777..c73cb802a0a3 100755 --- a/scripts/decode_stacktrace.sh +++ b/scripts/decode_stacktrace.sh @@ -242,8 +242,10 @@ debuginfod_get_vmlinux() { decode_code() { local scripts=`dirname "${BASH_SOURCE[0]}"` + local lim="Code: " - echo "$1" | $scripts/decodecode + echo -n "${1%%${lim}*}" + echo "${lim}${1##*${lim}}" | $scripts/decodecode } handle_line() { -- cgit From 347b564599fb01d8ae1e9f473b82820fea4c2767 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 11 Sep 2025 21:33:55 +0200 Subject: coccinelle: of_table: handle SPI device ID tables 'struct spi_device_id' tables also need to be NULL terminated. Link: https://lkml.kernel.org/r/20250911193354.56262-2-krzysztof.kozlowski@linaro.org Signed-off-by: Krzysztof Kozlowski Cc: Julia Lawall Cc: Nathan Chancellor Cc: Nicolas Palix Signed-off-by: Andrew Morton --- scripts/coccinelle/misc/of_table.cocci | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'scripts') diff --git a/scripts/coccinelle/misc/of_table.cocci b/scripts/coccinelle/misc/of_table.cocci index 4693ea744753..17881cb0884b 100644 --- a/scripts/coccinelle/misc/of_table.cocci +++ b/scripts/coccinelle/misc/of_table.cocci @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/// Make sure (of/i2c/platform)_device_id tables are NULL terminated +/// Make sure (of/i2c/platform/spi)_device_id tables are NULL terminated // // Keywords: of_table i2c_table platform_table // Confidence: Medium @@ -15,14 +15,14 @@ identifier var, arr; expression E; @@ ( -struct \(of_device_id \| i2c_device_id \| platform_device_id\) arr[] = { +struct \(of_device_id \| i2c_device_id \| platform_device_id \| spi_device_id\) arr[] = { ..., { .var = E, * } }; | -struct \(of_device_id \| i2c_device_id \| platform_device_id\) arr[] = { +struct \(of_device_id \| i2c_device_id \| platform_device_id \| spi_device_id\) arr[] = { ..., * { ..., E, ... }, }; @@ -33,7 +33,7 @@ identifier var, arr; expression E; @@ ( -struct \(of_device_id \| i2c_device_id \| platform_device_id\) arr[] = { +struct \(of_device_id \| i2c_device_id \| platform_device_id \| spi_device_id\) arr[] = { ..., { .var = E, @@ -42,7 +42,7 @@ struct \(of_device_id \| i2c_device_id \| platform_device_id\) arr[] = { + { } }; | -struct \(of_device_id \| i2c_device_id \| platform_device_id\) arr[] = { +struct \(of_device_id \| i2c_device_id \| platform_device_id \| spi_device_id\) arr[] = { ..., { ..., E, ... }, + { }, @@ -55,7 +55,7 @@ identifier var, arr; expression E; @@ ( -struct \(of_device_id \| i2c_device_id \| platform_device_id\) arr[] = { +struct \(of_device_id \| i2c_device_id \| platform_device_id \| spi_device_id\) arr[] = { ..., { .var = E, @@ -63,7 +63,7 @@ struct \(of_device_id \| i2c_device_id \| platform_device_id\) arr[] = { @p1 }; | -struct \(of_device_id \| i2c_device_id \| platform_device_id\) arr[] = { +struct \(of_device_id \| i2c_device_id \| platform_device_id \| spi_device_id\) arr[] = { ..., { ..., E, ... } @p1 -- cgit From f23e76a32dbfc45418a1448f94886eb608b35b98 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 11 Sep 2025 20:47:27 +0200 Subject: coccinelle: platform_no_drv_owner: handle also built-in drivers builtin_platform_driver() and others also use macro platform_driver_register() which sets the .owner=THIS_MODULE, so extend the cocci script to detect these as well. Link: https://lkml.kernel.org/r/20250911184726.23154-2-krzysztof.kozlowski@linaro.org Signed-off-by: Krzysztof Kozlowski Cc: Julia Lawall Cc: Nathan Chancellor Cc: Nicolas Palix Signed-off-by: Andrew Morton --- scripts/coccinelle/api/platform_no_drv_owner.cocci | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'scripts') diff --git a/scripts/coccinelle/api/platform_no_drv_owner.cocci b/scripts/coccinelle/api/platform_no_drv_owner.cocci index 8fa050eeb7e5..5e869858bda8 100644 --- a/scripts/coccinelle/api/platform_no_drv_owner.cocci +++ b/scripts/coccinelle/api/platform_no_drv_owner.cocci @@ -10,12 +10,21 @@ virtual org virtual report @match1@ +declarer name builtin_i2c_driver; +declarer name builtin_platform_driver; +declarer name builtin_platform_driver_probe; declarer name module_i2c_driver; declarer name module_platform_driver; declarer name module_platform_driver_probe; identifier __driver; @@ ( + builtin_i2c_driver(__driver); +| + builtin_platform_driver(__driver); +| + builtin_platform_driver_probe(__driver, ...); +| module_i2c_driver(__driver); | module_platform_driver(__driver); -- cgit From a40282dd3c484e6c882e93f4680e0a3ef3814453 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Sat, 20 Sep 2025 16:45:23 -0700 Subject: gcc-plugins: Remove TODO_verify_il for GCC >= 16 GCC now runs TODO_verify_il automatically[1], so it is no longer exposed to plugins. Only use the flag on GCC < 16. Link: https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=9739ae9384dd7cd3bb1c7683d6b80b7a9116eaf8 [1] Suggested-by: Christopher Fore Link: https://lore.kernel.org/r/20250920234519.work.915-kees@kernel.org Signed-off-by: Kees Cook --- scripts/gcc-plugins/gcc-common.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'scripts') diff --git a/scripts/gcc-plugins/gcc-common.h b/scripts/gcc-plugins/gcc-common.h index 6cb6d1051815..8f1b3500f8e2 100644 --- a/scripts/gcc-plugins/gcc-common.h +++ b/scripts/gcc-plugins/gcc-common.h @@ -173,10 +173,17 @@ static inline opt_pass *get_pass_for_id(int id) return g->get_passes()->get_pass_for_id(id); } +#if BUILDING_GCC_VERSION < 16000 #define TODO_verify_ssa TODO_verify_il #define TODO_verify_flow TODO_verify_il #define TODO_verify_stmts TODO_verify_il #define TODO_verify_rtl_sharing TODO_verify_il +#else +#define TODO_verify_ssa 0 +#define TODO_verify_flow 0 +#define TODO_verify_stmts 0 +#define TODO_verify_rtl_sharing 0 +#endif #define INSN_DELETED_P(insn) (insn)->deleted() -- cgit From 0ce5139fd96e9d415d3faaef1c575e238f9bbd67 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 18 Sep 2025 10:05:46 +0200 Subject: kbuild: always create intermediate vmlinux.unstripped Generate the intermediate vmlinux.unstripped regardless of CONFIG_ARCH_VMLINUX_NEEDS_RELOCS. If CONFIG_ARCH_VMLINUX_NEEDS_RELOCS is unset, vmlinux.unstripped and vmlinux are identiacal. This simplifies the build rule, and allows to strip more sections by adding them to remove-section-y. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier Link: https://patch.msgid.link/a48ca543fa2305bd17324f41606dcaed9b19f2d4.1758182101.git.legion@kernel.org Signed-off-by: Nathan Chancellor --- scripts/Makefile.vmlinux | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index b64862dc6f08..4f2d4c3fb737 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -9,20 +9,6 @@ include $(srctree)/scripts/Makefile.lib targets := -ifdef CONFIG_ARCH_VMLINUX_NEEDS_RELOCS -vmlinux-final := vmlinux.unstripped - -quiet_cmd_strip_relocs = RSTRIP $@ - cmd_strip_relocs = $(OBJCOPY) --remove-section='.rel*' --remove-section=!'.rel*.dyn' $< $@ - -vmlinux: $(vmlinux-final) FORCE - $(call if_changed,strip_relocs) - -targets += vmlinux -else -vmlinux-final := vmlinux -endif - %.o: %.c FORCE $(call if_changed_rule,cc_o_c) @@ -61,19 +47,19 @@ targets += .builtin-dtbs-list ifdef CONFIG_GENERIC_BUILTIN_DTB targets += .builtin-dtbs.S .builtin-dtbs.o -$(vmlinux-final): .builtin-dtbs.o +vmlinux.unstripped: .builtin-dtbs.o endif -# vmlinux +# vmlinux.unstripped # --------------------------------------------------------------------------- ifdef CONFIG_MODULES targets += .vmlinux.export.o -$(vmlinux-final): .vmlinux.export.o +vmlinux.unstripped: .vmlinux.export.o endif ifdef CONFIG_ARCH_WANTS_PRE_LINK_VMLINUX -$(vmlinux-final): arch/$(SRCARCH)/tools/vmlinux.arch.o +vmlinux.unstripped: arch/$(SRCARCH)/tools/vmlinux.arch.o arch/$(SRCARCH)/tools/vmlinux.arch.o: vmlinux.o FORCE $(Q)$(MAKE) $(build)=arch/$(SRCARCH)/tools $@ @@ -86,17 +72,30 @@ cmd_link_vmlinux = \ $< "$(LD)" "$(KBUILD_LDFLAGS)" "$(LDFLAGS_vmlinux)" "$@"; \ $(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) $@, true) -targets += $(vmlinux-final) -$(vmlinux-final): scripts/link-vmlinux.sh vmlinux.o $(KBUILD_LDS) FORCE +targets += vmlinux.unstripped +vmlinux.unstripped: scripts/link-vmlinux.sh vmlinux.o $(KBUILD_LDS) FORCE +$(call if_changed_dep,link_vmlinux) ifdef CONFIG_DEBUG_INFO_BTF -$(vmlinux-final): $(RESOLVE_BTFIDS) +vmlinux.unstripped: $(RESOLVE_BTFIDS) endif ifdef CONFIG_BUILDTIME_TABLE_SORT -$(vmlinux-final): scripts/sorttable +vmlinux.unstripped: scripts/sorttable endif +# vmlinux +# --------------------------------------------------------------------------- + +remove-section-y := +remove-section-$(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS) += '.rel*' + +quiet_cmd_strip_relocs = OBJCOPY $@ + cmd_strip_relocs = $(OBJCOPY) $(addprefix --remove-section=,$(remove-section-y)) $< $@ + +targets += vmlinux +vmlinux: vmlinux.unstripped FORCE + $(call if_changed,strip_relocs) + # modules.builtin.ranges # --------------------------------------------------------------------------- ifdef CONFIG_BUILTIN_MODULE_RANGES @@ -110,7 +109,7 @@ modules.builtin.ranges: $(srctree)/scripts/generate_builtin_ranges.awk \ modules.builtin vmlinux.map vmlinux.o.map FORCE $(call if_changed,modules_builtin_ranges) -vmlinux.map: $(vmlinux-final) +vmlinux.map: vmlinux.unstripped @: endif -- cgit From 3e86e4d74c0490e5fc5a7f8de8f29e7579c9ffe5 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 18 Sep 2025 10:05:47 +0200 Subject: kbuild: keep .modinfo section in vmlinux.unstripped Keep the .modinfo section during linking, but strip it from the final vmlinux. Adjust scripts/mksysmap to exclude modinfo symbols from kallsyms. This change will allow the next commit to extract the .modinfo section from the vmlinux.unstripped intermediate. Signed-off-by: Masahiro Yamada Signed-off-by: Alexey Gladkov Reviewed-by: Nicolas Schier Link: https://patch.msgid.link/aaf67c07447215463300fccaa758904bac42f992.1758182101.git.legion@kernel.org Signed-off-by: Nathan Chancellor --- scripts/Makefile.vmlinux | 7 +++++-- scripts/mksysmap | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index 4f2d4c3fb737..70856dab0f54 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -86,11 +86,14 @@ endif # vmlinux # --------------------------------------------------------------------------- -remove-section-y := +remove-section-y := .modinfo remove-section-$(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS) += '.rel*' +# To avoid warnings: "empty loadable segment detected at ..." from GNU objcopy, +# it is necessary to remove the PT_LOAD flag from the segment. quiet_cmd_strip_relocs = OBJCOPY $@ - cmd_strip_relocs = $(OBJCOPY) $(addprefix --remove-section=,$(remove-section-y)) $< $@ + cmd_strip_relocs = $(OBJCOPY) $(patsubst %,--set-section-flags %=noload,$(remove-section-y)) $< $@; \ + $(OBJCOPY) $(addprefix --remove-section=,$(remove-section-y)) $@ targets += vmlinux vmlinux: vmlinux.unstripped FORCE diff --git a/scripts/mksysmap b/scripts/mksysmap index 3accbdb269ac..a607a0059d11 100755 --- a/scripts/mksysmap +++ b/scripts/mksysmap @@ -79,6 +79,9 @@ / _SDA_BASE_$/d / _SDA2_BASE_$/d +# MODULE_INFO() +/ __UNIQUE_ID_modinfo[0-9]*$/d + # --------------------------------------------------------------------------- # Ignored patterns # (symbols that contain the pattern are ignored) -- cgit From 39cfd5b12160be4f57df1c3ba60139741c827616 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 18 Sep 2025 10:05:48 +0200 Subject: kbuild: extract modules.builtin.modinfo from vmlinux.unstripped Currently, we assume all the data for modules.builtin.modinfo are available in vmlinux.o. This makes it impossible for modpost, which is invoked after vmlinux.o, to add additional module info. This commit moves the modules.builtin.modinfo rule after modpost. Signed-off-by: Masahiro Yamada Signed-off-by: Alexey Gladkov Reviewed-by: Nicolas Schier Link: https://patch.msgid.link/cdb3e5b9a739666b755cd0097dc34ab69c350e51.1758182101.git.legion@kernel.org Signed-off-by: Nathan Chancellor --- scripts/Makefile.vmlinux | 26 ++++++++++++++++++++++++++ scripts/Makefile.vmlinux_o | 26 +------------------------- 2 files changed, 27 insertions(+), 25 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index 70856dab0f54..ce7946171497 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -99,6 +99,32 @@ targets += vmlinux vmlinux: vmlinux.unstripped FORCE $(call if_changed,strip_relocs) +# modules.builtin.modinfo +# --------------------------------------------------------------------------- + +OBJCOPYFLAGS_modules.builtin.modinfo := -j .modinfo -O binary + +targets += modules.builtin.modinfo +modules.builtin.modinfo: vmlinux.unstripped FORCE + $(call if_changed,objcopy) + +# modules.builtin +# --------------------------------------------------------------------------- + +__default: modules.builtin + +# The second line aids cases where multiple modules share the same object. + +quiet_cmd_modules_builtin = GEN $@ + cmd_modules_builtin = \ + tr '\0' '\n' < $< | \ + sed -n 's/^[[:alnum:]:_]*\.file=//p' | \ + tr ' ' '\n' | uniq | sed -e 's:^:kernel/:' -e 's/$$/.ko/' > $@ + +targets += modules.builtin +modules.builtin: modules.builtin.modinfo FORCE + $(call if_changed,modules_builtin) + # modules.builtin.ranges # --------------------------------------------------------------------------- ifdef CONFIG_BUILTIN_MODULE_RANGES diff --git a/scripts/Makefile.vmlinux_o b/scripts/Makefile.vmlinux_o index b024ffb3e201..23c8751285d7 100644 --- a/scripts/Makefile.vmlinux_o +++ b/scripts/Makefile.vmlinux_o @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only PHONY := __default -__default: vmlinux.o modules.builtin.modinfo modules.builtin +__default: vmlinux.o include include/config/auto.conf include $(srctree)/scripts/Kbuild.include @@ -73,30 +73,6 @@ vmlinux.o: $(initcalls-lds) vmlinux.a $(KBUILD_VMLINUX_LIBS) FORCE targets += vmlinux.o -# modules.builtin.modinfo -# --------------------------------------------------------------------------- - -OBJCOPYFLAGS_modules.builtin.modinfo := -j .modinfo -O binary - -targets += modules.builtin.modinfo -modules.builtin.modinfo: vmlinux.o FORCE - $(call if_changed,objcopy) - -# modules.builtin -# --------------------------------------------------------------------------- - -# The second line aids cases where multiple modules share the same object. - -quiet_cmd_modules_builtin = GEN $@ - cmd_modules_builtin = \ - tr '\0' '\n' < $< | \ - sed -n 's/^[[:alnum:]:_]*\.file=//p' | \ - tr ' ' '\n' | uniq | sed -e 's:^:kernel/:' -e 's/$$/.ko/' > $@ - -targets += modules.builtin -modules.builtin: modules.builtin.modinfo FORCE - $(call if_changed,modules_builtin) - # Add FORCE to the prerequisites of a target to force it to be always rebuilt. # --------------------------------------------------------------------------- -- cgit From 83fb49389bbe07defb85b063f7ff0fd016f06b35 Mon Sep 17 00:00:00 2001 From: Alexey Gladkov Date: Thu, 18 Sep 2025 10:05:50 +0200 Subject: modpost: Add modname to mod_device_table alias At this point, if a symbol is compiled as part of the kernel, information about which module the symbol belongs to is lost. To save this it is possible to add the module name to the alias name. It's not very pretty, but it's possible for now. Cc: Miguel Ojeda Cc: Andreas Hindborg Cc: Danilo Krummrich Cc: Alex Gaynor Cc: rust-for-linux@vger.kernel.org Signed-off-by: Alexey Gladkov Acked-by: Danilo Krummrich Acked-by: Nicolas Schier Link: https://patch.msgid.link/1a0d0bd87a4981d465b9ed21e14f4e78eaa03ded.1758182101.git.legion@kernel.org Signed-off-by: Nathan Chancellor --- scripts/mod/file2alias.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index 00586119a25b..1260bc2287fb 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -1476,7 +1476,7 @@ void handle_moddevtable(struct module *mod, struct elf_info *info, { void *symval; char *zeros = NULL; - const char *type, *name; + const char *type, *name, *modname; size_t typelen; static const char *prefix = "__mod_device_table__"; @@ -1488,10 +1488,19 @@ void handle_moddevtable(struct module *mod, struct elf_info *info, if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT) return; - /* All our symbols are of form __mod_device_table____. */ + /* All our symbols are of form __mod_device_table__kmod_____. */ if (!strstarts(symname, prefix)) return; - type = symname + strlen(prefix); + + modname = strstr(symname, "__kmod_"); + if (!modname) + return; + modname += strlen("__kmod_"); + + type = strstr(modname, "__"); + if (!type) + return; + type += strlen("__"); name = strstr(type, "__"); if (!name) -- cgit From 5ab23c7923a1d2ae1890026866a2d8506b010a4a Mon Sep 17 00:00:00 2001 From: Alexey Gladkov Date: Thu, 18 Sep 2025 10:05:51 +0200 Subject: modpost: Create modalias for builtin modules For some modules, modalias is generated using the modpost utility and the section is added to the module file. When a module is added inside vmlinux, modpost does not generate modalias for such modules and the information is lost. As a result kmod (which uses modules.builtin.modinfo in userspace) cannot determine that modalias is handled by a builtin kernel module. $ cat /sys/devices/pci0000:00/0000:00:14.0/modalias pci:v00008086d0000A36Dsv00001043sd00008694bc0Csc03i30 $ modinfo xhci_pci name: xhci_pci filename: (builtin) license: GPL file: drivers/usb/host/xhci-pci description: xHCI PCI Host Controller Driver Missing modalias "pci:v*d*sv*sd*bc0Csc03i30*" which will be generated by modpost if the module is built separately. To fix this it is necessary to generate the same modalias for vmlinux as for the individual modules. Fortunately '.vmlinux.export.o' is already generated from which '.modinfo' can be extracted in the same way as for vmlinux.o. Signed-off-by: Masahiro Yamada Signed-off-by: Alexey Gladkov Tested-by: Stephen Rothwell Reviewed-by: Nicolas Schier Link: https://patch.msgid.link/28d4da3b0e3fc8474142746bcf469e03752c3208.1758182101.git.legion@kernel.org Signed-off-by: Nathan Chancellor --- scripts/Makefile.vmlinux | 4 +++- scripts/mksysmap | 3 +++ scripts/mod/file2alias.c | 19 ++++++++++++++++++- scripts/mod/modpost.c | 15 +++++++++++++++ scripts/mod/modpost.h | 2 ++ 5 files changed, 41 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index ce7946171497..1e5e37aadcd0 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -89,11 +89,13 @@ endif remove-section-y := .modinfo remove-section-$(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS) += '.rel*' +remove-symbols := -w --strip-symbol='__mod_device_table__*' + # To avoid warnings: "empty loadable segment detected at ..." from GNU objcopy, # it is necessary to remove the PT_LOAD flag from the segment. quiet_cmd_strip_relocs = OBJCOPY $@ cmd_strip_relocs = $(OBJCOPY) $(patsubst %,--set-section-flags %=noload,$(remove-section-y)) $< $@; \ - $(OBJCOPY) $(addprefix --remove-section=,$(remove-section-y)) $@ + $(OBJCOPY) $(addprefix --remove-section=,$(remove-section-y)) $(remove-symbols) $@ targets += vmlinux vmlinux: vmlinux.unstripped FORCE diff --git a/scripts/mksysmap b/scripts/mksysmap index a607a0059d11..c4531eacde20 100755 --- a/scripts/mksysmap +++ b/scripts/mksysmap @@ -59,6 +59,9 @@ # EXPORT_SYMBOL (namespace) / __kstrtabns_/d +# MODULE_DEVICE_TABLE (symbol name) +/ __mod_device_table__/d + # --------------------------------------------------------------------------- # Ignored suffixes # (do not forget '$' after each pattern) diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index 1260bc2287fb..7da9735e7ab3 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -1477,7 +1477,7 @@ void handle_moddevtable(struct module *mod, struct elf_info *info, void *symval; char *zeros = NULL; const char *type, *name, *modname; - size_t typelen; + size_t typelen, modnamelen; static const char *prefix = "__mod_device_table__"; /* We're looking for a section relative symbol */ @@ -1500,6 +1500,7 @@ void handle_moddevtable(struct module *mod, struct elf_info *info, type = strstr(modname, "__"); if (!type) return; + modnamelen = type - modname; type += strlen("__"); name = strstr(type, "__"); @@ -1526,5 +1527,21 @@ void handle_moddevtable(struct module *mod, struct elf_info *info, } } + if (mod->is_vmlinux) { + struct module_alias *alias; + + /* + * If this is vmlinux, record the name of the builtin module. + * Traverse the linked list in the reverse order, and set the + * builtin_modname unless it has already been set in the + * previous call. + */ + list_for_each_entry_reverse(alias, &mod->aliases, node) { + if (alias->builtin_modname) + break; + alias->builtin_modname = xstrndup(modname, modnamelen); + } + } + free(zeros); } diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 5ca7c268294e..47c8aa2a6939 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -2067,11 +2067,26 @@ static void write_if_changed(struct buffer *b, const char *fname) static void write_vmlinux_export_c_file(struct module *mod) { struct buffer buf = { }; + struct module_alias *alias, *next; buf_printf(&buf, "#include \n"); add_exported_symbols(&buf, mod); + + buf_printf(&buf, + "#include \n" + "#undef __MODULE_INFO_PREFIX\n" + "#define __MODULE_INFO_PREFIX\n"); + + list_for_each_entry_safe(alias, next, &mod->aliases, node) { + buf_printf(&buf, "MODULE_INFO(%s.alias, \"%s\");\n", + alias->builtin_modname, alias->str); + list_del(&alias->node); + free(alias->builtin_modname); + free(alias); + } + write_if_changed(&buf, ".vmlinux.export.c"); free(buf.p); } diff --git a/scripts/mod/modpost.h b/scripts/mod/modpost.h index 9133e4c3803f..2aecb8f25c87 100644 --- a/scripts/mod/modpost.h +++ b/scripts/mod/modpost.h @@ -99,10 +99,12 @@ buf_write(struct buffer *buf, const char *s, int len); * struct module_alias - auto-generated MODULE_ALIAS() * * @node: linked to module::aliases + * @modname: name of the builtin module (only for vmlinux) * @str: a string for MODULE_ALIAS() */ struct module_alias { struct list_head node; + char *builtin_modname; char str[]; }; -- cgit From 3328d39a8dca2d6ed27197a0025df7540b99adf2 Mon Sep 17 00:00:00 2001 From: Alexey Gladkov Date: Thu, 18 Sep 2025 10:05:52 +0200 Subject: kbuild: vmlinux.unstripped should always depend on .vmlinux.export.o Since .vmlinux.export.c is used to add generated by modpost modaliases for builtin modules the .vmlinux.export.o is no longer optional and should always be created. The generation of this file is not dependent on CONFIG_MODULES. Signed-off-by: Alexey Gladkov Reviewed-by: Nicolas Schier Link: https://patch.msgid.link/0e63a9c7741fe8217e4fd7c60afcf057ffa2ef5a.1758182101.git.legion@kernel.org Signed-off-by: Nathan Chancellor --- scripts/Makefile.vmlinux | 9 ++------- scripts/link-vmlinux.sh | 5 +---- 2 files changed, 3 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index 1e5e37aadcd0..7c6ae9886f8f 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -53,11 +53,6 @@ endif # vmlinux.unstripped # --------------------------------------------------------------------------- -ifdef CONFIG_MODULES -targets += .vmlinux.export.o -vmlinux.unstripped: .vmlinux.export.o -endif - ifdef CONFIG_ARCH_WANTS_PRE_LINK_VMLINUX vmlinux.unstripped: arch/$(SRCARCH)/tools/vmlinux.arch.o @@ -72,8 +67,8 @@ cmd_link_vmlinux = \ $< "$(LD)" "$(KBUILD_LDFLAGS)" "$(LDFLAGS_vmlinux)" "$@"; \ $(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) $@, true) -targets += vmlinux.unstripped -vmlinux.unstripped: scripts/link-vmlinux.sh vmlinux.o $(KBUILD_LDS) FORCE +targets += vmlinux.unstripped .vmlinux.export.o +vmlinux.unstripped: scripts/link-vmlinux.sh vmlinux.o .vmlinux.export.o $(KBUILD_LDS) FORCE +$(call if_changed_dep,link_vmlinux) ifdef CONFIG_DEBUG_INFO_BTF vmlinux.unstripped: $(RESOLVE_BTFIDS) diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh index 51367c2bfc21..433849ff7529 100755 --- a/scripts/link-vmlinux.sh +++ b/scripts/link-vmlinux.sh @@ -73,10 +73,7 @@ vmlinux_link() objs="${objs} .builtin-dtbs.o" fi - if is_enabled CONFIG_MODULES; then - objs="${objs} .vmlinux.export.o" - fi - + objs="${objs} .vmlinux.export.o" objs="${objs} init/version-timestamp.o" if [ "${SRCARCH}" = "um" ]; then -- cgit From 64f4ea200eca05a248533b8374d7b5f0756a3990 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 23 Sep 2025 14:34:17 -0700 Subject: kconfig: Fix BrokenPipeError warnings in selftests The kconfig test harness ("make testconfig") was generating BrokenPipeError warnings when running interactive tests like oldaskconfig and oldconfig: /usr/lib/python3/dist-packages/_pytest/unraisableexception.py:85: PytestUnraisableExceptionWarning: Exception ignored in: <_io.BufferedWriter name=12> Traceback (most recent call last): File "/srv/code/scripts/kconfig/tests/conftest.py", line 127, in oldaskconfig return self._run_conf('--oldaskconfig', dot_config=dot_config, ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ interactive=True, in_keys=in_keys) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ BrokenPipeError: [Errno 32] Broken pipe The issue occurred when the test framework attempted to write to stdin after the conf subprocess had already exited. Wrap stdin write operations in try/except to catch BrokenPipeError and stop sending more input. Add explicit flush() after writes so we can see delivery errors immediately. Ignore BrokenPipeError when closing stdin. Explicitly call wait() to validate subprocess termination. Reviewed-by: Nathan Chancellor Tested-by: Nathan Chancellor Link: https://lore.kernel.org/r/20250923213422.1105654-1-kees@kernel.org Signed-off-by: Kees Cook --- scripts/kconfig/tests/conftest.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/kconfig/tests/conftest.py b/scripts/kconfig/tests/conftest.py index 2a2a7e2da060..d94b79e012c0 100644 --- a/scripts/kconfig/tests/conftest.py +++ b/scripts/kconfig/tests/conftest.py @@ -81,7 +81,22 @@ class Conf: # For interactive modes such as oldaskconfig, oldconfig, # send 'Enter' key until the program finishes. if interactive: - ps.stdin.write(b'\n') + try: + ps.stdin.write(b'\n') + ps.stdin.flush() + except (BrokenPipeError, OSError): + # Process has exited, stop sending input + break + + # Close stdin gracefully + try: + ps.stdin.close() + except (BrokenPipeError, OSError): + # Ignore broken pipe on close + pass + + # Wait for process to complete + ps.wait() self.retcode = ps.returncode self.stdout = ps.stdout.read().decode() -- cgit From f9afce4f32e9a120fc902fa6c9e0b90ad799a6ec Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 23 Sep 2025 14:34:18 -0700 Subject: kconfig: Add transitional symbol attribute for migration support During kernel option migrations (e.g. CONFIG_CFI_CLANG to CONFIG_CFI), existing .config files need to maintain backward compatibility while preventing deprecated options from appearing in newly generated configurations. This is challenging with existing Kconfig mechanisms because: 1. Simply removing old options breaks existing .config files. 2. Manually listing an option as "deprecated" leaves it needlessly visible and still writes them to new .config files. 3. Using any method to remove visibility (.e.g no 'prompt', 'if n', etc) prevents the option from being processed at all. Add a "transitional" attribute that creates symbols which are: - Processed during configuration (can influence other symbols' defaults) - Hidden from user menus (no prompts appear) - Omitted from newly written .config files (gets migrated) - Restricted to only having help sections (no defaults, selects, etc) making it truly just a "prior value pass-through" option. The transitional syntax requires a type argument and prevents type redefinition: config NEW_OPTION bool "New option" default OLD_OPTION config OLD_OPTION bool transitional help Transitional config for OLD_OPTION migration. This allows seamless migration: olddefconfig processes existing CONFIG_OLD_OPTION=y settings to enable CONFIG_NEW_OPTION=y, while CONFIG_OLD_OPTION is omitted from newly generated .config files. Added positive and negative testing via "testconfig" make target. Co-developed-by: Vegard Nossum Signed-off-by: Vegard Nossum Reviewed-by: Nathan Chancellor Tested-by: Nathan Chancellor Link: https://lore.kernel.org/r/20250923213422.1105654-2-kees@kernel.org Signed-off-by: Kees Cook --- scripts/kconfig/expr.h | 1 + scripts/kconfig/lexer.l | 1 + scripts/kconfig/parser.y | 47 ++++++++++ scripts/kconfig/symbol.c | 7 +- scripts/kconfig/tests/err_transitional/Kconfig | 52 +++++++++++ scripts/kconfig/tests/err_transitional/__init__.py | 14 +++ .../kconfig/tests/err_transitional/expected_stderr | 7 ++ scripts/kconfig/tests/transitional/Kconfig | 100 +++++++++++++++++++++ scripts/kconfig/tests/transitional/__init__.py | 18 ++++ scripts/kconfig/tests/transitional/expected_config | 12 +++ scripts/kconfig/tests/transitional/initial_config | 16 ++++ 11 files changed, 274 insertions(+), 1 deletion(-) create mode 100644 scripts/kconfig/tests/err_transitional/Kconfig create mode 100644 scripts/kconfig/tests/err_transitional/__init__.py create mode 100644 scripts/kconfig/tests/err_transitional/expected_stderr create mode 100644 scripts/kconfig/tests/transitional/Kconfig create mode 100644 scripts/kconfig/tests/transitional/__init__.py create mode 100644 scripts/kconfig/tests/transitional/expected_config create mode 100644 scripts/kconfig/tests/transitional/initial_config (limited to 'scripts') diff --git a/scripts/kconfig/expr.h b/scripts/kconfig/expr.h index fe2231e0e6a4..5f900d18dae0 100644 --- a/scripts/kconfig/expr.h +++ b/scripts/kconfig/expr.h @@ -145,6 +145,7 @@ struct symbol { #define SYMBOL_CONST 0x0001 /* symbol is const */ #define SYMBOL_CHECK 0x0008 /* used during dependency checking */ #define SYMBOL_VALID 0x0080 /* set when symbol.curr is calculated */ +#define SYMBOL_TRANS 0x0100 /* symbol is transitional only (not visible)*/ #define SYMBOL_WRITE 0x0200 /* write symbol to file (KCONFIG_CONFIG) */ #define SYMBOL_WRITTEN 0x0800 /* track info to avoid double-write to .config */ #define SYMBOL_CHECKED 0x2000 /* used during dependency checking */ diff --git a/scripts/kconfig/lexer.l b/scripts/kconfig/lexer.l index 9c2cdfc33c6f..6d2c92c6095d 100644 --- a/scripts/kconfig/lexer.l +++ b/scripts/kconfig/lexer.l @@ -126,6 +126,7 @@ n [A-Za-z0-9_-] "select" return T_SELECT; "source" return T_SOURCE; "string" return T_STRING; +"transitional" return T_TRANSITIONAL; "tristate" return T_TRISTATE; "visible" return T_VISIBLE; "||" return T_OR; diff --git a/scripts/kconfig/parser.y b/scripts/kconfig/parser.y index e9c3c664e925..49b79dde1725 100644 --- a/scripts/kconfig/parser.y +++ b/scripts/kconfig/parser.y @@ -75,6 +75,7 @@ struct menu *current_menu, *current_entry, *current_choice; %token T_SELECT %token T_SOURCE %token T_STRING +%token T_TRANSITIONAL %token T_TRISTATE %token T_VISIBLE %token T_EOL @@ -205,6 +206,12 @@ config_option: T_PROMPT T_WORD_QUOTE if_expr T_EOL printd(DEBUG_PARSE, "%s:%d:prompt\n", cur_filename, cur_lineno); }; +config_option: T_TRANSITIONAL T_EOL +{ + current_entry->sym->flags |= SYMBOL_TRANS; + printd(DEBUG_PARSE, "%s:%d:transitional\n", cur_filename, cur_lineno); +}; + config_option: default expr if_expr T_EOL { menu_add_expr(P_DEFAULT, $2, $3); @@ -482,6 +489,43 @@ assign_val: %% +/** + * transitional_check_sanity - check transitional symbols have no other + * properties + * + * @menu: menu of the potentially transitional symbol + * + * Return: -1 if an error is found, 0 otherwise. + */ +static int transitional_check_sanity(const struct menu *menu) +{ + struct property *prop; + + if (!menu->sym || !(menu->sym->flags & SYMBOL_TRANS)) + return 0; + + /* Check for depends and visible conditions. */ + if ((menu->dep && !expr_is_yes(menu->dep)) || + (menu->visibility && !expr_is_yes(menu->visibility))) { + fprintf(stderr, "%s:%d: error: %s", + menu->filename, menu->lineno, + "transitional symbols can only have help sections\n"); + return -1; + } + + /* Check for any property other than "help". */ + for (prop = menu->sym->prop; prop; prop = prop->next) { + if (prop->type != P_COMMENT) { + fprintf(stderr, "%s:%d: error: %s", + prop->filename, prop->lineno, + "transitional symbols can only have help sections\n"); + return -1; + } + } + + return 0; +} + /** * choice_check_sanity - check sanity of a choice member * @@ -558,6 +602,9 @@ void conf_parse(const char *name) if (menu->sym && sym_check_deps(menu->sym)) yynerrs++; + if (transitional_check_sanity(menu)) + yynerrs++; + if (menu->sym && sym_is_choice(menu->sym)) { menu_for_each_sub_entry(child, menu) if (child->sym && choice_check_sanity(child)) diff --git a/scripts/kconfig/symbol.c b/scripts/kconfig/symbol.c index 26ab10c0fd76..760cac998381 100644 --- a/scripts/kconfig/symbol.c +++ b/scripts/kconfig/symbol.c @@ -214,6 +214,11 @@ static void sym_calc_visibility(struct symbol *sym) struct property *prop; tristate tri; + if (sym->flags & SYMBOL_TRANS) { + sym->visible = yes; + return; + } + /* any prompt visible? */ tri = no; for_all_prompts(sym, prop) { @@ -526,7 +531,7 @@ void sym_calc_value(struct symbol *sym) } } - if (sym_is_choice(sym)) + if (sym_is_choice(sym) || sym->flags & SYMBOL_TRANS) sym->flags &= ~SYMBOL_WRITE; } diff --git a/scripts/kconfig/tests/err_transitional/Kconfig b/scripts/kconfig/tests/err_transitional/Kconfig new file mode 100644 index 000000000000..a75ed3b2fe5e --- /dev/null +++ b/scripts/kconfig/tests/err_transitional/Kconfig @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-2.0 +# Test that transitional symbols cannot have properties other than help + +config BAD_DEFAULT + bool + transitional + default y + help + This transitional symbol illegally has a default property. + +config BAD_PROMPT + bool + transitional + prompt "Bad prompt" + help + This transitional symbol illegally has a prompt. + +config BAD_SELECT + bool + transitional + select OTHER_SYMBOL + help + This transitional symbol illegally has a select. + +config BAD_IMPLY + bool + transitional + imply OTHER_SYMBOL + help + This transitional symbol illegally has an imply. + +config BAD_DEPENDS + bool + transitional + depends on OTHER_SYMBOL + help + This transitional symbol illegally has a depends. + +config BAD_RANGE + int + transitional + range 1 10 + help + This transitional symbol illegally has a range. + +config BAD_NO_TYPE + transitional + help + This transitional symbol illegally has no type specified. + +config OTHER_SYMBOL + bool diff --git a/scripts/kconfig/tests/err_transitional/__init__.py b/scripts/kconfig/tests/err_transitional/__init__.py new file mode 100644 index 000000000000..7dffb5b0833f --- /dev/null +++ b/scripts/kconfig/tests/err_transitional/__init__.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 +""" +Test that transitional symbols with invalid properties are rejected. + +Transitional symbols can only have help sections. Any other properties +(default, select, depends, etc.) should cause a parser error. +""" + +def test(conf): + # This should fail with exit code 1 due to invalid transitional symbol + assert conf.olddefconfig() == 1 + + # Check that the error message is about transitional symbols + assert conf.stderr_contains('expected_stderr') diff --git a/scripts/kconfig/tests/err_transitional/expected_stderr b/scripts/kconfig/tests/err_transitional/expected_stderr new file mode 100644 index 000000000000..b52db4f680f4 --- /dev/null +++ b/scripts/kconfig/tests/err_transitional/expected_stderr @@ -0,0 +1,7 @@ +Kconfig:46:warning: config symbol defined without type +Kconfig:7: error: transitional symbols can only have help sections +Kconfig:14: error: transitional symbols can only have help sections +Kconfig:21: error: transitional symbols can only have help sections +Kconfig:28: error: transitional symbols can only have help sections +Kconfig:32: error: transitional symbols can only have help sections +Kconfig:42: error: transitional symbols can only have help sections diff --git a/scripts/kconfig/tests/transitional/Kconfig b/scripts/kconfig/tests/transitional/Kconfig new file mode 100644 index 000000000000..62c3b24665b9 --- /dev/null +++ b/scripts/kconfig/tests/transitional/Kconfig @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: GPL-2.0 +# Test transitional symbols for config migration with all Kconfig types + +# Enable module support for tristate testing +config MODULES + bool "Enable loadable module support" + modules + default y + +# Basic migration tests for all types +config NEW_BOOL + bool "New bool option" + default OLD_BOOL + +config OLD_BOOL + bool + transitional + +config NEW_TRISTATE + tristate "New tristate option" + default OLD_TRISTATE + +config OLD_TRISTATE + tristate + transitional + +config NEW_STRING + string "New string option" + default OLD_STRING + +config OLD_STRING + string + transitional + +config NEW_HEX + hex "New hex option" + default OLD_HEX + +config OLD_HEX + hex + transitional + +config NEW_INT + int "New int option" + default OLD_INT + +config OLD_INT + int + transitional + +# Precedence tests for all types +config NEW_BOOL_PRECEDENCE + bool "New bool option with precedence" + default OLD_BOOL_PRECEDENCE + +config OLD_BOOL_PRECEDENCE + bool + transitional + +config NEW_STRING_PRECEDENCE + string "New string option with precedence" + default OLD_STRING_PRECEDENCE + +config OLD_STRING_PRECEDENCE + string + transitional + +config NEW_TRISTATE_PRECEDENCE + tristate "New tristate option with precedence" + default OLD_TRISTATE_PRECEDENCE + +config OLD_TRISTATE_PRECEDENCE + tristate + transitional + +config NEW_HEX_PRECEDENCE + hex "New hex option with precedence" + default OLD_HEX_PRECEDENCE + +config OLD_HEX_PRECEDENCE + hex + transitional + +config NEW_INT_PRECEDENCE + int "New int option with precedence" + default OLD_INT_PRECEDENCE + +config OLD_INT_PRECEDENCE + int + transitional + +# Test that help sections are allowed for transitional symbols +config OLD_WITH_HELP + bool + transitional + help + This transitional symbol has a help section to validate that help is allowed. + +config REGULAR_OPTION + bool "Regular option" diff --git a/scripts/kconfig/tests/transitional/__init__.py b/scripts/kconfig/tests/transitional/__init__.py new file mode 100644 index 000000000000..61937d10edf1 --- /dev/null +++ b/scripts/kconfig/tests/transitional/__init__.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0 +""" +Test transitional symbol migration functionality for all Kconfig types. + +This tests that: +- OLD_* options in existing .config cause NEW_* options to be set +- OLD_* options are not written to the new .config file +- NEW_* options appear in the new .config file with correct values +- All Kconfig types work correctly: bool, tristate, string, hex, int +- User-set NEW values take precedence over conflicting OLD transitional values +""" + +def test(conf): + # Run olddefconfig to process the migration with the initial config + assert conf.olddefconfig(dot_config='initial_config') == 0 + + # Check that the configuration matches expected output + assert conf.config_contains('expected_config') diff --git a/scripts/kconfig/tests/transitional/expected_config b/scripts/kconfig/tests/transitional/expected_config new file mode 100644 index 000000000000..846e9ddcab91 --- /dev/null +++ b/scripts/kconfig/tests/transitional/expected_config @@ -0,0 +1,12 @@ +CONFIG_MODULES=y +CONFIG_NEW_BOOL=y +CONFIG_NEW_TRISTATE=m +CONFIG_NEW_STRING="test string" +CONFIG_NEW_HEX=0x1234 +CONFIG_NEW_INT=42 +# CONFIG_NEW_BOOL_PRECEDENCE is not set +CONFIG_NEW_STRING_PRECEDENCE="user value" +CONFIG_NEW_TRISTATE_PRECEDENCE=y +CONFIG_NEW_HEX_PRECEDENCE=0xABCD +CONFIG_NEW_INT_PRECEDENCE=100 +# CONFIG_REGULAR_OPTION is not set diff --git a/scripts/kconfig/tests/transitional/initial_config b/scripts/kconfig/tests/transitional/initial_config new file mode 100644 index 000000000000..e648a65e504c --- /dev/null +++ b/scripts/kconfig/tests/transitional/initial_config @@ -0,0 +1,16 @@ +CONFIG_MODULES=y +CONFIG_OLD_BOOL=y +CONFIG_OLD_TRISTATE=m +CONFIG_OLD_STRING="test string" +CONFIG_OLD_HEX=0x1234 +CONFIG_OLD_INT=42 +# CONFIG_NEW_BOOL_PRECEDENCE is not set +CONFIG_OLD_BOOL_PRECEDENCE=y +CONFIG_NEW_STRING_PRECEDENCE="user value" +CONFIG_OLD_STRING_PRECEDENCE="old value" +CONFIG_NEW_TRISTATE_PRECEDENCE=y +CONFIG_OLD_TRISTATE_PRECEDENCE=m +CONFIG_NEW_HEX_PRECEDENCE=0xABCD +CONFIG_OLD_HEX_PRECEDENCE=0x5678 +CONFIG_NEW_INT_PRECEDENCE=100 +CONFIG_OLD_INT_PRECEDENCE=200 -- cgit From 57c49d2355729c12475554b4c51dbf830b02d08d Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Thu, 18 Sep 2025 13:43:46 +0300 Subject: scripts/coccinelle: Find PTR_ERR() to %pe candidates Add a new Coccinelle script to identify places where PTR_ERR() is used in print functions and suggest using the %pe format specifier instead. For printing error pointers (i.e., a pointer for which IS_ERR() is true) %pe will print a symbolic error name (e.g,. -EINVAL), opposed to the raw errno (e.g,. -22) produced by PTR_ERR(). It also makes the code cleaner by saving a redundant call to PTR_ERR(). The script supports context, report, and org modes. Example transformation: printk("Error: %ld\n", PTR_ERR(ptr)); // Before printk("Error: %pe\n", ptr); // After Signed-off-by: Gal Pressman Reviewed-by: Alexei Lazar Signed-off-by: Tariq Toukan Reviewed-by: Simon Horman Link: https://patch.msgid.link/1758192227-701925-2-git-send-email-tariqt@nvidia.com Signed-off-by: Jakub Kicinski --- scripts/coccinelle/misc/ptr_err_to_pe.cocci | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 scripts/coccinelle/misc/ptr_err_to_pe.cocci (limited to 'scripts') diff --git a/scripts/coccinelle/misc/ptr_err_to_pe.cocci b/scripts/coccinelle/misc/ptr_err_to_pe.cocci new file mode 100644 index 000000000000..0494c7709245 --- /dev/null +++ b/scripts/coccinelle/misc/ptr_err_to_pe.cocci @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only +/// Use %pe format specifier instead of PTR_ERR() for printing error pointers. +/// +/// For printing error pointers (i.e., a pointer for which IS_ERR() is true) +/// %pe will print a symbolic error name (e.g., -EINVAL), opposed to the raw +/// errno (e.g., -22) produced by PTR_ERR(). +/// It also makes the code cleaner by saving a redundant call to PTR_ERR(). +/// +// Confidence: High +// Copyright: (C) 2025 NVIDIA CORPORATION & AFFILIATES. +// URL: https://coccinelle.gitlabpages.inria.fr/website +// Options: --no-includes --include-headers + +virtual context +virtual org +virtual report + +@r@ +expression ptr; +constant fmt; +position p; +identifier print_func; +@@ +* print_func(..., fmt, ..., PTR_ERR@p(ptr), ...) + +@script:python depends on r && report@ +p << r.p; +@@ +coccilib.report.print_report(p[0], "WARNING: Consider using %pe to print PTR_ERR()") + +@script:python depends on r && org@ +p << r.p; +@@ +coccilib.org.print_todo(p[0], "WARNING: Consider using %pe to print PTR_ERR()") -- cgit From 77e46093e83a2788e2cf80ba8c8731375e587ad2 Mon Sep 17 00:00:00 2001 From: Markus Heidelberg Date: Fri, 26 Sep 2025 12:57:49 +0200 Subject: scripts: dt_to_config: fix grammar and a typo in --help text - grammar: singular/plural inconsistency - typo: "of" -> "or" Signed-off-by: Markus Heidelberg Signed-off-by: Rob Herring (Arm) --- scripts/dtc/dt_to_config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/dtc/dt_to_config b/scripts/dtc/dt_to_config index 299d1c2b20d7..70d6d5f06bdc 100755 --- a/scripts/dtc/dt_to_config +++ b/scripts/dtc/dt_to_config @@ -51,10 +51,10 @@ $num_pr_flags = $pr_flag_pos_config_test_fail + 1; "compatible is white listed", "matching driver and/or kernel config is hard coded", "kernel config hard coded in Makefile", - "one or more kernel config file options is not set", - "one or more kernel config file options is set to 'm'", - "one or more kernel config file options is set to 'y'", - "one of more kernel config file options fails to have correct value" + "one or more kernel config file options are not set", + "one or more kernel config file options are set to 'm'", + "one or more kernel config file options are set to 'y'", + "one or more kernel config file options fail to have correct value" ); -- cgit From 2ea77fca84f07849aa995271271340d262d0c2e9 Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Sat, 27 Sep 2025 21:28:06 -0700 Subject: modpost: Initialize builtin_modname to stop SIGSEGVs Segmentation fault ./scripts/mod/modpost -o vmlinux.symvers vmlinux.o stops the kernel build. It comes when write_vmlinux_export_c_file() tries to buf_printf alias->builtin_modname. malloc'ed memory is not necessarily zeroed. NULL new->builtin_modname before adding to aliases. Fixes: 5ab23c7923a1 ("modpost: Create modalias for builtin modules") Signed-off-by: Hugh Dickins Reviewed-by: Nicolas Schier Link: https://patch.msgid.link/4590a243-0a7e-b7e6-e2d3-cd1b41a12237@google.com Signed-off-by: Nathan Chancellor --- scripts/mod/file2alias.c | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index 7da9735e7ab3..b3333560b95e 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -94,6 +94,7 @@ module_alias_printf(struct module *mod, bool append_wildcard, } } + new->builtin_modname = NULL; list_add_tail(&new->node, &mod->aliases); } -- cgit From 99b70ece33d87500ef7bee8e32cb99772c45ce14 Mon Sep 17 00:00:00 2001 From: Suchit Karunakaran Date: Tue, 23 Sep 2025 22:47:21 +0530 Subject: checkpatch: suppress strscpy warnings for userspace tools The checkpatch.pl script currently warns against the use of strcpy, strlcpy, and strncpy, recommending strscpy as a safer alternative. However, these warnings are also triggered for code under tools/ and scripts/, which are userspace utilities where strscpy is not available. This patch suppresses these warnings for files in tools/ and scripts/. Link: https://lkml.kernel.org/r/20250923171722.7798-1-suchitkarunakaran@gmail.com Signed-off-by: Suchit Karunakaran Acked-by: Joe Perches Cc: Andy Whitcroft Cc: Dwaipayan Ray Cc: Lukas Bulwahn Cc: Shuah Khan Signed-off-by: Andrew Morton --- scripts/checkpatch.pl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 319cc5f85885..92669904eecc 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -2636,6 +2636,11 @@ sub exclude_global_initialisers { $realfile =~ m@/bpf/.*\.bpf\.c$@; } +sub is_userspace { + my ($realfile) = @_; + return ($realfile =~ m@^tools/@ || $realfile =~ m@^scripts/@); +} + sub process { my $filename = shift; @@ -7018,21 +7023,20 @@ sub process { # } # } # } - # strcpy uses that should likely be strscpy - if ($line =~ /\bstrcpy\s*\(/) { + if ($line =~ /\bstrcpy\s*\(/ && !is_userspace($realfile)) { WARN("STRCPY", "Prefer strscpy over strcpy - see: https://github.com/KSPP/linux/issues/88\n" . $herecurr); } # strlcpy uses that should likely be strscpy - if ($line =~ /\bstrlcpy\s*\(/) { + if ($line =~ /\bstrlcpy\s*\(/ && !is_userspace($realfile)) { WARN("STRLCPY", "Prefer strscpy over strlcpy - see: https://github.com/KSPP/linux/issues/89\n" . $herecurr); } # strncpy uses that should likely be strscpy or strscpy_pad - if ($line =~ /\bstrncpy\s*\(/) { + if ($line =~ /\bstrncpy\s*\(/ && !is_userspace($realfile)) { WARN("STRNCPY", "Prefer strscpy, strscpy_pad, or __nonstring over strncpy - see: https://github.com/KSPP/linux/issues/90\n" . $herecurr); } -- cgit From de7342228b7343774d6a9981c2ddbfb5e201044b Mon Sep 17 00:00:00 2001 From: Rong Tao Date: Sat, 4 Oct 2025 22:23:29 +0800 Subject: bpf: Finish constification of 1st parameter of bpf_d_path() The commit 1b8abbb12128 ("bpf...d_path(): constify path argument") constified the first parameter of the bpf_d_path(), but failed to update it in all places. Finish constification. Otherwise the selftest fail to build: .../selftests/bpf/bpf_experimental.h:222:12: error: conflicting types for 'bpf_path_d_path' 222 | extern int bpf_path_d_path(const struct path *path, char *buf, size_t buf__sz) __ksym; | ^ .../selftests/bpf/tools/include/vmlinux.h:153922:12: note: previous declaration is here 153922 | extern int bpf_path_d_path(struct path *path, char *buf, size_t buf__sz) __weak __ksym; Fixes: 1b8abbb12128 ("bpf...d_path(): constify path argument") Signed-off-by: Rong Tao Signed-off-by: Alexei Starovoitov --- scripts/bpf_doc.py | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index c77dc40f7689..15d113a1bc1d 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -788,6 +788,7 @@ class PrinterHelpersHeader(Printer): 'struct task_struct', 'struct cgroup', 'struct path', + 'const struct path', 'struct btf_ptr', 'struct inode', 'struct socket', -- cgit From 0902b3cb23ce7f436bddbdf6ba7b1ed427b36bd9 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 30 Sep 2025 08:45:19 -0700 Subject: kconfig: Avoid prompting for transitional symbols The "transitional" symbol keyword, while working with the "olddefconfig" target, was prompting during "oldconfig". This occurred because these symbols were not being marked as user-defined when they received values from transitional symbols that had user values. The "olddefconfig" target explicitly doesn't prompt for anything, so this deficiency wasn't noticed. The issue manifested when a symbol's value came from a transitional symbol's user value but the receiving symbol wasn't marked with SYMBOL_DEF_USER. Thus the "oldconfig" logic would then prompt for these symbols unnecessarily. Check after value calculation whether a symbol without a user value gets its value from a single transitional symbol that does have a user value. In such cases, mark the receiving symbol as user-defined to prevent prompting. Update regression tests to verify that symbols with transitional defaults are not prompted in "oldconfig", except when conditional defaults evaluate to 'no' and should legitimately be prompted. Build tested with "make testconfig". Reported-by: Linus Torvalds Closes: https://lore.kernel.org/lkml/CAHk-=wgZjUk4Cy2XgNkTrQoO8XCmNUHrTe5D519Fij1POK+3qw@mail.gmail.com/ Fixes: f9afce4f32e9 ("kconfig: Add transitional symbol attribute for migration support") Cc: Vegard Nossum Link: https://lore.kernel.org/r/20250930154514.it.623-kees@kernel.org Signed-off-by: Kees Cook --- scripts/kconfig/symbol.c | 15 +++++++++- scripts/kconfig/tests/transitional/Kconfig | 32 ++++++++++++++++++++++ scripts/kconfig/tests/transitional/__init__.py | 7 +++++ scripts/kconfig/tests/transitional/expected_config | 3 ++ scripts/kconfig/tests/transitional/expected_stdout | 1 + scripts/kconfig/tests/transitional/initial_config | 4 +++ 6 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 scripts/kconfig/tests/transitional/expected_stdout (limited to 'scripts') diff --git a/scripts/kconfig/symbol.c b/scripts/kconfig/symbol.c index 760cac998381..7e81b3676ee9 100644 --- a/scripts/kconfig/symbol.c +++ b/scripts/kconfig/symbol.c @@ -411,7 +411,7 @@ bool sym_dep_errors(void) void sym_calc_value(struct symbol *sym) { struct symbol_value newval, oldval; - struct property *prop; + struct property *prop = NULL; struct menu *choice_menu; if (!sym) @@ -520,6 +520,19 @@ void sym_calc_value(struct symbol *sym) ; } + /* + * If the symbol lacks a user value but its value comes from a + * single transitional symbol with an existing user value, mark + * this symbol as having a user value to avoid prompting. + */ + if (prop && !sym_has_value(sym)) { + struct symbol *ds = prop_get_symbol(prop); + if (ds && (ds->flags & SYMBOL_TRANS) && sym_has_value(ds)) { + sym->def[S_DEF_USER] = newval; + sym->flags |= SYMBOL_DEF_USER; + } + } + sym->curr = newval; sym_validate_range(sym); diff --git a/scripts/kconfig/tests/transitional/Kconfig b/scripts/kconfig/tests/transitional/Kconfig index 62c3b24665b9..faa4d396f828 100644 --- a/scripts/kconfig/tests/transitional/Kconfig +++ b/scripts/kconfig/tests/transitional/Kconfig @@ -96,5 +96,37 @@ config OLD_WITH_HELP help This transitional symbol has a help section to validate that help is allowed. +# Test that we can set something to =n via transitional symbol +config NEW_DISABLED + tristate "Check for setting to disabled" + default OLD_DISABLED + +config OLD_DISABLED + tristate + transitional + +# Test that a potential new value disappears if it lacks a prompt +config NEW_DISABLED_UNSAVED + tristate + default OLD_DISABLED + +config OLD_DISABLED_UNSAVED + tristate + transitional + +# Test conditional default: transitional value should not prevent prompting +# when default visibility makes the expression evaluate to 'no' +config DEPENDENCY_TEST + bool "Dependency for testing" + default n + +config NEW_CONDITIONAL_DEFAULT + bool "New option with conditional default" + default OLD_CONDITIONAL_DEFAULT if DEPENDENCY_TEST + +config OLD_CONDITIONAL_DEFAULT + bool + transitional + config REGULAR_OPTION bool "Regular option" diff --git a/scripts/kconfig/tests/transitional/__init__.py b/scripts/kconfig/tests/transitional/__init__.py index 61937d10edf1..b50ba2397548 100644 --- a/scripts/kconfig/tests/transitional/__init__.py +++ b/scripts/kconfig/tests/transitional/__init__.py @@ -6,6 +6,7 @@ This tests that: - OLD_* options in existing .config cause NEW_* options to be set - OLD_* options are not written to the new .config file - NEW_* options appear in the new .config file with correct values +- NEW_* options with defaults from transitional symbols are not prompted - All Kconfig types work correctly: bool, tristate, string, hex, int - User-set NEW values take precedence over conflicting OLD transitional values """ @@ -16,3 +17,9 @@ def test(conf): # Check that the configuration matches expected output assert conf.config_contains('expected_config') + + # Test oldconfig to ensure symbols with transitional defaults are not prompted + assert conf.oldconfig(dot_config='initial_config', in_keys='n\n') == 0 + + # Except for when conditional default evaluates to 'no' + assert conf.stdout_contains('expected_stdout') diff --git a/scripts/kconfig/tests/transitional/expected_config b/scripts/kconfig/tests/transitional/expected_config index 846e9ddcab91..e01f5f070a26 100644 --- a/scripts/kconfig/tests/transitional/expected_config +++ b/scripts/kconfig/tests/transitional/expected_config @@ -9,4 +9,7 @@ CONFIG_NEW_STRING_PRECEDENCE="user value" CONFIG_NEW_TRISTATE_PRECEDENCE=y CONFIG_NEW_HEX_PRECEDENCE=0xABCD CONFIG_NEW_INT_PRECEDENCE=100 +# CONFIG_NEW_DISABLED is not set +# CONFIG_DEPENDENCY_TEST is not set +# CONFIG_NEW_CONDITIONAL_DEFAULT is not set # CONFIG_REGULAR_OPTION is not set diff --git a/scripts/kconfig/tests/transitional/expected_stdout b/scripts/kconfig/tests/transitional/expected_stdout new file mode 100644 index 000000000000..6f0b285d6469 --- /dev/null +++ b/scripts/kconfig/tests/transitional/expected_stdout @@ -0,0 +1 @@ +New option with conditional default (NEW_CONDITIONAL_DEFAULT) [N/y/?] (NEW) n diff --git a/scripts/kconfig/tests/transitional/initial_config b/scripts/kconfig/tests/transitional/initial_config index e648a65e504c..68b7da672426 100644 --- a/scripts/kconfig/tests/transitional/initial_config +++ b/scripts/kconfig/tests/transitional/initial_config @@ -14,3 +14,7 @@ CONFIG_NEW_HEX_PRECEDENCE=0xABCD CONFIG_OLD_HEX_PRECEDENCE=0x5678 CONFIG_NEW_INT_PRECEDENCE=100 CONFIG_OLD_INT_PRECEDENCE=200 +# CONFIG_OLD_DISABLED is not set +# CONFIG_OLD_DISABLED_UNSAVED is not set +# CONFIG_DEPENDENCY_TEST is not set +CONFIG_OLD_CONDITIONAL_DEFAULT=y -- cgit From 7ded7d37e5f5b36b4acd74380156cf07b6640c5b Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 6 Oct 2025 14:39:56 -0700 Subject: scripts/Makefile.extrawarn: Respect CONFIG_WERROR / W=e for hostprogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 27758d8c2583 ("kbuild: enable -Werror for hostprogs") unconditionally enabled -Werror for the compiler, assembler, and linker when building the host programs, as the build footprint of the host programs is small (thus risk of build failures from warnings are low) and that stage of the build may not have Kconfig values (thus CONFIG_WERROR could not be used as a precondition). While turning warnings into errors unconditionally happens in a few places within the kernel, it can be disruptive to people who may be building with newer compilers, such as while doing a bisect. While it is possible to avoid this behavior by passing HOSTCFLAGS=-w or HOSTCFLAGS=-Wno-error, it may not be the most intuitive for regular users not intimately familiar with Kbuild. Avoid being disruptive to the entire build by depending on the explicit opt-in of CONFIG_WERROR or W=e to enable -Werror and the like while building the host programs. While this means there is a small portion of the build that does not have -Werror enabled (namely scripts/kconfig/* and scripts/basic/fixdep), it is better than not having it altogether. Fixes: 27758d8c2583 ("kbuild: enable -Werror for hostprogs") Acked-by: Miguel Ojeda Reported-by: Askar Safin Closes: https://lore.kernel.org/20251005011100.1035272-1-safinaskar@gmail.com/ Reviewed-by: Thomas Weißschuh Tested-by: Miguel Ojeda # Rust Link: https://patch.msgid.link/20251006-kbuild-hostprogs-werror-fix-v1-1-23cf1ffced5c@kernel.org Signed-off-by: Nathan Chancellor --- scripts/Makefile.extrawarn | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn index 1434cb6208cb..6af392f9cd02 100644 --- a/scripts/Makefile.extrawarn +++ b/scripts/Makefile.extrawarn @@ -223,9 +223,11 @@ KBUILD_USERCFLAGS += -Werror KBUILD_USERLDFLAGS += -Wl,--fatal-warnings KBUILD_RUSTFLAGS += -Dwarnings -endif - -# Hostprog flags are used during build bootstrapping and can not rely on CONFIG_ symbols. +# While hostprog flags are used during build bootstrapping (thus should not +# depend on CONFIG_ symbols), -Werror is disruptive and should be opted into. +# Only apply -Werror to hostprogs built after the initial Kconfig stage. KBUILD_HOSTCFLAGS += -Werror KBUILD_HOSTLDFLAGS += -Wl,--fatal-warnings KBUILD_HOSTRUSTFLAGS += -Dwarnings + +endif -- cgit From 4b47a3aefb29c523ca66f0d28de8db15a10f9352 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 8 Oct 2025 15:46:44 -0700 Subject: kbuild: Restore pattern to avoid stripping .rela.dyn from vmlinux Commit 0ce5139fd96e ("kbuild: always create intermediate vmlinux.unstripped") removed the pattern to avoid stripping .rela.dyn sections added by commit e9d86b8e17e7 ("scripts: Do not strip .rela.dyn section"). Restore it so that .rela.dyn sections remain in the final vmlinux. Fixes: 0ce5139fd96e ("kbuild: always create intermediate vmlinux.unstripped") Acked-by: Ard Biesheuvel Acked-by: Alexey Gladkov Acked-by: Nicolas Schier Link: https://patch.msgid.link/20251008-kbuild-fix-modinfo-regressions-v1-1-9fc776c5887c@kernel.org Signed-off-by: Nathan Chancellor --- scripts/Makefile.vmlinux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index 7c6ae9886f8f..a62639982be5 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -82,7 +82,7 @@ endif # --------------------------------------------------------------------------- remove-section-y := .modinfo -remove-section-$(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS) += '.rel*' +remove-section-$(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS) += '.rel*' '!.rel*.dyn' remove-symbols := -w --strip-symbol='__mod_device_table__*' -- cgit From 8ec3af916fe3954381cf3555ea03dc5adf4d0e8e Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 8 Oct 2025 15:46:45 -0700 Subject: kbuild: Add '.rel.*' strip pattern for vmlinux Prior to binutils commit c12d9fa2afe ("Support objcopy --remove-section=.relaFOO") [1] in 2.32, stripping relocation sections required the trailing period (i.e., '.rel.*') to work properly. After commit 3e86e4d74c04 ("kbuild: keep .modinfo section in vmlinux.unstripped"), there is an error with binutils 2.31.1 or earlier because these sections are not properly removed: s390-linux-objcopy: st6tO8Ev: symbol `.modinfo' required but not present s390-linux-objcopy:st6tO8Ev: no symbols Add the old pattern to resolve this issue (along with a comment to allow cleaning this when binutils 2.32 or newer is the minimum supported version). While the aforementioned kbuild change exposes this, the pattern was originally changed by commit 71d815bf5dfd ("kbuild: Strip runtime const RELA sections correctly"), where it would still be incorrect with binutils older than 2.32. Fixes: 71d815bf5dfd ("kbuild: Strip runtime const RELA sections correctly") Link: https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=c12d9fa2afe7abcbe407a00e15719e1a1350c2a7 [1] Reported-by: Linux Kernel Functional Testing Closes: https://lore.kernel.org/CA+G9fYvVktRhFtZXdNgVOL8j+ArsJDpvMLgCitaQvQmCx=hwOQ@mail.gmail.com/ Acked-by: Ard Biesheuvel Acked-by: Alexey Gladkov Acked-by: Nicolas Schier Link: https://patch.msgid.link/20251008-kbuild-fix-modinfo-regressions-v1-2-9fc776c5887c@kernel.org Signed-off-by: Nathan Chancellor --- scripts/Makefile.vmlinux | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index a62639982be5..c02f85c2e241 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -83,6 +83,9 @@ endif remove-section-y := .modinfo remove-section-$(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS) += '.rel*' '!.rel*.dyn' +# for compatibility with binutils < 2.32 +# https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=c12d9fa2afe7abcbe407a00e15719e1a1350c2a7 +remove-section-$(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS) += '.rel.*' remove-symbols := -w --strip-symbol='__mod_device_table__*' -- cgit From b0f2942a16017f88395d768afedd7373860968ce Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Fri, 10 Oct 2025 14:49:27 -0700 Subject: kbuild: Use '--strip-unneeded-symbol' for removing module device table symbols After commit 5ab23c7923a1 ("modpost: Create modalias for builtin modules"), relocatable RISC-V kernels with CONFIG_KASAN=y start failing when attempting to strip the module device table symbols: riscv64-linux-objcopy: not stripping symbol `__mod_device_table__kmod_irq_starfive_jh8100_intc__of__starfive_intc_irqchip_match_table' because it is named in a relocation make[4]: *** [scripts/Makefile.vmlinux:97: vmlinux] Error 1 The relocation appears to come from .LASANLOC5 in .data.rel.local: $ llvm-objdump --disassemble-symbols=.LASANLOC5 --disassemble-all -r drivers/irqchip/irq-starfive-jh8100-intc.o drivers/irqchip/irq-starfive-jh8100-intc.o: file format elf64-littleriscv Disassembly of section .data.rel.local: 0000000000000180 <.LASANLOC5>: ... 1d0: 0000 unimp 00000000000001d0: R_RISCV_64 __mod_device_table__kmod_irq_starfive_jh8100_intc__of__starfive_intc_irqchip_match_table ... This section appears to come from GCC for including additional information about global variables that may be protected by KASAN. There appears to be no way to opt out of the generation of these symbols through either a flag or attribute. Attempting to remove '.LASANLOC*' with '--strip-symbol' results in the same error as above because these symbols may refer to (thus have relocation between) each other. Avoid this build breakage by switching to '--strip-unneeded-symbol' for removing __mod_device_table__ symbols, as it will only remove the symbol when there is no relocation pointing to it. While this may result in a little more bloat in the symbol table in certain configurations, it is not as bad as outright build failures. Fixes: 5ab23c7923a1 ("modpost: Create modalias for builtin modules") Reported-by: Charles Mirabile Closes: https://lore.kernel.org/20251007011637.2512413-1-cmirabil@redhat.com/ Suggested-by: Alexey Gladkov Tested-by: Nicolas Schier Signed-off-by: Nathan Chancellor --- scripts/Makefile.vmlinux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index c02f85c2e241..ced4379550d7 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -87,7 +87,7 @@ remove-section-$(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS) += '.rel*' '!.rel*.dyn' # https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=c12d9fa2afe7abcbe407a00e15719e1a1350c2a7 remove-section-$(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS) += '.rel.*' -remove-symbols := -w --strip-symbol='__mod_device_table__*' +remove-symbols := -w --strip-unneeded-symbol='__mod_device_table__*' # To avoid warnings: "empty loadable segment detected at ..." from GNU objcopy, # it is necessary to remove the PT_LOAD flag from the segment. -- cgit From 5ff90d427ef841fa48608d0c19a81c48d6126d46 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Thu, 16 Oct 2025 10:14:17 +0100 Subject: kbuild: install-extmod-build: Fix when given dir outside the build dir Commit b5e395653546 ("kbuild: install-extmod-build: Fix build when specifying KBUILD_OUTPUT") tried to address the "build" variable expecting a relative path by using `realpath --relative-base=.`, but this only works when the given directory is below the current directory. `realpath --relative-to=.` will return a relative path in all cases. Fixes: b5e395653546 ("kbuild: install-extmod-build: Fix build when specifying KBUILD_OUTPUT") Signed-off-by: James Le Cuirot Reviewed-by: Nicolas Schier Link: https://patch.msgid.link/20251016091417.9985-1-chewi@gentoo.org Signed-off-by: Nathan Chancellor --- scripts/package/install-extmod-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/package/install-extmod-build b/scripts/package/install-extmod-build index b96538787f3d..054fdf45cc37 100755 --- a/scripts/package/install-extmod-build +++ b/scripts/package/install-extmod-build @@ -63,7 +63,7 @@ if [ "${CC}" != "${HOSTCC}" ]; then # Clear VPATH and srcroot because the source files reside in the output # directory. # shellcheck disable=SC2016 # $(MAKE) and $(build) will be expanded by Make - "${MAKE}" run-command KBUILD_RUN_COMMAND='+$(MAKE) HOSTCC='"${CC}"' VPATH= srcroot=. $(build)='"$(realpath --relative-base=. "${destdir}")"/scripts + "${MAKE}" run-command KBUILD_RUN_COMMAND='+$(MAKE) HOSTCC='"${CC}"' VPATH= srcroot=. $(build)='"$(realpath --relative-to=. "${destdir}")"/scripts rm -f "${destdir}/scripts/Kbuild" fi -- cgit From 3927c4a1084c48ef97f11281a0a43ecb2cb4d6f1 Mon Sep 17 00:00:00 2001 From: Jakub Horký Date: Tue, 14 Oct 2025 17:49:32 +0200 Subject: kconfig/mconf: Initialize the default locale at startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix bug where make menuconfig doesn't initialize the default locale, which causes ncurses menu borders to be displayed incorrectly (lqqqqk) in UTF-8 terminals that don't support VT100 ACS by default, such as PuTTY. Signed-off-by: Jakub Horký Link: https://patch.msgid.link/20251014154933.3990990-1-jakub.git@horky.net [nathan: Alphabetize locale.h include] Signed-off-by: Nathan Chancellor --- scripts/kconfig/mconf.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/kconfig/mconf.c b/scripts/kconfig/mconf.c index 84ea9215c0a7..b8b7bba84a65 100644 --- a/scripts/kconfig/mconf.c +++ b/scripts/kconfig/mconf.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -931,6 +932,8 @@ int main(int ac, char **av) signal(SIGINT, sig_handler); + setlocale(LC_ALL, ""); + if (ac > 1 && strcmp(av[1], "-s") == 0) { silent = 1; /* Silence conf_read() until the real callback is set up */ -- cgit From 43c2931a95e6b295bfe9e3b90dbe0f7596933e91 Mon Sep 17 00:00:00 2001 From: Jakub Horký Date: Tue, 14 Oct 2025 16:44:06 +0200 Subject: kconfig/nconf: Initialize the default locale at startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix bug where make nconfig doesn't initialize the default locale, which causes ncurses menu borders to be displayed incorrectly (lqqqqk) in UTF-8 terminals that don't support VT100 ACS by default, such as PuTTY. Signed-off-by: Jakub Horký Link: https://patch.msgid.link/20251014144405.3975275-2-jakub.git@horky.net [nathan: Alphabetize locale.h include] Signed-off-by: Nathan Chancellor --- scripts/kconfig/nconf.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/kconfig/nconf.c b/scripts/kconfig/nconf.c index ae1fe5f60327..521700ed7152 100644 --- a/scripts/kconfig/nconf.c +++ b/scripts/kconfig/nconf.c @@ -7,6 +7,7 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif +#include #include #include #include @@ -1478,6 +1479,8 @@ int main(int ac, char **av) int lines, columns; char *mode; + setlocale(LC_ALL, ""); + if (ac > 1 && strcmp(av[1], "-s") == 0) { /* Silence conf_read() until the real callback is set up */ conf_set_message_callback(NULL); -- cgit From a26a6c93edfeee82cb73f55e87d995eea59ddfe8 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 5 Nov 2025 15:30:27 -0700 Subject: kbuild: Strip trailing padding bytes from modules.builtin.modinfo After commit d50f21091358 ("kbuild: align modinfo section for Secureboot Authenticode EDK2 compat"), running modules_install with certain versions of kmod (such as 29.1 in Ubuntu Jammy) in certain configurations may fail with: depmod: ERROR: kmod_builtin_iter_next: unexpected string without modname prefix The additional padding bytes to ensure .modinfo is aligned within vmlinux.unstripped are unexpected by kmod, as this section has always just been null-terminated strings. Strip the trailing padding bytes from modules.builtin.modinfo after it has been extracted from vmlinux.unstripped to restore the format that kmod expects while keeping .modinfo aligned within vmlinux.unstripped to avoid regressing the Authenticode calculation fix for EDK2. Cc: stable@vger.kernel.org Fixes: d50f21091358 ("kbuild: align modinfo section for Secureboot Authenticode EDK2 compat") Reported-by: Omar Sandoval Reported-by: Samir M Reported-by: Venkat Rao Bagalkote Closes: https://lore.kernel.org/7fef7507-ad64-4e51-9bb8-c9fb6532e51e@linux.ibm.com/ Tested-by: Omar Sandoval Tested-by: Samir M Tested-by: Venkat Rao Bagalkote Reviewed-by: Nicolas Schier Link: https://patch.msgid.link/20251105-kbuild-fix-builtin-modinfo-for-kmod-v1-1-b419d8ad4606@kernel.org Signed-off-by: Nathan Chancellor --- scripts/Makefile.vmlinux | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index ced4379550d7..cd788cac9d91 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -102,11 +102,24 @@ vmlinux: vmlinux.unstripped FORCE # modules.builtin.modinfo # --------------------------------------------------------------------------- +# .modinfo in vmlinux.unstripped is aligned to 8 bytes for compatibility with +# tools that expect vmlinux to have sufficiently aligned sections but the +# additional bytes used for padding .modinfo to satisfy this requirement break +# certain versions of kmod with +# +# depmod: ERROR: kmod_builtin_iter_next: unexpected string without modname prefix +# +# Strip the trailing padding bytes after extracting .modinfo to comply with +# what kmod expects to parse. +quiet_cmd_modules_builtin_modinfo = GEN $@ + cmd_modules_builtin_modinfo = $(cmd_objcopy); \ + sed -i 's/\x00\+$$/\x00/g' $@ + OBJCOPYFLAGS_modules.builtin.modinfo := -j .modinfo -O binary targets += modules.builtin.modinfo modules.builtin.modinfo: vmlinux.unstripped FORCE - $(call if_changed,objcopy) + $(call if_changed,modules_builtin_modinfo) # modules.builtin # --------------------------------------------------------------------------- -- cgit From 002621a4df3c166fab1427e8e502bc15acc26b13 Mon Sep 17 00:00:00 2001 From: Jean Delvare Date: Fri, 7 Nov 2025 19:29:33 +0100 Subject: kbuild: Let kernel-doc.py use PYTHON3 override It is possible to force a specific version of python to be used when building the kernel by passing PYTHON3= on the make command line. However kernel-doc.py is currently called with python3 hard-coded and thus ignores this setting. Use $(PYTHON3) to run $(KERNELDOC) so that the desired version of python is used. Signed-off-by: Jean Delvare Reviewed-by: Nicolas Schier Reviewed-by: Mauro Carvalho Chehab Link: https://patch.msgid.link/20251107192933.2bfe9e57@endymion Signed-off-by: Nathan Chancellor --- scripts/Makefile.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.build b/scripts/Makefile.build index d0ee33a487be..52c08c4eb0b9 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -167,7 +167,7 @@ else ifeq ($(KBUILD_CHECKSRC),2) endif ifneq ($(KBUILD_EXTRA_WARN),) - cmd_checkdoc = PYTHONDONTWRITEBYTECODE=1 $(KERNELDOC) -none $(KDOCFLAGS) \ + cmd_checkdoc = PYTHONDONTWRITEBYTECODE=1 $(PYTHON3) $(KERNELDOC) -none $(KDOCFLAGS) \ $(if $(findstring 2, $(KBUILD_EXTRA_WARN)), -Wall) \ $< endif -- cgit From 7d9f7d390f6af3a29614e81e802e2b9c238eb7b2 Mon Sep 17 00:00:00 2001 From: Carlos Llamas Date: Thu, 30 Oct 2025 01:03:33 +0000 Subject: scripts/decode_stacktrace.sh: fix build ID and PC source parsing Support for parsing PC source info in stacktraces (e.g. '(P)') was added in commit 2bff77c665ed ("scripts/decode_stacktrace.sh: fix decoding of lines with an additional info"). However, this logic was placed after the build ID processing. This incorrect order fails to parse lines containing both elements, e.g.: drm_gem_mmap_obj+0x114/0x200 [drm 03d0564e0529947d67bb2008c3548be77279fd27] (P) This patch fixes the problem by extracting the PC source info first and then processing the module build ID. With this change, the line above is now properly parsed as such: drm_gem_mmap_obj (./include/linux/mmap_lock.h:212 ./include/linux/mm.h:811 drivers/gpu/drm/drm_gem.c:1177) drm (P) While here, also add a brief explanation the build ID section. Link: https://lkml.kernel.org/r/20251030010347.2731925-1-cmllamas@google.com Fixes: 2bff77c665ed ("scripts/decode_stacktrace.sh: fix decoding of lines with an additional info") Signed-off-by: Carlos Llamas Reviewed-by: Matthieu Baerts (NGI0) Reviewed-by: Luca Ceresoli Cc: Breno Leitao Cc: Catalin Marinas Cc: Marc Rutland Cc: Mark Brown Cc: Matthieu Baerts Cc: Miroslav Benes Cc: Puranjay Mohan Cc: Signed-off-by: Andrew Morton --- scripts/decode_stacktrace.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/decode_stacktrace.sh b/scripts/decode_stacktrace.sh index c73cb802a0a3..8d01b741de62 100755 --- a/scripts/decode_stacktrace.sh +++ b/scripts/decode_stacktrace.sh @@ -277,12 +277,6 @@ handle_line() { fi done - if [[ ${words[$last]} =~ ^[0-9a-f]+\] ]]; then - words[$last-1]="${words[$last-1]} ${words[$last]}" - unset words[$last] spaces[$last] - last=$(( $last - 1 )) - fi - # Extract info after the symbol if present. E.g.: # func_name+0x54/0x80 (P) # ^^^ @@ -295,6 +289,14 @@ handle_line() { last=$(( $last - 1 )) fi + # Join module name with its build id if present, as these were + # split during tokenization (e.g. "[module" and "modbuildid]"). + if [[ ${words[$last]} =~ ^[0-9a-f]+\] ]]; then + words[$last-1]="${words[$last-1]} ${words[$last]}" + unset words[$last] spaces[$last] + last=$(( $last - 1 )) + fi + if [[ ${words[$last]} =~ \[([^]]+)\] ]]; then module=${words[$last]} # some traces format is "(%pS)", which like "(foo+0x0/0x1 [bar])" -- cgit From fdf302e6bea1822a9144a0cc2e8e17527e746162 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Mon, 10 Nov 2025 14:19:13 +0100 Subject: gendwarfksyms: Skip files with no exports Starting with Rust 1.91.0 (released 2025-10-30), in upstream commit ab91a63d403b ("Ignore intrinsic calls in cross-crate-inlining cost model") [1][2], `bindings.o` stops containing DWARF debug information because the `Default` implementations contained `write_bytes()` calls which are now ignored in that cost model (note that `CLIPPY=1` does not reproduce it). This means `gendwarfksyms` complains: RUSTC L rust/bindings.o error: gendwarfksyms: process_module: dwarf_get_units failed: no debugging information? There are several alternatives that would work here: conditionally skipping in the cases needed (but that is subtle and brittle), forcing DWARF generation with e.g. a dummy `static` (ugly and we may need to do it in several crates), skipping the call to the tool in the Kbuild command when there are no exports (fine) or teaching the tool to do so itself (simple and clean). Thus do the last one: don't attempt to process files if we have no symbol versions to calculate. [ I used the commit log of my patch linked below since it explained the root issue and expanded it a bit more to summarize the alternatives. - Miguel ] Cc: stable@vger.kernel.org # Needed in 6.17.y. Reported-by: Haiyue Wang Closes: https://lore.kernel.org/rust-for-linux/b8c1c73d-bf8b-4bf2-beb1-84ffdcd60547@163.com/ Suggested-by: Miguel Ojeda Link: https://lore.kernel.org/rust-for-linux/CANiq72nKC5r24VHAp9oUPR1HVPqT+=0ab9N0w6GqTF-kJOeiSw@mail.gmail.com/ Link: https://github.com/rust-lang/rust/commit/ab91a63d403b0105cacd72809cd292a72984ed99 [1] Link: https://github.com/rust-lang/rust/pull/145910 [2] Signed-off-by: Sami Tolvanen Tested-by: Haiyue Wang Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20251110131913.1789896-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/gendwarfksyms/gendwarfksyms.c | 3 ++- scripts/gendwarfksyms/gendwarfksyms.h | 2 +- scripts/gendwarfksyms/symbols.c | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/gendwarfksyms/gendwarfksyms.c b/scripts/gendwarfksyms/gendwarfksyms.c index 08ae61eb327e..f5203d1640ee 100644 --- a/scripts/gendwarfksyms/gendwarfksyms.c +++ b/scripts/gendwarfksyms/gendwarfksyms.c @@ -138,7 +138,8 @@ int main(int argc, char **argv) error("no input files?"); } - symbol_read_exports(stdin); + if (!symbol_read_exports(stdin)) + return 0; if (symtypes_file) { symfile = fopen(symtypes_file, "w"); diff --git a/scripts/gendwarfksyms/gendwarfksyms.h b/scripts/gendwarfksyms/gendwarfksyms.h index d9c06d2cb1df..32cec8f7695a 100644 --- a/scripts/gendwarfksyms/gendwarfksyms.h +++ b/scripts/gendwarfksyms/gendwarfksyms.h @@ -123,7 +123,7 @@ struct symbol { typedef void (*symbol_callback_t)(struct symbol *, void *arg); bool is_symbol_ptr(const char *name); -void symbol_read_exports(FILE *file); +int symbol_read_exports(FILE *file); void symbol_read_symtab(int fd); struct symbol *symbol_get(const char *name); void symbol_set_ptr(struct symbol *sym, Dwarf_Die *ptr); diff --git a/scripts/gendwarfksyms/symbols.c b/scripts/gendwarfksyms/symbols.c index 35ed594f0749..ecddcb5ffcdf 100644 --- a/scripts/gendwarfksyms/symbols.c +++ b/scripts/gendwarfksyms/symbols.c @@ -128,7 +128,7 @@ static bool is_exported(const char *name) return for_each(name, NULL, NULL) > 0; } -void symbol_read_exports(FILE *file) +int symbol_read_exports(FILE *file) { struct symbol *sym; char *line = NULL; @@ -159,6 +159,8 @@ void symbol_read_exports(FILE *file) free(line); debug("%d exported symbols", nsym); + + return nsym; } static void get_symbol(struct symbol *sym, void *arg) -- cgit