scripts/kernel_doc.py: better handle exported symbols

Change the logic which detects internal/external symbols in a way
that we can re-use it when calling via Sphinx extension.

While here, remove an unused self.config var and let it clearer
that self.config variables are read-only. This helps to allow
handling multiple times in parallel if ever needed.

Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
Signed-off-by: Jonathan Corbet <corbet@lwn.net>
Link: https://lore.kernel.org/r/6a69ba8d2b7ee6a6427abb53e60d09bd4d3565ee.1744106242.git.mchehab+huawei@kernel.org

Author: mchehab

scripts/kernel-doc.py

@@ -287,7 +287,7 @@ def main():
 
     for t in kfiles.msg(enable_lineno=args.enable_lineno, export=args.export,
                         internal=args.internal, symbol=args.symbol,
-                        nosymbol=args.nosymbol,
+                        nosymbol=args.nosymbol, export_file=args.export_file,
                         no_doc_sections=args.no_doc_sections):
         msg = t[1]
         if msg:

scripts/lib/kdoc/kdoc_files.py

@@ -68,6 +68,9 @@ def parse_files(self, file_list, file_not_found_cb):
         handling directories if any
         """
 
+        if not file_list:
+            return
+
         for fname in file_list:
             if self.srctree:
                 f = os.path.join(self.srctree, fname)
@@ -84,40 +87,70 @@ def parse_files(self, file_list, file_not_found_cb):
 
 class KernelFiles():
     """
-    Parse lernel-doc tags on multiple kernel source files.
+    Parse kernel-doc tags on multiple kernel source files.
+
+    There are two type of parsers defined here:
+        - self.parse_file(): parses both kernel-doc markups and
+          EXPORT_SYMBOL* macros;
+        - self.process_export_file(): parses only EXPORT_SYMBOL* macros.
     """
 
+    def warning(self, msg):
+        """Ancillary routine to output a warning and increment error count"""
+
+        self.config.log.warning(msg)
+        self.errors += 1
+
+    def error(self, msg):
+        """Ancillary routine to output an error and increment error count"""
+
+        self.config.log.error(msg)
+        self.errors += 1
+
     def parse_file(self, fname):
         """
         Parse a single Kernel source.
         """
 
+        # Prevent parsing the same file twice if results are cached
+        if fname in self.files:
+            return
+
         doc = KernelDoc(self.config, fname)
-        doc.run()
+        export_table, entries = doc.parse_kdoc()
+
+        self.export_table[fname] = export_table
+
+        self.files.add(fname)
+        self.export_files.add(fname)      # parse_kdoc() already check exports
 
-        return doc.entries
+        self.results[fname] = entries
 
     def process_export_file(self, fname):
         """
         Parses EXPORT_SYMBOL* macros from a single Kernel source file.
         """
-        try:
-            with open(fname, "r", encoding="utf8",
-                      errors="backslashreplace") as fp:
-                for line in fp:
-                    KernelDoc.process_export(self.config.function_table, line)
 
-        except IOError:
-            self.config.log.error("Error: Cannot open fname %s", fname)
-            self.config.errors += 1
+        # Prevent parsing the same file twice if results are cached
+        if fname in self.export_files:
+            return
+
+        doc = KernelDoc(self.config, fname)
+        export_table = doc.parse_export()
+
+        if not export_table:
+            self.error(f"Error: Cannot check EXPORT_SYMBOL* on {fname}")
+            export_table = set()
+
+        self.export_table[fname] = export_table
+        self.export_files.add(fname)
 
     def file_not_found_cb(self, fname):
         """
         Callback to warn if a file was not found.
         """
 
-        self.config.log.error("Cannot find file %s", fname)
-        self.config.errors += 1
+        self.error(f"Cannot find file {fname}")
 
     def __init__(self, verbose=False, out_style=None,
                  werror=False, wreturn=False, wshort_desc=False,
@@ -147,7 +180,9 @@ def __init__(self, verbose=False, out_style=None,
             if kdoc_werror:
                 werror = kdoc_werror
 
-        # Set global config data used on all files
+        # Some variables are global to the parser logic as a whole as they are
+        # used to send control configuration to KernelDoc class. As such,
+        # those variables are read-only inside the KernelDoc.
         self.config = argparse.Namespace
 
         self.config.verbose = verbose
@@ -156,27 +191,25 @@ def __init__(self, verbose=False, out_style=None,
         self.config.wshort_desc = wshort_desc
         self.config.wcontents_before_sections = wcontents_before_sections
 
-        self.config.function_table = set()
-        self.config.source_map = {}
-
         if not logger:
             self.config.log = logging.getLogger("kernel-doc")
         else:
             self.config.log = logger
 
-        self.config.kernel_version = os.environ.get("KERNELVERSION",
-                                                    "unknown kernel version'")
+        self.config.warning = self.warning
+
         self.config.src_tree = os.environ.get("SRCTREE", None)
 
-        self.out_style = out_style
+        # Initialize variables that are internal to KernelFiles
 
-        # Initialize internal variables
+        self.out_style = out_style
 
-        self.config.errors = 0
+        self.errors = 0
         self.results = {}
 
         self.files = set()
         self.export_files = set()
+        self.export_table = {}
 
     def parse(self, file_list, export_file=None):
         """
@@ -185,28 +218,11 @@ def parse(self, file_list, export_file=None):
 
         glob = GlobSourceFiles(srctree=self.config.src_tree)
 
-        # Prevent parsing the same file twice to speedup parsing and
-        # avoid reporting errors multiple times
-
         for fname in glob.parse_files(file_list, self.file_not_found_cb):
-            if fname not in self.files:
-                self.results[fname] = self.parse_file(fname)
-                self.files.add(fname)
-
-        # If a list of export files was provided, parse EXPORT_SYMBOL*
-        # from files that weren't fully parsed
-
-        if not export_file:
-            return
-
-        self.export_files |= self.files
-
-        glob = GlobSourceFiles(srctree=self.config.src_tree)
+            self.parse_file(fname)
 
         for fname in glob.parse_files(export_file, self.file_not_found_cb):
-            if fname not in self.export_files:
-                self.process_export_file(fname)
-                self.export_files.add(fname)
+            self.process_export_file(fname)
 
     def out_msg(self, fname, name, arg):
         """
@@ -223,32 +239,35 @@ def out_msg(self, fname, name, arg):
 
     def msg(self, enable_lineno=False, export=False, internal=False,
             symbol=None, nosymbol=None, no_doc_sections=False,
-            filenames=None):
+            filenames=None, export_file=None):
         """
         Interacts over the kernel-doc results and output messages,
         returning kernel-doc markups on each interaction
         """
 
-        function_table = self.config.function_table
-
-        if symbol:
-            for s in symbol:
-                function_table.add(s)
-
-        # Output none mode: only warnings will be shown
-        if not self.out_style:
-            return
-
         self.out_style.set_config(self.config)
 
-        self.out_style.set_filter(export, internal, symbol, nosymbol,
-                                  function_table, enable_lineno,
-                                  no_doc_sections)
-
         if not filenames:
             filenames = sorted(self.results.keys())
 
         for fname in filenames:
+            function_table = set()
+
+            if internal or export:
+                if not export_file:
+                    export_file = [fname]
+
+                for f in export_file:
+                    function_table |= self.export_table[f]
+
+            if symbol:
+                for s in symbol:
+                    function_table.add(s)
+
+            self.out_style.set_filter(export, internal, symbol, nosymbol,
+                                      function_table, enable_lineno,
+                                      no_doc_sections)
+
             msg = ""
             for name, arg in self.results[fname]:
                 msg += self.out_msg(fname, name, arg)
@@ -261,12 +280,3 @@ def msg(self, enable_lineno=False, export=False, internal=False,
                                             fname, ln, dtype)
             if msg:
                 yield fname, msg
-
-    @property
-    def errors(self):
-        """
-        Return a count of the number of warnings found, including
-        the ones displayed while interacting over self.msg.
-        """
-
-        return self.config.errors

scripts/lib/kdoc/kdoc_output.py

@@ -69,7 +69,7 @@ def __init__(self):
         self.enable_lineno = None
         self.nosymbol = {}
         self.symbol = None
-        self.function_table = set()
+        self.function_table = None
         self.config = None
         self.no_doc_sections = False
 
@@ -94,10 +94,10 @@ def set_filter(self, export, internal, symbol, nosymbol, function_table,
 
         self.enable_lineno = enable_lineno
         self.no_doc_sections = no_doc_sections
+        self.function_table = function_table
 
         if symbol:
             self.out_mode = self.OUTPUT_INCLUDE
-            function_table = symbol
         elif export:
             self.out_mode = self.OUTPUT_EXPORTED
         elif internal:
@@ -108,8 +108,6 @@ def set_filter(self, export, internal, symbol, nosymbol, function_table,
         if nosymbol:
             self.nosymbol = set(nosymbol)
 
-        if function_table:
-            self.function_table = function_table
 
     def highlight_block(self, block):
         """
@@ -129,8 +127,7 @@ def out_warnings(self, args):
         warnings = args.get('warnings', [])
 
         for log_msg in warnings:
-            self.config.log.warning(log_msg)
-            self.config.errors += 1
+            self.config.warning(log_msg)
 
     def check_doc(self, name, args):
         """Check if DOC should be output"""

scripts/lib/kdoc/kdoc_parser.py

@@ -1133,21 +1133,25 @@ def dump_typedef(self, ln, proto):
         self.emit_warning(ln, "error: Cannot parse typedef!")
 
     @staticmethod
-    def process_export(function_table, line):
+    def process_export(function_set, line):
         """
         process EXPORT_SYMBOL* tags
 
-        This method is called both internally and externally, so, it
-        doesn't use self.
+        This method doesn't use any variable from the class, so declare it
+        with a staticmethod decorator.
         """
 
+        # Note: it accepts only one EXPORT_SYMBOL* per line, as having
+        # multiple export lines would violate Kernel coding style.
+
         if export_symbol.search(line):
             symbol = export_symbol.group(2)
-            function_table.add(symbol)
+            function_set.add(symbol)
+            return
 
         if export_symbol_ns.search(line):
             symbol = export_symbol_ns.group(2)
-            function_table.add(symbol)
+            function_set.add(symbol)
 
     def process_normal(self, ln, line):
         """
@@ -1617,17 +1621,39 @@ def process_docblock(self, ln, line):
         elif doc_content.search(line):
             self.entry.contents += doc_content.group(1) + "\n"
 
-    def run(self):
+    def parse_export(self):
+        """
+        Parses EXPORT_SYMBOL* macros from a single Kernel source file.
+        """
+
+        export_table = set()
+
+        try:
+            with open(self.fname, "r", encoding="utf8",
+                      errors="backslashreplace") as fp:
+
+                for line in fp:
+                    self.process_export(export_table, line)
+
+        except IOError:
+            return None
+
+        return export_table
+
+    def parse_kdoc(self):
         """
         Open and process each line of a C source file.
-        he parsing is controlled via a state machine, and the line is passed
+        The parsing is controlled via a state machine, and the line is passed
         to a different process function depending on the state. The process
         function may update the state as needed.
+
+        Besides parsing kernel-doc tags, it also parses export symbols.
         """
 
         cont = False
         prev = ""
         prev_ln = None
+        export_table = set()
 
         try:
             with open(self.fname, "r", encoding="utf8",
@@ -1659,6 +1685,16 @@ def run(self):
                                           self.st_inline_name[self.inline_doc_state],
                                           line)
 
+                    # This is an optimization over the original script.
+                    # There, when export_file was used for the same file,
+                    # it was read twice. Here, we use the already-existing
+                    # loop to parse exported symbols as well.
+                    #
+                    # TODO: It should be noticed that not all states are
+                    # needed here. On a future cleanup, process export only
+                    # at the states that aren't handling comment markups.
+                    self.process_export(export_table, line)
+
                     # Hand this line to the appropriate state handler
                     if self.state == self.STATE_NORMAL:
                         self.process_normal(ln, line)
@@ -1675,3 +1711,5 @@ def run(self):
                         self.process_docblock(ln, line)
         except OSError:
             self.config.log.error(f"Error: Cannot open file {self.fname}")
+
+        return export_table, self.entries