feat: Configure dvsim and FuseSoC from a config file outside the repository (or the command line)

src/dvsim/cli/run.py

@@ -31,7 +31,15 @@
 from importlib.metadata import version
 from pathlib import Path
 
+from dvsim.config import (
+    check_top_level_keys,
+    find_config_file,
+    load_config_file,
+    proj_root_from_config,
+)
 from dvsim.flow.factory import make_cfg
+from dvsim.fusesoc import CONFIG_SECTION as FUSESOC_CONFIG_SECTION
+from dvsim.fusesoc import resolve_options as resolve_fusesoc_options
 from dvsim.instrumentation.factory import InstrumentationFactory
 from dvsim.instrumentation.runtime import set_instrumentation
 from dvsim.job.deploy import RunTest
@@ -174,12 +182,14 @@ def parse_resource(s: str) -> tuple[str, int | None]:
         raise argparse.ArgumentTypeError(msg) from e
 
 
-def resolve_branch(branch):
+def resolve_branch(branch, cwd=None):
     """Choose a branch name for output files.
 
     If the --branch argument was passed on the command line, the branch
-    argument is the branch name to use. Otherwise it is None and we use git to
-    find the name of the current branch in the working directory.
+    argument is the branch name to use.  Otherwise it is None and we use git to
+    find the name of the current branch in `cwd`, which is the project root
+    rather than the working directory: dvsim may be invoked from outside the
+    project, where there is no repository to read.
 
     Note, as this name will be used to generate output files any forward slashes
     are replaced with single dashes to avoid being interpreted as directory hierarchy.
@@ -190,6 +200,8 @@ def resolve_branch(branch):
     result = subprocess.run(
         ["git", "rev-parse", "--abbrev-ref", "HEAD"],
         stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        cwd=cwd,
         check=False,
     )
     branch = result.stdout.decode("utf-8").strip().replace("/", "-")
@@ -216,6 +228,33 @@ def get_proj_root():
     return proj_root
 
 
+def find_proj_root(args) -> str:
+    """Choose the project root.
+
+    In order of precedence: the --proj-root argument, relative to the working
+    directory; then `proj_root` from dvsim's config file, relative to the
+    directory holding that file; then the git repository containing the working
+    directory.
+
+    Both settings may equally be given as absolute paths.  Resolving them here
+    is what allows dvsim to be invoked from outside the project, since the value
+    is handed on to flows (and from there to tools such as FuseSoC) that run
+    with a different working directory.
+    """
+    if args.proj_root:
+        return str(Path(args.proj_root).expanduser().resolve())
+
+    from_config = (
+        proj_root_from_config(args.dvsim_config_data, args.dvsim_config_path)
+        if args.dvsim_config_path is not None
+        else None
+    )
+    if from_config is not None:
+        return str(from_config)
+
+    return get_proj_root()
+
+
 def resolve_proj_root(args):
     """Update proj_root based on how DVSim is invoked.
 
@@ -228,7 +267,8 @@ def resolve_proj_root(args):
     --remote switch is not set, the destination path is identical to the src
     path. Likewise, if --dry-run is set.
     """
-    proj_root_src = args.proj_root or get_proj_root()
+    proj_root_src = find_proj_root(args)
+    args.branch = resolve_branch(args.branch, proj_root_src)
 
     # Check if jobs are dispatched to external compute machines. If yes,
     # then the repo needs to be copied over to the scratch area
@@ -489,9 +529,11 @@ def parse_args(argv: list[str] | None = None):
         "-pr",
         metavar="PATH",
         help=(
-            "The root directory of the project. If not "
-            "specified, dvsim will search for a git "
-            "repository containing the current directory."
+            "The root directory of the project, relative to the current "
+            "directory unless absolute. If not specified, dvsim uses "
+            "`proj_root` from its config file, resolved relative to that "
+            "file; failing that, it searches for a git repository containing "
+            "the current directory."
         ),
     )
 
@@ -527,6 +569,33 @@ def parse_args(argv: list[str] | None = None):
         help="Clean the scratch directory before running.",
     )
 
+    fusesocg = parser.add_argument_group("FuseSoC integration")
+
+    fusesocg.add_argument(
+        "--fusesoc-mapping",
+        action="append",
+        default=[],
+        metavar="[OLD=]NEW",
+        help=(
+            "Select a FuseSoC mapping. OLD=NEW replaces an existing "
+            "--mapping=OLD in the generated FuseSoC command line; a bare NEW "
+            "appends a mapping. Repeatable. Also settable from the config "
+            "file, whose values are applied first."
+        ),
+    )
+
+    fusesocg.add_argument(
+        "--fusesoc-extra-cores-root",
+        action="append",
+        default=[],
+        metavar="PATH",
+        help=(
+            "Add a --cores-root to the generated FuseSoC command line, so that "
+            "cores outside the project tree are discoverable. Repeatable. Also "
+            "settable from the config file."
+        ),
+    )
+
     buildg = parser.add_argument_group("Options for building")
 
     buildg.add_argument(
@@ -828,6 +897,18 @@ def parse_args(argv: list[str] | None = None):
 
     dvg = parser.add_argument_group("Controlling DVSim itself")
 
+    dvg.add_argument(
+        "--dvsim-config",
+        metavar="FILE",
+        help=(
+            "Path to dvsim's own config file. If not specified, the nearest "
+            "dvsim.hjson found by walking up from the current directory is "
+            "used, else $XDG_CONFIG_HOME/lowRISC/dvsim/dvsim.hjson. This "
+            "configures the tool, and is distinct from the flow config file "
+            "passed as the positional argument."
+        ),
+    )
+
     dvg.add_argument(
         "--instrument",
         dest="instrumentation",
@@ -944,7 +1025,24 @@ def main(argv: list[str] | None = None) -> None:
         log.fatal("Path to config file %s appears to be invalid.", args.cfg)
         sys.exit(1)
 
-    args.branch = resolve_branch(args.branch)
+    # Load dvsim's own config file once, before anything reads it.  The keys it
+    # may define are listed here because this is the only place that knows about
+    # every feature that owns a section.
+    args.dvsim_config_path = find_config_file(args.dvsim_config)
+    args.dvsim_config_data = (
+        load_config_file(args.dvsim_config_path) if args.dvsim_config_path is not None else {}
+    )
+    if args.dvsim_config_path is not None:
+        check_top_level_keys(
+            args.dvsim_config_data,
+            frozenset({"proj_root", FUSESOC_CONFIG_SECTION}),
+            args.dvsim_config_path,
+        )
+        log.info("[dvsim_config]: %s", args.dvsim_config_path)
+
+    # proj_root is resolved before the branch, because the branch is read from
+    # the git repository at proj_root rather than from the working directory:
+    # dvsim may be invoked from outside the project entirely.
     proj_root_src, proj_root = resolve_proj_root(args)
     args.scratch_root = resolve_scratch_root(args.scratch_root, proj_root)
     log.info("[proj_root]: %s", proj_root)
@@ -980,6 +1078,13 @@ def main(argv: list[str] | None = None) -> None:
     Launcher.max_odirs = args.max_odirs
     RuntimeBackend.max_output_dirs = args.max_odirs
 
+    # Resolve the FuseSoC options once, from the config file and the command
+    # line, and stash them on args so that every FlowCfg -- including the
+    # children of a primary cfg -- sees the same set.
+    args.resolved_fusesoc_options = resolve_fusesoc_options(
+        args, args.dvsim_config_data, args.dvsim_config_path
+    )
+
     # Configure the runtime backend.
     set_backend_type(is_local=args.local, fake=args.fake)
 

src/dvsim/config.py

@@ -0,0 +1,146 @@
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+"""DVSim's own configuration file.
+
+This is distinct from the flow configs dvsim takes as its positional argument:
+it configures the tool rather than describing a flow, and it is meant to be
+checked in next to a project so that a workspace's settings do not have to be
+repeated on every command line.
+
+The file is hjson, the same format as the flow configs. Each feature owns a
+section of it; see e.g. :mod:`dvsim.fusesoc` for the ``fusesoc`` section.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import hjson
+
+__all__ = (
+    "CONFIG_BASENAME",
+    "XDG_SUBPATH",
+    "as_str_list",
+    "check_top_level_keys",
+    "find_config_file",
+    "load_config_file",
+    "proj_root_from_config",
+    "read_section",
+    "resolve_path",
+)
+
+#: Name of the discoverable config file. Deliberately not hidden.
+CONFIG_BASENAME = "dvsim.hjson"
+
+#: Location of the per-user config file, under $XDG_CONFIG_HOME.
+XDG_SUBPATH = Path("lowRISC") / "dvsim" / CONFIG_BASENAME
+
+
+def find_config_file(explicit: str | None, start_dir: Path | None = None) -> Path | None:
+    """Locate the config file.
+
+    Search order:
+
+    1. ``explicit``, if given (an error if it does not exist),
+    2. the nearest :data:`CONFIG_BASENAME` walking upwards from ``start_dir``,
+    3. ``$XDG_CONFIG_HOME/lowRISC/dvsim/dvsim.hjson``.
+
+    Walking upwards matters because dvsim is normally invoked from inside the
+    project it is building, while a config file describing a workspace
+    naturally lives alongside that project rather than inside it.
+    """
+    if explicit is not None:
+        path = Path(explicit).expanduser()
+        if not path.is_file():
+            msg = f"dvsim config file not found: {path}"
+            raise FileNotFoundError(msg)
+        return path
+
+    start = (start_dir or Path.cwd()).resolve()
+    for directory in (start, *start.parents):
+        candidate = directory / CONFIG_BASENAME
+        if candidate.is_file():
+            return candidate
+
+    xdg = Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config")
+    candidate = xdg.expanduser() / XDG_SUBPATH
+    return candidate if candidate.is_file() else None
+
+
+def load_config_file(path: Path) -> dict:
+    """Parse a config file, returning its top-level dict."""
+    try:
+        data = hjson.loads(path.read_text())
+    except Exception as e:
+        msg = f"Failed to parse dvsim config file {path}: {e}"
+        raise RuntimeError(msg) from e
+
+    if not isinstance(data, dict):
+        msg = f"dvsim config file {path} must contain a dict at the top level"
+        raise RuntimeError(msg)
+
+    return data
+
+
+def read_section(data: dict, name: str, known_keys: frozenset[str], path: Path) -> dict:
+    """Return one section of a config file, rejecting unknown keys.
+
+    Rejecting rather than ignoring them turns a typo into an error at the point
+    it is made, instead of a setting that silently does nothing.
+    """
+    section = data.get(name, {})
+    if not isinstance(section, dict):
+        msg = f"dvsim config file {path}: '{name}' must be a dict"
+        raise RuntimeError(msg)
+
+    unknown = set(section) - known_keys
+    if unknown:
+        msg = f"dvsim config file {path}: unknown key(s) in '{name}': {sorted(unknown)}"
+        raise RuntimeError(msg)
+
+    return section
+
+
+def as_str_list(section: dict, key: str, name: str, path: Path) -> list[str]:
+    """Read a section key that may be given as a string or a list of strings."""
+    value = section.get(key, [])
+    if isinstance(value, str):
+        return [value]
+    if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
+        msg = f"dvsim config file {path}: '{name}.{key}' must be a string or list of strings"
+        raise RuntimeError(msg)
+    return list(value)
+
+
+def resolve_path(value: str, base: Path) -> Path:
+    """Resolve a config-file path value against the config file's own directory.
+
+    Absolute values are left alone.  Relative ones are taken to be relative to
+    the directory holding the config file rather than to the working directory,
+    so that a checked-in file means the same thing wherever dvsim is invoked
+    from.
+    """
+    path = Path(value).expanduser()
+    return path if path.is_absolute() else (base / path).resolve()
+
+
+def check_top_level_keys(data: dict, known_keys: frozenset[str], path: Path) -> None:
+    """Reject unknown top-level keys, so that a typo fails rather than doing nothing."""
+    unknown = set(data) - known_keys
+    if unknown:
+        msg = f"dvsim config file {path}: unknown top-level key(s): {sorted(unknown)}"
+        raise RuntimeError(msg)
+
+
+def proj_root_from_config(data: dict, path: Path) -> Path | None:
+    """Read the top-level ``proj_root``, resolved against the config file's directory."""
+    value = data.get("proj_root")
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        msg = f"dvsim config file {path}: 'proj_root' must be a string"
+        raise RuntimeError(msg)
+    return resolve_path(value, path.parent)

src/dvsim/flow/base.py

@@ -18,6 +18,7 @@
 
 import dvsim.instrumentation.runtime as instrumentation
 from dvsim.flow.hjson import set_target_attribute
+from dvsim.fusesoc import rewrite_fusesoc_opts
 from dvsim.job.data import CompletedJobStatus, JobSpec, WorkspaceConfig
 from dvsim.job.status import JobStatus
 from dvsim.logging import log
@@ -225,6 +226,13 @@ def _expand(self) -> None:
             ignore_error=partial,
         )
 
+        # Now that the wildcards are substituted, the FuseSoC command lines hold
+        # literal arguments and can be rewritten.  This has to happen at the end
+        # of _expand() rather than after it, because subclasses call
+        # _create_objects() from their own _expand(), and the objects they
+        # create take a copy of these option lists.
+        self._apply_fusesoc_opts()
+
     def _post_init(self) -> None:
         # Run some post init checks
         if not self.is_primary_cfg:  # noqa: SIM102
@@ -319,6 +327,34 @@ def _conv_inline_cfg_to_hjson(self, idict: Mapping) -> str | None:
         # Return the temp cfg file created
         return temp_cfg_file
 
+    def _apply_fusesoc_opts(self) -> None:
+        """Rewrite the FuseSoC command lines this cfg will run.
+
+        Only option lists whose command is actually FuseSoC are touched, so the
+        --fusesoc-* arguments are inert for flows driven by something else.
+        """
+        options = getattr(self.args, "resolved_fusesoc_options", None)
+        if not options:
+            return
+
+        for cmd_attr, opts_attr in (
+            ("build_cmd", "build_opts"),
+            ("sv_flist_gen_cmd", "sv_flist_gen_opts"),
+        ):
+            opts = getattr(self, opts_attr, None)
+            if not opts:
+                continue
+
+            cmd = str(getattr(self, cmd_attr, "") or "")
+            # The command may carry a prefix, e.g. "{job_prefix} fusesoc".
+            if not any(Path(word).name == "fusesoc" for word in cmd.split()):
+                continue
+
+            # Name the config these arguments belong to, so that the overrides
+            # logged from dvsim.fusesoc can be traced back to it.
+            context = f"{getattr(self, 'name', '?')} {opts_attr} [{self.flow_cfg_file}]"
+            setattr(self, opts_attr, rewrite_fusesoc_opts(opts, options, context))
+
     def _process_overrides(self) -> None:
         # Look through the dict and find available overrides.
         # If override is available, check if the type of the value for existing