Create rule coverage functionality

Cargo.toml

@@ -12,6 +12,7 @@ exclude = ["src/main.rs", "docs", "BrailleDocs", "PythonScripts"]    # should ha
 
 
 [features]
+"rule-coverage" = ["dep:serde_json"]
 "include-zip" = []
 "enable-logs" = ["android_logger"]
 "tts" = [ "natural-tts" ]
@@ -38,6 +39,7 @@ cfg-if = "1.0.1"
 fastrand = { version = "2.3.0" }
 clap = { version = "4.5.60", features = ["derive"] }
 html-escape = "0.2.13"
+serde_json = { version = "1.0", optional = true }
 elements_rs = { version = "0.2.8", default-features = false }
 
 # Runtime extracts inner language/braille zips (BZIP2). The package-rules binary also

PythonScripts/README.md

@@ -9,3 +9,12 @@ If you run from the repo root instead of inside `PythonScripts`, point uv at the
 uv sync --project PythonScripts
 uv run --project PythonScripts audit-translations de
 ```
+
+To run rule YAML coverage from the repo root, use the audit tool:
+
+```bash
+uv run --project PythonScripts audit-translations --rule-coverage
+```
+
+The tool opens `target/rule-coverage/index.html` in a browser when the run finishes. Expand a YAML file to see each active rule's name, tag, hit count, and match status. Hover over a rule to see which tests hit it. Coverage fractions include percentages.
+The Rust test output remains at `target/rule-coverage/test.log`.

PythonScripts/audit_translations/README.md

@@ -67,11 +67,13 @@ The tool automatically adjusts its matching logic based on the file type:
 uv run audit-translations <language> [--file <specific_file>]
 uv run audit-translations <language> --source <source-language>
 uv run audit-translations --list
+uv run audit-translations --rule-coverage
 
 # If running from the repo root, point uv at the project:
 uv run --project PythonScripts audit-translations <language>
 uv run --project PythonScripts audit-translations <language> --source <source-language>
 uv run --project PythonScripts audit-translations --list
+uv run --project PythonScripts audit-translations --rule-coverage
 ```
 
 **Convenience Features:**
@@ -82,6 +84,7 @@ uv run --project PythonScripts audit-translations --list
 * `--rules-dir`: Override the Rules/Languages directory path.
 * `--only`: Filter issue types (comma-separated): `missing`, `untranslated`, `extra`, `diffs`, `all`.
 * `--verbose`: Show detailed output including source/target snippets for rule differences.
+* `--rule-coverage`: Run the full Rust test suite, generate file- and rule-level pattern coverage under `target/rule-coverage/`, and open the HTML report. File coverage includes files with active rules; empty and include-only files are not separate coverage targets. Expand a file to inspect matched and unmatched rules by name and tag, their hit counts, and hover over a rule to see the tests that matched it. Coverage fractions include percentages. This runs independently of a language audit.
 * **Summary Stats:** Provides a statistical summary after every run.
 
 **Examples:**

PythonScripts/audit_translations/cli.py

@@ -10,28 +10,33 @@
 from .auditor import audit_language, list_languages
 from .errors import AuditError
 from .renderer import console
+from .rulecoverage.rule_coverage import run as run_rule_coverage
 
 
 def main() -> None:
     """Main entry point for the audit tool"""
     sys.stdout.reconfigure(encoding="utf-8")
 
     parser = argparse.ArgumentParser(
-        description="Audit MathCAT translation files against a source language",
+        description="Audit MathCAT translations or report rule YAML test coverage",
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog="""
 Examples:
     uv run audit-translations es
     uv run audit-translations nb --source sv
     uv run audit-translations de --file SharedRules/default.yaml
     uv run audit-translations --list
+    uv run audit-translations --rule-coverage
         """,
     )
 
     parser.add_argument("language", nargs="?", help="Language code to audit (e.g., 'es', 'de', 'fi')")
     parser.add_argument("--source", default="en", help="Source/reference language code (default: 'en')")
     parser.add_argument("--file", dest="specific_file", help="Audit only a specific file (e.g., 'SharedRules/default.yaml')")
     parser.add_argument("--list", action="store_true", help="List available languages")
+    parser.add_argument(
+        "--rule-coverage", action="store_true", help="Run the full Rust tests and open the rule YAML coverage report"
+    )
     parser.add_argument("--rules-dir", help="Override Rules/Languages directory path")
     parser.add_argument(
         "--only",
@@ -45,7 +50,11 @@ def main() -> None:
 
     args = parser.parse_args()
 
-    if args.list:
+    if args.rule_coverage:
+        if args.language or args.list or args.specific_file or args.rules_dir or args.only or args.verbose or args.source != "en":
+            parser.error("--rule-coverage cannot be combined with translation audit options")
+        sys.exit(run_rule_coverage())
+    elif args.list:
         list_languages(args.rules_dir)
     elif not args.language:
         parser.print_help()

PythonScripts/audit_translations/rulecoverage/__init__.py

@@ -0,0 +1 @@
+"""Rule coverage collection and report generation."""

PythonScripts/audit_translations/rulecoverage/rule_coverage.html

@@ -0,0 +1,93 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Rule YAML coverage</title>
+<style>
+body { max-width: 75rem; margin: 2rem auto; padding: 0 1rem; font: 1rem/1.5 system-ui, sans-serif; color: #17212b; }
+h1, h2 { line-height: 1.2; }
+.summary { display: flex; flex-wrap: wrap; gap: .5rem 2rem; }
+.problems { padding: .5rem 1rem; background: #fff0ed; border-left: .25rem solid #b42318; }
+.controls { display: flex; flex-wrap: wrap; gap: .75rem; margin: 1.5rem 0; }
+input, select { font: inherit; padding: .35rem .5rem; }
+input { flex: 1 1 20rem; }
+.file { border: 1px solid #cad4dc; border-radius: .4rem; margin: .5rem 0; padding: .6rem 1rem; }
+summary { cursor: pointer; }
+.count { float: right; color: #52616d; }
+.badge { display: inline-block; border-radius: .3rem; padding: .05rem .4rem; font-size: .85rem; font-weight: 600; }
+.badge.matched { color: #155c36; background: #dcf5e5; }
+.badge.unmatched { color: #8b3d00; background: #fff0cf; }
+.rules { list-style: none; padding-left: 1rem; }
+.rules li { position: relative; padding: .2rem 0; }
+.rule-tooltip { display: none; position: absolute; z-index: 1; top: 100%; left: 0;
+  max-width: min(35rem, 90vw); max-height: 60vh; overflow: auto; padding: .5rem .7rem;
+  border: 1px solid #8396a5; border-radius: .3rem; background: #fff; color: #17212b;
+  box-shadow: 0 .2rem .6rem #0003; white-space: pre-line; overflow-wrap: anywhere; }
+.rules li:hover .rule-tooltip, .rules li:focus .rule-tooltip { display: block; }
+.rule-name { font-weight: 600; }
+[hidden] { display: none !important; }
+</style>
+</head>
+<body>
+<h1>Rule YAML coverage</h1>
+<p>Status: <strong>{{ status }}</strong></p>
+<div class="summary"><span>{{ rule_file_count }} files with active rules</span><span>{{ file_coverage }} rule files matched</span>
+<span>{{ rule_coverage }} active rules matched</span></div>
+<p>Paths are relative to <code>Rules/</code>. Files with active rules are coverage targets;
+a file is matched when a rule from it completes its replacement successfully.</p>
+{% if errors %}
+<section class="problems"><h2>Problems</h2><ul>
+{% for error in errors %}<li>{{ error }}</li>{% endfor %}
+</ul></section>
+{% endif %}
+<div class="controls"><input id="search" type="search" aria-label="Search files and rules"
+placeholder="Search paths, rule names, or tags">
+<select id="status" aria-label="Filter by coverage"><option value="all">All</option>
+<option value="matched">Matched</option><option value="unmatched">No match</option></select></div>
+<div id="files">
+{% for file in files %}
+<details class="file" data-file-search="{{ file.path }}" data-status="{{ 'matched' if file.matched else 'unmatched' }}">
+<summary><span class="badge {{ 'matched' if file.matched else 'unmatched' }}">{{ 'Matched' if file.matched else 'No match' }}</span>
+<code>{{ file.path }}</code> <span class="count">{{ file.coverage }} rules</span></summary>
+<p>{{ file.coverage }} active rules matched</p><ul class="rules">
+{% set file_index = loop.index0 %}
+{% for rule in file.rules %}
+<li data-rule-search="{{ rule.name }} {{ rule.tag }}" data-status="{{ 'matched' if rule.matched else 'unmatched' }}" tabindex="0" aria-describedby="rule-tooltip-{{ file_index }}-{{ loop.index0 }}">
+<span class="badge {{ 'matched' if rule.matched else 'unmatched' }}">{{ 'Matched' if rule.matched else 'No match' }}</span>
+<span class="rule-name">{{ rule.name }}</span> <code>{{ rule.tag }}</code> <span class="count">{{ rule.hits }} {{ 'hit' if rule.hits == 1 else 'hits' }}</span>
+<span class="rule-tooltip" id="rule-tooltip-{{ file_index }}-{{ loop.index0 }}" role="tooltip">{{ rule.tooltip }}</span></li>
+{% endfor %}
+</ul>
+</details>
+{% endfor %}
+</div>
+<p id="no-results" hidden>No matching files or rules</p>
+<script>
+const search = document.getElementById('search');
+const status = document.getElementById('status');
+const files = [...document.querySelectorAll('.file')];
+function filter() {
+  const query = search.value.trim().toLocaleLowerCase();
+  let visible = 0;
+  for (const file of files) {
+    const pathMatches = file.dataset.fileSearch.toLocaleLowerCase().includes(query);
+    let visibleRules = 0;
+    for (const rule of file.querySelectorAll('[data-rule-search]')) {
+      const show = (status.value === 'all' || rule.dataset.status === status.value)
+        && (pathMatches || rule.dataset.ruleSearch.toLocaleLowerCase().includes(query));
+      rule.hidden = !show;
+      if (show) visibleRules++;
+    }
+    const fileStatusMatches = status.value === 'all' || file.dataset.status === status.value;
+    const showFile = (pathMatches && fileStatusMatches) || visibleRules > 0;
+    file.hidden = !showFile;
+    if (showFile) visible++;
+  }
+  document.getElementById('no-results').hidden = visible > 0;
+}
+search.addEventListener('input', filter);
+status.addEventListener('change', filter);
+</script>
+</body>
+</html>

PythonScripts/audit_translations/rulecoverage/rule_coverage.py

@@ -0,0 +1,116 @@
+"""Run the full Rust test suite and report which rule YAML files it exercises."""
+
+import json
+import shutil
+import subprocess
+import sys
+import webbrowser
+from collections import Counter, defaultdict
+from pathlib import Path, PurePosixPath
+
+from .rule_coverage_report import RuleKey, coverage, render_html
+
+ROOT = Path(__file__).resolve().parents[3]
+OUTPUT = ROOT / "target" / "rule-coverage"
+EVENTS = OUTPUT / "events"
+
+
+def read_events() -> tuple[set[RuleKey], dict[RuleKey, Counter[str]], list[str]]:
+    defined_rules: set[RuleKey] = set()
+    hits_by_test: dict[RuleKey, Counter[str]] = defaultdict(Counter)
+    errors: list[str] = []
+    for event_file in sorted(EVENTS.glob("*.jsonl")):
+        for number, line in enumerate(event_file.read_text(encoding="utf-8").splitlines(), 1):
+            try:
+                event = json.loads(line)
+            except json.JSONDecodeError:
+                errors.append(f"Invalid JSON in {event_file.name}:{number}")
+                continue
+            if not isinstance(event, dict):
+                errors.append(f"Invalid event in {event_file.name}:{number}")
+                continue
+            kind = event.get("kind")
+            path = event.get("path")
+            if not isinstance(path, str):
+                errors.append(f"Invalid event in {event_file.name}:{number}")
+                continue
+            path = path.replace("\\", "/")
+            parts = PurePosixPath(path).parts
+            if (
+                not parts
+                or PurePosixPath(path).is_absolute()
+                or ".." in parts
+                or PurePosixPath(path).suffix not in (".yaml", ".yml")
+            ):
+                errors.append(f"Invalid event in {event_file.name}:{number}")
+            elif kind in ("defined-rule", "matched-rule") and {"name", "tag"} <= event.keys():
+                name, tag = event["name"], event["tag"]
+                if not isinstance(name, str) or not isinstance(tag, str) or not name or not tag:
+                    errors.append(f"Empty rule identity in {event_file.name}:{number}")
+                elif kind == "defined-rule" and event.keys() == {"kind", "path", "name", "tag"}:
+                    defined_rules.add((path, name, tag))
+                elif (
+                    kind == "matched-rule"
+                    and event.keys() == {"kind", "path", "name", "tag", "test"}
+                    and isinstance(event["test"], str)
+                    and event["test"]
+                ):
+                    key = (path, name, tag)
+                    hits_by_test[key][event["test"]] += 1
+                else:
+                    errors.append(f"Invalid event in {event_file.name}:{number}")
+            else:
+                errors.append(f"Invalid event in {event_file.name}:{number}")
+    return defined_rules, dict(hits_by_test), errors
+
+
+def run() -> int:
+    """Generate the HTML report and open it when the run finishes."""
+    OUTPUT.mkdir(parents=True, exist_ok=True)
+    (OUTPUT / "report.md").unlink(missing_ok=True)
+    if EVENTS.exists():
+        shutil.rmtree(EVENTS)
+    EVENTS.mkdir()
+
+    command = ["cargo", "test", "--features", "rule-coverage"]
+    log_path = OUTPUT / "test.log"
+    print(f"Running {' '.join(command)}; saving output to {log_path.relative_to(ROOT)}", flush=True)
+    with log_path.open("w", encoding="utf-8") as log:
+        try:
+            result = subprocess.run(command, cwd=ROOT, stdout=log, stderr=subprocess.STDOUT, check=False)
+            test_status = result.returncode
+        except OSError as error:
+            log.write(f"Could not run cargo: {error}\n")
+            test_status = 1
+
+    defined_rules, hits_by_test, errors = read_events()
+    matched_rules = set(hits_by_test)
+    rule_files = {path for path, _, _ in defined_rules}
+    matched_files = {path for path, _, _ in matched_rules}
+    if test_status:
+        errors.insert(0, f"cargo test failed (exit status {test_status}); see test.log")
+    if not defined_rules:
+        errors.append("No active rule definitions found")
+    if not matched_rules:
+        errors.append("No matched rule events found")
+    if matched_rules - defined_rules:
+        errors.append("Matched rules lack definition events")
+
+    status = "Incomplete" if errors else "Complete"
+    html_path = OUTPUT / "index.html"
+    html_path.write_text(render_html(defined_rules, hits_by_test, errors), encoding="utf-8")
+    print(
+        f"{status}: {coverage(len(matched_files), len(rule_files))} rule files matched; "
+        f"{coverage(len(matched_rules), len(defined_rules))} rules matched; report: {html_path}"
+    )
+    try:
+        opened = webbrowser.open(html_path.resolve().as_uri())
+    except OSError, webbrowser.Error:
+        opened = False
+    if not opened:
+        print(f"Browser unavailable; open {html_path} manually")
+    return 1 if errors else 0
+
+
+if __name__ == "__main__":
+    sys.exit(run())