From bcd42acefff5022d9f7770ed7501ab5eef8afa43 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 9 Oct 2026 13:24:04 -0500 Subject: [PATCH] Support Lunaris provider directories in OGM harvesting --- .../services/ogm_harvest/aardvark_reader.py | 17 +++++-- backend/app/services/ogm_harvest/layout.py | 9 ++++ backend/scripts/populate_ogm_repos.py | 12 +++-- .../tests/scripts/test_populate_ogm_repos.py | 31 ++++++++++++ .../tests/services/test_aardvark_reader.py | 48 +++++++++++++++++++ docs/harvester.md | 7 +++ 6 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 backend/app/services/ogm_harvest/layout.py create mode 100644 backend/tests/services/test_aardvark_reader.py diff --git a/backend/app/services/ogm_harvest/aardvark_reader.py b/backend/app/services/ogm_harvest/aardvark_reader.py index d53bcd0..6a66e2b 100644 --- a/backend/app/services/ogm_harvest/aardvark_reader.py +++ b/backend/app/services/ogm_harvest/aardvark_reader.py @@ -5,6 +5,8 @@ from pathlib import Path from typing import Dict, Generator, Iterable +from app.services.ogm_harvest.layout import metadata_root + @dataclass(frozen=True) class AardvarkRecordRef: @@ -13,11 +15,17 @@ class AardvarkRecordRef: def iter_aardvark_json_files(repo_dir: Path) -> Iterable[Path]: - """Yield Aardvark JSON files from metadata-aardvark directories.""" - base = repo_dir / "metadata-aardvark" + """Yield JSON candidates from the repository's supported metadata root.""" + base = repo_dir / metadata_root(repo_dir.name) if not base.exists(): return [] - return (p for p in base.rglob("*.json") if p.name != "layers.json") + return ( + p + for p in base.rglob("*.json") + if p.is_file() + and p.name != "layers.json" + and not any(part.startswith(".") for part in p.relative_to(base).parts) + ) def read_aardvark_records(repo_dir: Path) -> Generator[AardvarkRecordRef, None, None]: @@ -45,6 +53,9 @@ def read_aardvark_records(repo_dir: Path) -> Generator[AardvarkRecordRef, None, rel = str(path.relative_to(repo_dir)) for record in records: + # Root layouts can contain unrelated JSON or older schema exports. + if metadata_root(repo_dir.name) == "." and record.get("gbl_mdVersion_s") != "Aardvark": + continue yield AardvarkRecordRef(record=record, source_path=rel) diff --git a/backend/app/services/ogm_harvest/layout.py b/backend/app/services/ogm_harvest/layout.py new file mode 100644 index 0000000..d024b01 --- /dev/null +++ b/backend/app/services/ogm_harvest/layout.py @@ -0,0 +1,9 @@ +"""Metadata locations shared by GitHub discovery and checkout harvesting.""" + + +def metadata_root(repo_name: str) -> str: + # Lunaris publishes Aardvark JSON directly under provider directories. + # Keep this explicit: scanning arbitrary utility repos would ingest fixtures. + if repo_name == "ca.lunaris": + return "." + return "metadata-aardvark" diff --git a/backend/scripts/populate_ogm_repos.py b/backend/scripts/populate_ogm_repos.py index 3a3ca3d..fb86c83 100644 --- a/backend/scripts/populate_ogm_repos.py +++ b/backend/scripts/populate_ogm_repos.py @@ -6,7 +6,7 @@ This script: - lists repos in a GitHub org via the REST API -- checks whether each repo has a top-level `metadata-aardvark/` directory +- checks whether each repo has its supported metadata directory - upserts into Postgres table `ogm_repos` - flags repos missing aardvark via `ogm_tags["ogm_missing_aardvark"] = true` @@ -29,6 +29,8 @@ from sqlalchemy import create_engine from sqlalchemy.dialects.postgresql import insert as pg_insert +from app.services.ogm_harvest.layout import metadata_root + # Keep script self-contained: import the SQLAlchemy Table definitions. from db.models import ogm_repos @@ -119,7 +121,9 @@ def repo_has_metadata_aardvark( org: str, repo_name: str, default_branch: Optional[str], token: Optional[str] ) -> bool: # GitHub contents API for a directory returns a JSON array (200) if present, 404 if missing. - url = f"https://api.github.com/repos/{org}/{repo_name}/contents/metadata-aardvark" + root = metadata_root(repo_name) + suffix = "" if root == "." else f"/{root}" + url = f"https://api.github.com/repos/{org}/{repo_name}/contents{suffix}" params = {} if default_branch: params["ref"] = default_branch @@ -131,7 +135,7 @@ def repo_has_metadata_aardvark( return False # Treat other errors as non-fatal but recordable upstream; caller can decide. raise RuntimeError( - f"GitHub API error checking metadata-aardvark for {org}/{repo_name}: " + f"GitHub API error checking metadata root {root} for {org}/{repo_name}: " f"{resp.status_code} {resp.text[:500]}" ) @@ -155,7 +159,7 @@ def build_repo_row(repo: Dict[str, Any], *, has_aardvark: bool) -> Dict[str, Any } # Policy: - # - if no aardvark directory, disable by default (so it won't be harvested) + # - if no supported metadata directory, disable by default # - otherwise enable and default to both webhook + nightly reconciliation # (can be edited via admin endpoint) ogm_enabled = bool(has_aardvark and not archived) diff --git a/backend/tests/scripts/test_populate_ogm_repos.py b/backend/tests/scripts/test_populate_ogm_repos.py index dce38e8..adf054e 100644 --- a/backend/tests/scripts/test_populate_ogm_repos.py +++ b/backend/tests/scripts/test_populate_ogm_repos.py @@ -20,6 +20,37 @@ def setup_function(): populate_ogm_repos._REJECTED_GITHUB_TOKENS.clear() +@pytest.mark.parametrize("name,suffix", [("ca.lunaris", ""), ("edu.example", "/metadata-aardvark")]) +def test_discovery_uses_supported_metadata_root(monkeypatch, name, suffix): + def fake_get(url, token, params, timeout): + assert url == f"https://api.github.com/repos/OpenGeoMetadata/{name}/contents{suffix}" + assert params == {"ref": "master"} + return FakeResponse(200, [{"name": "provider", "type": "dir"}]) + + monkeypatch.setattr(populate_ogm_repos, "_github_get", fake_get) + has_aardvark = populate_ogm_repos.repo_has_metadata_aardvark( + "OpenGeoMetadata", name, "master", None + ) + row = populate_ogm_repos.build_repo_row( + {"name": name, "archived": False}, has_aardvark=has_aardvark + ) + assert row["ogm_enabled"] is True + assert row["ogm_watch_mode"] == "both" + + +def test_missing_standard_directory_remains_disabled(monkeypatch): + monkeypatch.setattr(populate_ogm_repos, "_github_get", lambda *a, **kw: FakeResponse(404, {})) + has_aardvark = populate_ogm_repos.repo_has_metadata_aardvark( + "OpenGeoMetadata", "utility", "main", None + ) + assert ( + populate_ogm_repos.build_repo_row({"name": "utility"}, has_aardvark=has_aardvark)[ + "ogm_enabled" + ] + is False + ) + + def test_list_org_repos_retries_without_rejected_token(monkeypatch, capsys): responses = [ FakeResponse(401, {"message": "Bad credentials"}), diff --git a/backend/tests/services/test_aardvark_reader.py b/backend/tests/services/test_aardvark_reader.py new file mode 100644 index 0000000..a47e555 --- /dev/null +++ b/backend/tests/services/test_aardvark_reader.py @@ -0,0 +1,48 @@ +import json + +from app.services.ogm_harvest.aardvark_reader import read_aardvark_records + + +def write_json(repo, path, value): + target = repo / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(value)) + + +def test_lunaris_provider_layout_filters_records_and_preserves_provenance(tmp_path): + repo = tmp_path / "ca.lunaris" + record = {"id": "transit", "gbl_mdVersion_s": "Aardvark"} + write_json(repo, "abacus-open-data/transit.json", record) + write_json( + repo, + "another-provider/nested/records.json", + [ + {"id": "second", "gbl_mdVersion_s": "Aardvark"}, + {"id": "legacy", "geoblacklight_version": "1.0"}, + "not a record", + ], + ) + write_json(repo, "package.json", {"name": "unrelated"}) + write_json(repo, ".git/hidden.json", record) + write_json(repo, "abacus-open-data/layers.json", record) + (repo / "broken.json").write_text("not json") + + refs = sorted(read_aardvark_records(repo), key=lambda ref: ref.record["id"]) + assert [ref.record["id"] for ref in refs] == ["second", "transit"] + assert refs[1].source_path == "abacus-open-data/transit.json" + + +def test_standard_layout_keeps_existing_behavior(tmp_path): + repo = tmp_path / "edu.example" + record = {"id": "existing"} + write_json(repo, "metadata-aardvark/nested/record.json", record) + write_json(repo, "fixtures/unrelated.json", {"id": "fixture", "gbl_mdVersion_s": "Aardvark"}) + refs = list(read_aardvark_records(repo)) + assert [ref.record for ref in refs] == [record] + assert refs[0].source_path == "metadata-aardvark/nested/record.json" + + +def test_unknown_repo_does_not_fall_back_to_root(tmp_path): + repo = tmp_path / "utility" + write_json(repo, "fixtures/record.json", {"id": "fixture", "gbl_mdVersion_s": "Aardvark"}) + assert list(read_aardvark_records(repo)) == [] diff --git a/docs/harvester.md b/docs/harvester.md index 980d7e0..618d633 100644 --- a/docs/harvester.md +++ b/docs/harvester.md @@ -64,6 +64,13 @@ The harvester denylist automatically excludes non-metadata repositories such as ## Related Backend Flows +Application discovery and harvesting share explicit metadata locations. Most +repositories use `metadata-aardvark/`; `ca.lunaris` uses the repository root with +records nested under provider directories. For that root layout, the reader +accepts only records declaring `gbl_mdVersion_s: Aardvark` and skips hidden paths +and `layers.json`. After deploying this support, the next repository discovery +refresh enables Lunaris and the nightly pipeline harvests its records. + For the full application workflow, these backend scripts usually matter more than the standalone harvester: - `scripts/populate_ogm_repos.py`