JP: MAFF's own Fude Polygon downloads, adding 2025 and 2026

CHANGELOG.md

@@ -50,7 +50,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - `fiboa publish` no longer uploads to S3 or generates README/LICENSE files. It now creates GeoParquet, PMTiles and a STAC Collection with relative links, checksums and web-map-links.
 - Updated `aiohttp` to support Zenodo responses that include both returned `Content-Type` headers.
 - Improved geometry axis handling so generated tiles and bounding boxes keep x/y order consistent in output.
-- Updated vecorel-cli to 0.4.0:
+- Updated vecorel-cli to 0.4.1:
+  - A converter can set `request_headers` for its downloads, e.g. a browser User-Agent.
+  - The DuckDB converter downloads archives and other sources it cannot read in place also without a cache, and no longer turns `key=value` folders in a path into columns.
   - GeoParquet files under 250,000 rows get at least ten row groups, so a reader can skip by area.
   - `create-stac-collection` takes the bbox from the geometries, in EPSG:4326.
   - **Breaking:** `fiboa merge` keeps all properties by default (`--include`/`--exclude` restrict them), fails if the merged file would be invalid (`--no-strict` warns instead), and refuses datasets with different specification or extension versions.
@@ -75,6 +77,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - FR: Editions now cover 2017-2024, mapped through one shared crop code list (https://fiboa.org/code/fr/fr.csv).
 - HR: Editions now cover 2011-2024.
 - IT-1: Reads the JRC's newest EuroCrops V2 release (`gpqtv202`).
+- JP: Reads MAFF's own Fude Polygon downloads instead of Pacific Spatial's copy, which adds the 2025 and 2026 editions.
 - LV:
   - Editions now cover 2015-2025 from yearly data.gov.lv releases.
   - `block_id` is published from the field-block identifier and `crop:name` comes from the official code list.

fiboa_cli/datasets/jp.py

@@ -1,14 +1,30 @@
+import os
+import zipfile
+
+import geopandas as gpd
+import pandas as pd
+import pyogrio
+import requests
 from vecorel_cli.conversion.duckdb import DuckDBBaseConverter
 
 from ..conversion.fiboa_converter import FiboaBaseConverter
 
+API = "https://restapi.fude.maff.go.jp"
+BASE = "https://opendata.fude.maff.go.jp"
+# MAFF's CDN refuses requests that do not look like a browser
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36"
+}
+PREFECTURES = [f"{code:02d}" for code in range(1, 48)]
+
 
 class JPConverter(DuckDBBaseConverter, FiboaBaseConverter):
+    # One zip of municipality GeoJSONs per prefecture; the URLs only work signed, see download_files
     variants = {
-        "2024": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2024.parquet",
-        "2023": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2023.parquet",
-        "2022": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2022.parquet",
-        "2021": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2021.parquet",
+        str(year): {
+            f"{BASE}/{year}/{year}_{pref}.zip": f"jp_fude_{year}_{pref}.zip" for pref in PREFECTURES
+        }
+        for year in range(2026, 2020, -1)
     }
 
     id = "jp"
@@ -21,19 +37,21 @@ class JPConverter(DuckDBBaseConverter, FiboaBaseConverter):
 current conditions. Fude Polygons are created for the purpose of roughly indicating the locations of farmland.
     """
 
-    provider = "Japanese Ministry of Agriculture, Forestry and Fisheries (MAFF, 農林水産省) <https://www.maff.go.jp/>"
-    attribution = "Fude Polygon Data (2021-2024). Japanese Ministry of Agriculture, Forestry and Fisheries. Processed by Pacific Spatial Solutions, Inc"
+    provider = "Japanese Ministry of Agriculture, Forestry and Fisheries (MAFF, 農林水産省) <https://download.fude.maff.go.jp/>"
+    attribution = "出典：「筆ポリゴンデータ」（農林水産省）を加工して作成"
     license = "CC-BY-4.0"
+    request_headers = HEADERS
 
     columns = {
-        "GEOM": "geometry",
+        "geometry": "geometry",
         "polygon_uuid": "id",
-        "land_type_en": "land_type_en",
+        "land_type": "land_type_en",
         "local_government_cd": "admin_local_code",
         "issue_year": "determination:datetime",
     }
     # SQL migrations (DuckDB converter): per-feature determination date from the issue year
     column_migrations = {
+        "land_type": "CASE land_type WHEN 100 THEN 'rice_field' WHEN 200 THEN 'field' END",
         "issue_year": "make_timestamp(CAST(issue_year AS INTEGER), 1, 1, 0, 0, 0) AT TIME ZONE 'UTC'",
     }
     missing_schemas = {
@@ -42,3 +60,66 @@ class JPConverter(DuckDBBaseConverter, FiboaBaseConverter):
             "admin_local_code": {"type": "string"},
         }
     }
+
+    def select_variant(self, variant):
+        super().select_variant(variant)
+        # MAFF's terms ask for this wording, naming the edition and that it was processed
+        self.attribution = (
+            f"出典：「筆ポリゴンデータ（{self.variant}年度公開）」（農林水産省）を加工して作成"
+        )
+
+    def download_files(self, uris, cache_folder=None, **kwargs):
+        cache_fs, cache_folder = self.get_cache(cache_folder)
+        paths = []
+        for uri, name in uris.items():
+            if uri.startswith(BASE) and not cache_fs.exists(os.path.join(cache_folder, name)):
+                uri = self.sign([uri])[uri]
+            for path in super().download_files({uri: name}, cache_folder, **kwargs):
+                paths.append(
+                    self.to_parquet(path[0] if isinstance(path, tuple) else path, cache_folder)
+                )
+        return paths
+
+    def sign(self, uris):
+        """The download URLs of the API, signed for 30 minutes, for the given unsigned URLs."""
+        if not uris:
+            return {}
+        prefectures = [uri.rsplit("_", 1)[1].removesuffix(".zip") for uri in uris]
+        body = {
+            "issue_year": self.variant,
+            "pref_cd": prefectures,
+            "pref_name": self.prefecture_names(prefectures),
+            "city_cd": [],
+            "city_name": [],
+        }
+        response = requests.post(
+            f"{API}/download/lambda-download", json=body, headers=HEADERS, timeout=60
+        )
+        response.raise_for_status()
+        signed = {found["url"].split("?")[0]: found["url"] for found in response.json()["found"]}
+        return {uri: signed[uri] for uri in uris}
+
+    def prefecture_names(self, codes):
+        url = f"{API}/getprefcitycd/lambda-getprefcitycd?issue_year={self.variant}"
+        response = requests.get(url, headers=HEADERS, timeout=60)
+        response.raise_for_status()
+        names = {p["pref_cd"]: p["pref_name"] for p in response.json()[0]}
+        return [names[code] for code in codes]
+
+    def to_parquet(self, zip_path, cache_folder):
+        """One GeoParquet file of all municipality GeoJSONs in a prefecture zip, kept in the cache."""
+        stem = os.path.splitext(os.path.basename(zip_path))[0]
+        target = os.path.join(cache_folder, f"extracted.{stem}", f"{stem}.parquet")
+        if os.path.exists(target):
+            return target
+        os.makedirs(os.path.dirname(target), exist_ok=True)
+        with zipfile.ZipFile(zip_path) as archive:
+            members = sorted(n for n in archive.namelist() if n.endswith(".json"))
+        frames = [pyogrio.read_dataframe(f"/vsizip/{zip_path}/{member}") for member in members]
+        gdf = gpd.GeoDataFrame(pd.concat(frames, ignore_index=True), crs=frames[0].crs)
+        # each geometry declares JGD2011, which GDAL ignores
+        gdf = gdf.set_crs(6668, allow_override=True)
+        temporary_target = target + ".part"
+        gdf.drop(columns=["history"]).to_parquet(temporary_target)
+        os.replace(temporary_target, target)
+        return target