diff --git a/CHANGELOG.md b/CHANGELOG.md index a73b0de7..9b39ed7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,7 +50,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `fiboa publish` no longer uploads to S3 or generates README/LICENSE files. It now creates GeoParquet, PMTiles and a STAC Collection with relative links, checksums and web-map-links. - Updated `aiohttp` to support Zenodo responses that include both returned `Content-Type` headers. - Improved geometry axis handling so generated tiles and bounding boxes keep x/y order consistent in output. -- Updated vecorel-cli to 0.4.0: +- Updated vecorel-cli to 0.4.1: + - A converter can set `request_headers` for its downloads, e.g. a browser User-Agent. + - The DuckDB converter downloads archives and other sources it cannot read in place also without a cache, and no longer turns `key=value` folders in a path into columns. - GeoParquet files under 250,000 rows get at least ten row groups, so a reader can skip by area. - `create-stac-collection` takes the bbox from the geometries, in EPSG:4326. - **Breaking:** `fiboa merge` keeps all properties by default (`--include`/`--exclude` restrict them), fails if the merged file would be invalid (`--no-strict` warns instead), and refuses datasets with different specification or extension versions. @@ -75,6 +77,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - FR: Editions now cover 2017-2024, mapped through one shared crop code list (https://fiboa.org/code/fr/fr.csv). - HR: Editions now cover 2011-2024. - IT-1: Reads the JRC's newest EuroCrops V2 release (`gpqtv202`). +- JP: Reads MAFF's own Fude Polygon downloads instead of Pacific Spatial's copy, which adds the 2025 and 2026 editions. - LV: - Editions now cover 2015-2025 from yearly data.gov.lv releases. - `block_id` is published from the field-block identifier and `crop:name` comes from the official code list. diff --git a/fiboa_cli/datasets/jp.py b/fiboa_cli/datasets/jp.py index d2174330..07026765 100644 --- a/fiboa_cli/datasets/jp.py +++ b/fiboa_cli/datasets/jp.py @@ -1,14 +1,30 @@ +import os +import zipfile + +import geopandas as gpd +import pandas as pd +import pyogrio +import requests from vecorel_cli.conversion.duckdb import DuckDBBaseConverter from ..conversion.fiboa_converter import FiboaBaseConverter +API = "https://restapi.fude.maff.go.jp" +BASE = "https://opendata.fude.maff.go.jp" +# MAFF's CDN refuses requests that do not look like a browser +HEADERS = { + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36" +} +PREFECTURES = [f"{code:02d}" for code in range(1, 48)] + class JPConverter(DuckDBBaseConverter, FiboaBaseConverter): + # One zip of municipality GeoJSONs per prefecture; the URLs only work signed, see download_files variants = { - "2024": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2024.parquet", - "2023": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2023.parquet", - "2022": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2022.parquet", - "2021": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2021.parquet", + str(year): { + f"{BASE}/{year}/{year}_{pref}.zip": f"jp_fude_{year}_{pref}.zip" for pref in PREFECTURES + } + for year in range(2026, 2020, -1) } id = "jp" @@ -21,19 +37,21 @@ class JPConverter(DuckDBBaseConverter, FiboaBaseConverter): current conditions. Fude Polygons are created for the purpose of roughly indicating the locations of farmland. """ - provider = "Japanese Ministry of Agriculture, Forestry and Fisheries (MAFF, 農林水産省) " - attribution = "Fude Polygon Data (2021-2024). Japanese Ministry of Agriculture, Forestry and Fisheries. Processed by Pacific Spatial Solutions, Inc" + provider = "Japanese Ministry of Agriculture, Forestry and Fisheries (MAFF, 農林水産省) " + attribution = "出典:「筆ポリゴンデータ」(農林水産省)を加工して作成" license = "CC-BY-4.0" + request_headers = HEADERS columns = { - "GEOM": "geometry", + "geometry": "geometry", "polygon_uuid": "id", - "land_type_en": "land_type_en", + "land_type": "land_type_en", "local_government_cd": "admin_local_code", "issue_year": "determination:datetime", } # SQL migrations (DuckDB converter): per-feature determination date from the issue year column_migrations = { + "land_type": "CASE land_type WHEN 100 THEN 'rice_field' WHEN 200 THEN 'field' END", "issue_year": "make_timestamp(CAST(issue_year AS INTEGER), 1, 1, 0, 0, 0) AT TIME ZONE 'UTC'", } missing_schemas = { @@ -42,3 +60,66 @@ class JPConverter(DuckDBBaseConverter, FiboaBaseConverter): "admin_local_code": {"type": "string"}, } } + + def select_variant(self, variant): + super().select_variant(variant) + # MAFF's terms ask for this wording, naming the edition and that it was processed + self.attribution = ( + f"出典:「筆ポリゴンデータ({self.variant}年度公開)」(農林水産省)を加工して作成" + ) + + def download_files(self, uris, cache_folder=None, **kwargs): + cache_fs, cache_folder = self.get_cache(cache_folder) + paths = [] + for uri, name in uris.items(): + if uri.startswith(BASE) and not cache_fs.exists(os.path.join(cache_folder, name)): + uri = self.sign([uri])[uri] + for path in super().download_files({uri: name}, cache_folder, **kwargs): + paths.append( + self.to_parquet(path[0] if isinstance(path, tuple) else path, cache_folder) + ) + return paths + + def sign(self, uris): + """The download URLs of the API, signed for 30 minutes, for the given unsigned URLs.""" + if not uris: + return {} + prefectures = [uri.rsplit("_", 1)[1].removesuffix(".zip") for uri in uris] + body = { + "issue_year": self.variant, + "pref_cd": prefectures, + "pref_name": self.prefecture_names(prefectures), + "city_cd": [], + "city_name": [], + } + response = requests.post( + f"{API}/download/lambda-download", json=body, headers=HEADERS, timeout=60 + ) + response.raise_for_status() + signed = {found["url"].split("?")[0]: found["url"] for found in response.json()["found"]} + return {uri: signed[uri] for uri in uris} + + def prefecture_names(self, codes): + url = f"{API}/getprefcitycd/lambda-getprefcitycd?issue_year={self.variant}" + response = requests.get(url, headers=HEADERS, timeout=60) + response.raise_for_status() + names = {p["pref_cd"]: p["pref_name"] for p in response.json()[0]} + return [names[code] for code in codes] + + def to_parquet(self, zip_path, cache_folder): + """One GeoParquet file of all municipality GeoJSONs in a prefecture zip, kept in the cache.""" + stem = os.path.splitext(os.path.basename(zip_path))[0] + target = os.path.join(cache_folder, f"extracted.{stem}", f"{stem}.parquet") + if os.path.exists(target): + return target + os.makedirs(os.path.dirname(target), exist_ok=True) + with zipfile.ZipFile(zip_path) as archive: + members = sorted(n for n in archive.namelist() if n.endswith(".json")) + frames = [pyogrio.read_dataframe(f"/vsizip/{zip_path}/{member}") for member in members] + gdf = gpd.GeoDataFrame(pd.concat(frames, ignore_index=True), crs=frames[0].crs) + # each geometry declares JGD2011, which GDAL ignores + gdf = gdf.set_crs(6668, allow_override=True) + temporary_target = target + ".part" + gdf.drop(columns=["history"]).to_parquet(temporary_target) + os.replace(temporary_target, target) + return target diff --git a/pixi.lock b/pixi.lock index 0b111fa5..246e6601 100644 --- a/pixi.lock +++ b/pixi.lock @@ -176,7 +176,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -340,7 +340,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -505,7 +505,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -668,7 +668,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -823,7 +823,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -962,7 +962,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -1102,7 +1102,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -1241,7 +1241,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -1358,7 +1358,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -1460,7 +1460,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -1563,7 +1563,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -1663,7 +1663,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -1807,7 +1807,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -1936,7 +1936,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -2066,7 +2066,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -2192,7 +2192,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -2335,7 +2335,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/95/0c/a71501bbc1a674ff72c4d6c2b75f4d9a5af819f5244c3a7558080a8802c5/yarl-1.25.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: ./ @@ -2464,7 +2464,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/e6/cd145cff8e5cf60b8b3c41fbecfa2a45028a8dec3fbc52bec03595ff3d3b/yarl-1.25.1-cp311-cp311-macosx_10_9_x86_64.whl - pypi: ./ @@ -2593,7 +2593,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7c/fbb40fe2d53747c40aa36a9e2bd2178a202f942bda0b670f3306d4aefbce/yarl-1.25.1-cp311-cp311-macosx_11_0_arm64.whl - pypi: ./ @@ -2719,7 +2719,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/aa/9d/8cdceec66a9b940700cb45931741403f045b162afd79bcb93c41cadd0972/yarl-1.25.1-cp311-cp311-win_amd64.whl @@ -2862,7 +2862,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/be/415b00c0fe5a0615b062a456b26623d7ec91c2bee20faea1a14045aa0469/yarl-1.25.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: ./ @@ -2992,7 +2992,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d7/a8/e6c1be0e6761d0f2d10bbf33a3e1e02b99dc83874d92945d7b461a72481e/yarl-1.25.1-cp314-cp314-macosx_10_15_x86_64.whl - pypi: ./ @@ -3122,7 +3122,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/bb/dda344765ffd3430afe1a1c66c866a57fae67786537d4f14607df6505ac1/yarl-1.25.1-cp314-cp314-macosx_11_0_arm64.whl - pypi: ./ @@ -3249,7 +3249,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f5/01/349c0386caedbbe488d519f252df54efac8a1459282d466c474bdd84a620/yarl-1.25.1-cp314-cp314-win_amd64.whl @@ -4765,9 +4765,9 @@ packages: - pypi: ./ name: fiboa-cli version: 0.21.0 - sha256: 0bc20c88d77a364347f871a2c3f14049f11fe421a5e2b321e013435351ada440 + sha256: cc05f5ea116fe87497bf02339b7f9bf7546850874c5fdd2d7da25d109dd97f56 requires_dist: - - vecorel-cli==0.4.0 + - vecorel-cli==0.4.1 - beautifulsoup4>=4.12 - spdx-license-list==3.27.0 requires_python: '>=3.11' @@ -13091,10 +13091,10 @@ packages: purls: [] size: 155910 timestamp: 1785359349999 -- pypi: https://files.pythonhosted.org/packages/6d/0b/10bac3a0771493d8582235a17206109af678e8177e2e85464c79e7ead13f/vecorel_cli-0.4.0-py3-none-any.whl +- pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl name: vecorel-cli - version: 0.4.0 - sha256: 725e479bd55635719e05b905d08a0106c2f3a9c0105c423d9986b444368fae93 + version: 0.4.1 + sha256: 7d08ad5a1193ac8c25d16da13cbf78dc441ce8b5746427979dfe7a15b7bd27d9 requires_dist: - pyyaml>=6.0,<7.0 - click>=8.1,<9.0 diff --git a/pyproject.toml b/pyproject.toml index d6cb60eb..a97c66bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ ] requires-python = ">=3.11" dependencies = [ - "vecorel-cli==0.4.0", + "vecorel-cli==0.4.1", "beautifulsoup4>=4.12", "spdx-license-list==3.27.0", ] diff --git a/tests/data-files/convert/jp/jp_field_polygons_2024.parquet b/tests/data-files/convert/jp/jp_field_polygons_2024.parquet deleted file mode 100644 index 384fa415..00000000 Binary files a/tests/data-files/convert/jp/jp_field_polygons_2024.parquet and /dev/null differ diff --git a/tests/data-files/convert/jp/jp_fude_2026_13.zip b/tests/data-files/convert/jp/jp_fude_2026_13.zip new file mode 100644 index 00000000..b66c82c0 Binary files /dev/null and b/tests/data-files/convert/jp/jp_fude_2026_13.zip differ diff --git a/tests/test_convert.py b/tests/test_convert.py index 797ed5ac..86f5f2b3 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -137,6 +137,7 @@ def _input_files(converter, *names): "es_nc": {"variant": "2025", **_input_files("es_nc", "es_nc.gpkg")}, "ai4sf": _input_files("ai4sf", "1_vietnam_areas.gpkg", "4_cambodia_areas.gpkg"), "nl": {"variant": "2023"}, + "jp": {"variant": "2026", **_input_files("jp", "jp_fude_2026_13.zip")}, "dk#2008": {"variant": "2008"}, "pt": {"variant": "2023"}, **{f"pt#{year}": {"variant": year} for year in PT_EDITIONS}, diff --git a/tests/test_convert_jp.py b/tests/test_convert_jp.py new file mode 100644 index 00000000..c40a473d --- /dev/null +++ b/tests/test_convert_jp.py @@ -0,0 +1,139 @@ +"""Tests for the JP converter's download path. + +MAFF's download URLs only work signed, so the converter asks the API to sign the files +that are not cached yet. These tests answer the API with a fake service and record +what reaches the regular download. +""" + +import os +import shutil + +import pytest +from vecorel_cli.conversion.base import BaseConverter + +from fiboa_cli.datasets import jp + +FIXTURE = "tests/data-files/convert/jp/jp_fude_2026_13.zip" +NAMES = {"01": "北海道", "13": "東京都"} + + +class FakeAPI: + """Signs every requested prefecture and records the requests.""" + + def __init__(self): + self.posts = [] + self.gets = [] + + def get(self, url, headers=None, **kwargs): + self.gets.append((url, headers)) + return FakeResponse([[{"pref_cd": c, "pref_name": n} for c, n in NAMES.items()]]) + + def post(self, url, json=None, headers=None, **kwargs): + self.posts.append((url, json, headers)) + found = [ + {"url": f"{jp.BASE}/{json['issue_year']}/{json['issue_year']}_{code}.zip?sig=x"} + for code in json["pref_cd"] + ] + return FakeResponse({"found": found}) + + +class FakeResponse: + def __init__(self, body): + self.body = body + + def raise_for_status(self): + pass + + def json(self): + return self.body + + +@pytest.fixture +def setup(monkeypatch, tmp_path): + api = FakeAPI() + monkeypatch.setattr(jp.requests, "get", api.get) + monkeypatch.setattr(jp.requests, "post", api.post) + + downloads = [] + + def download_files(self, uris, cache_folder=None, **kwargs): + downloads.append((dict(uris), kwargs)) + paths = [] + for name in uris.values(): + path = tmp_path / "cache" / name + if not path.exists(): + shutil.copy(FIXTURE, path) + paths.append(str(path)) + return paths + + monkeypatch.setattr(BaseConverter, "download_files", download_files) + (tmp_path / "cache").mkdir() + + converter = jp.JPConverter() + converter.select_variant("2026") + return converter, api, downloads, tmp_path / "cache" + + +def maff(pref): + return f"{jp.BASE}/2026/2026_{pref}.zip" + + +def test_signs_the_missing_files_only(setup): + converter, api, downloads, cache = setup + shutil.copy(FIXTURE, cache / "jp_fude_2026_01.zip") + uris = {maff("01"): "jp_fude_2026_01.zip", maff("13"): "jp_fude_2026_13.zip"} + + paths = converter.download_files(uris, str(cache)) + + assert len(api.posts) == 1 + url, body, headers = api.posts[0] + assert url.endswith("/download/lambda-download") + assert body["issue_year"] == "2026" + assert body["pref_cd"] == ["13"] + assert body["pref_name"] == ["東京都"] + assert headers == jp.HEADERS + forwarded = {uri: name for batch, _ in downloads for uri, name in batch.items()} + assert forwarded == { + maff("01"): "jp_fude_2026_01.zip", + f"{maff('13')}?sig=x": "jp_fude_2026_13.zip", + } + assert converter.request_headers == jp.HEADERS + assert [os.path.basename(p) for p in paths] == [ + "jp_fude_2026_01.parquet", + "jp_fude_2026_13.parquet", + ] + + +def test_signs_everything_when_nothing_is_cached(setup): + converter, api, downloads, cache = setup + uris = {maff("01"): "jp_fude_2026_01.zip", maff("13"): "jp_fude_2026_13.zip"} + + converter.download_files(uris, str(cache)) + + # each file is signed just before its download, as a signature lasts 30 minutes + assert [body["pref_cd"] for _, body, _ in api.posts] == [["01"], ["13"]] + assert [body["pref_name"] for _, body, _ in api.posts] == [["北海道"], ["東京都"]] + assert [list(batch) for batch, _ in downloads] == [ + [f"{maff('01')}?sig=x"], + [f"{maff('13')}?sig=x"], + ] + + +def test_a_complete_cache_needs_no_api(setup): + converter, api, downloads, cache = setup + shutil.copy(FIXTURE, cache / "jp_fude_2026_13.zip") + + converter.download_files({maff("13"): "jp_fude_2026_13.zip"}, str(cache)) + + assert api.posts == [] and api.gets == [] + assert downloads[0][0] == {maff("13"): "jp_fude_2026_13.zip"} + + +def test_other_urls_are_not_signed(setup): + converter, api, downloads, cache = setup + mirror = "https://mirror.example/tokyo.zip" + + converter.download_files({mirror: "jp_fude_2026_13.zip"}, str(cache)) + + assert api.posts == [] and api.gets == [] + assert downloads[0][0] == {mirror: "jp_fude_2026_13.zip"}