diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f5e20d6..9201e30e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `fiboa create-stac-collection` describes the fiboa properties in `table:columns`. - Added `pixi run check-hcat`, which compares the HCAT mapping tables the converters read online with the taxonomy. - Added `FiboaDuckDBBaseConverter` for SQL-based conversion of large Parquet sources. -- Added `PerFileBaseConverter` to process multi-file sources incrementally. - Added support for supplementary HCAT/crop mappings via `hcat_mapping_supplements`. - Added `get_hcat_mapping()` and `hcat_lookup()` to `AddHCATMixin` to load the HCAT mapping once and look up codes or names in it. - Added support for Esri JSON output and server-side filters in REST converters. @@ -50,8 +49,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `fiboa publish` no longer uploads to S3 or generates README/LICENSE files. It now creates GeoParquet, PMTiles and a STAC Collection with relative links, checksums and web-map-links. - Updated `aiohttp` to support Zenodo responses that include both returned `Content-Type` headers. - Improved geometry axis handling so generated tiles and bounding boxes keep x/y order consistent in output. -- Updated vecorel-cli to 0.4.1: +- Updated vecorel-cli to 0.4.2: - A converter can set `request_headers` for its downloads, e.g. a browser User-Agent. + - A converter can convert each source on its own and merge the parts (`PerFileConverterMixin`), for sources that do not fit into memory together. + - HTTP downloads no longer fail after 5 minutes. + - `--compression none` no longer fails in DuckDB-based conversions and merges. - The DuckDB converter downloads archives and other sources it cannot read in place also without a cache, and no longer turns `key=value` folders in a path into columns. - GeoParquet files under 250,000 rows get at least ten row groups, so a reader can skip by area. - `create-stac-collection` takes the bbox from the geometries, in EPSG:4326. diff --git a/fiboa_cli/datasets/es.py b/fiboa_cli/datasets/es.py index 742c803e..ab5b893e 100644 --- a/fiboa_cli/datasets/es.py +++ b/fiboa_cli/datasets/es.py @@ -1,13 +1,15 @@ import re import requests +from vecorel_cli.conversion.per_file import PerFileConverterMixin from vecorel_cli.vecorel.extensions import ADMIN_DIVISION from ..conversion.fiboa_converter import FiboaBaseConverter from .commons.hcat import AddHCATMixin -class Converter(AddHCATMixin, FiboaBaseConverter): +# 50 provincial GeoPackages that do not fit in memory together +class Converter(PerFileConverterMixin, AddHCATMixin, FiboaBaseConverter): id = "es" short_name = "Spain" title = "Spain Declared Crops (Cultivos Declarados SIGPAC)" diff --git a/pixi.lock b/pixi.lock index 246e6601..0be78451 100644 --- a/pixi.lock +++ b/pixi.lock @@ -176,7 +176,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -340,7 +340,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -505,7 +505,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -668,7 +668,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -823,7 +823,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -962,7 +962,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -1102,7 +1102,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -1241,7 +1241,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -1358,7 +1358,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -1460,7 +1460,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -1563,7 +1563,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -1663,7 +1663,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -1807,7 +1807,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -1936,7 +1936,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -2066,7 +2066,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -2192,7 +2192,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -2335,7 +2335,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/95/0c/a71501bbc1a674ff72c4d6c2b75f4d9a5af819f5244c3a7558080a8802c5/yarl-1.25.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: ./ @@ -2464,7 +2464,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/e6/cd145cff8e5cf60b8b3c41fbecfa2a45028a8dec3fbc52bec03595ff3d3b/yarl-1.25.1-cp311-cp311-macosx_10_9_x86_64.whl - pypi: ./ @@ -2593,7 +2593,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7c/fbb40fe2d53747c40aa36a9e2bd2178a202f942bda0b670f3306d4aefbce/yarl-1.25.1-cp311-cp311-macosx_11_0_arm64.whl - pypi: ./ @@ -2719,7 +2719,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/aa/9d/8cdceec66a9b940700cb45931741403f045b162afd79bcb93c41cadd0972/yarl-1.25.1-cp311-cp311-win_amd64.whl @@ -2862,7 +2862,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/be/415b00c0fe5a0615b062a456b26623d7ec91c2bee20faea1a14045aa0469/yarl-1.25.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: ./ @@ -2992,7 +2992,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d7/a8/e6c1be0e6761d0f2d10bbf33a3e1e02b99dc83874d92945d7b461a72481e/yarl-1.25.1-cp314-cp314-macosx_10_15_x86_64.whl - pypi: ./ @@ -3122,7 +3122,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/bb/dda344765ffd3430afe1a1c66c866a57fae67786537d4f14607df6505ac1/yarl-1.25.1-cp314-cp314-macosx_11_0_arm64.whl - pypi: ./ @@ -3249,7 +3249,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f5/01/349c0386caedbbe488d519f252df54efac8a1459282d466c474bdd84a620/yarl-1.25.1-cp314-cp314-win_amd64.whl @@ -4765,9 +4765,9 @@ packages: - pypi: ./ name: fiboa-cli version: 0.21.0 - sha256: cc05f5ea116fe87497bf02339b7f9bf7546850874c5fdd2d7da25d109dd97f56 + sha256: 9db0cd7abe351cfb29587fd239a6f831829f4cf6d143121c973cfb51a9b04931 requires_dist: - - vecorel-cli==0.4.1 + - vecorel-cli==0.4.2 - beautifulsoup4>=4.12 - spdx-license-list==3.27.0 requires_python: '>=3.11' @@ -13091,10 +13091,10 @@ packages: purls: [] size: 155910 timestamp: 1785359349999 -- pypi: https://files.pythonhosted.org/packages/6d/81/ad34b5cb65bed8ac70e1eb4fb19a9978ef5bab5b94c2261e053d161ff137/vecorel_cli-0.4.1-py3-none-any.whl +- pypi: https://files.pythonhosted.org/packages/8d/c3/0289ff35f44da218111f779711ae5aee15b403824098f8c14cdbd5192781/vecorel_cli-0.4.2-py3-none-any.whl name: vecorel-cli - version: 0.4.1 - sha256: 7d08ad5a1193ac8c25d16da13cbf78dc441ce8b5746427979dfe7a15b7bd27d9 + version: 0.4.2 + sha256: 049adf899bb60646971d42e569e29c36c54a922758d6459a11364648322c5be9 requires_dist: - pyyaml>=6.0,<7.0 - click>=8.1,<9.0 diff --git a/pyproject.toml b/pyproject.toml index a97c66bc..0fc8c7f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ ] requires-python = ">=3.11" dependencies = [ - "vecorel-cli==0.4.1", + "vecorel-cli==0.4.2", "beautifulsoup4>=4.12", "spdx-license-list==3.27.0", ] diff --git a/tests/test_per_file.py b/tests/test_per_file.py new file mode 100644 index 00000000..45bebb44 --- /dev/null +++ b/tests/test_per_file.py @@ -0,0 +1,45 @@ +import sqlite3 +import zipfile + +import geopandas as gpd + +from fiboa_cli.converters import Converters + +FIXTURE = "tests/data-files/convert/es/1501_ALAVA_cd_2025_20250105.gpkg.zip" + + +def _as_province(tmp_path, code): + """The fixture's GeoPackage, moved to another province, as a zip of its own.""" + with zipfile.ZipFile(FIXTURE) as archive: + name = next(n for n in archive.namelist() if n.endswith(".gpkg")) + gpkg = archive.extract(name, tmp_path / "extracted") + with sqlite3.connect(gpkg) as db: + # the spatial index triggers call SpatiaLite functions that plain sqlite3 lacks + triggers = db.execute("SELECT name FROM sqlite_master WHERE type = 'trigger'").fetchall() + for (trigger,) in triggers: + db.execute(f'DROP TRIGGER "{trigger}"') + db.execute("UPDATE cultivo_declarado SET provincia = ?", (code,)) + copy = tmp_path / f"province_{code}.gpkg.zip" + with zipfile.ZipFile(copy, "w") as archive: + archive.write(gpkg, name) + return copy + + +def test_each_source_is_converted_on_its_own_and_merged(tmp_path): + from fiboa_cli import Registry # noqa + + copy = _as_province(tmp_path, 5) + output = tmp_path / "es.parquet" + + Converters().load("es").convert( + str(output), input_files={FIXTURE: ["*.gpkg"], str(copy): ["*.gpkg"]} + ) + + merged = gpd.read_parquet(output) + assert len(merged) == 20 # 10 rows per source + assert merged["id"].is_unique + # constant within each part, so it only survives the merge as a column + assert sorted(merged["admin_province_code"].unique()) == ["01", "05"] + assert sorted(p.name for p in tmp_path.iterdir()) == sorted( + ["es.parquet", "extracted", copy.name] + ) # the parts are gone