From 2c8ff4023a04cc2099eac315f0587ae38bcc7a3e Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Tue, 27 Oct 2026 23:00:00 +0100 Subject: [PATCH 1/4] DE-TH, EC-LT, EC-LV, EC-SI: write the fields their extensions require Co-Authored-By: Claude Opus 5.5 --- fiboa_cli/datasets/de_th.py | 4 +++- fiboa_cli/datasets/ec_lt.py | 2 +- fiboa_cli/datasets/ec_lv.py | 4 +++- fiboa_cli/datasets/ec_si.py | 11 +++++++++-- tests/test_convert.py | 4 ++++ 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/fiboa_cli/datasets/de_th.py b/fiboa_cli/datasets/de_th.py index 60e8bc6e..efbff628 100644 --- a/fiboa_cli/datasets/de_th.py +++ b/fiboa_cli/datasets/de_th.py @@ -63,6 +63,8 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): ), "FBI_VJ": lambda column: column.str.split(Converter.delim, regex=True), "GEO_UPDAT": lambda column: pd.to_datetime(column, format="%d.%m.%Y", utc=True), + # a field block that is new this year had no area last year, which the source writes as 0 + "FB_FL_VJ": lambda column: column.where(column > 0), } column_filters = {"LF": lambda col: col == "LF"} @@ -70,7 +72,7 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): # Schemas for the fields that are not defined in fiboa # Keys must be the values from the COLUMNS dict, not the keys missing_schemas = { - "required": ["valid_year", "area_last_year", "tk10", "bnk"], + "required": ["valid_year", "tk10", "bnk"], "properties": { "valid_year": { # could also be uint16 or string diff --git a/fiboa_cli/datasets/ec_lt.py b/fiboa_cli/datasets/ec_lt.py index 24a73728..273bb867 100644 --- a/fiboa_cli/datasets/ec_lt.py +++ b/fiboa_cli/datasets/ec_lt.py @@ -26,7 +26,7 @@ class Converter(EuroCropsConverterMixin, FiboaBaseConverter): columns = { "NMA_ID": "claimant_id", - "GRUPE": "crop:name", + "GRUPE": ["crop:name", "crop:code"], # the code list is keyed on the name "Shape_Leng": "metrics:perimeter", "Shape_Area": "metrics:area", "geometry": "geometry", diff --git a/fiboa_cli/datasets/ec_lv.py b/fiboa_cli/datasets/ec_lv.py index a94fa2a3..859b50e7 100644 --- a/fiboa_cli/datasets/ec_lv.py +++ b/fiboa_cli/datasets/ec_lv.py @@ -2,6 +2,7 @@ from ..conversion.fiboa_converter import FiboaBaseConverter from .commons.ec import EuroCropsConverterMixin +from .commons.hcat import hcat_mapping_url class Converter(EuroCropsConverterMixin, FiboaBaseConverter): @@ -54,5 +55,6 @@ class Converter(EuroCropsConverterMixin, FiboaBaseConverter): } def add_hcat(self, gdf): - # skip adding hcat + # the source carries HCAT already; only the code list is added + gdf["crop:code_list"] = hcat_mapping_url(self.hcat_mapping_csv) return gdf diff --git a/fiboa_cli/datasets/ec_si.py b/fiboa_cli/datasets/ec_si.py index f732e055..c8cf2821 100644 --- a/fiboa_cli/datasets/ec_si.py +++ b/fiboa_cli/datasets/ec_si.py @@ -1,5 +1,6 @@ from ..conversion.fiboa_converter import FiboaBaseConverter from .commons.ec import EuroCropsConverterMixin +from .commons.hcat import hcat_mapping_url class Converter(EuroCropsConverterMixin, FiboaBaseConverter): @@ -25,7 +26,7 @@ class Converter(EuroCropsConverterMixin, FiboaBaseConverter): "ID": "id", "AREA": "metrics:area", "GERK_PID": "gerk_pid", - "SIFRA_KMRS": "crop_type_class", + "SIFRA_KMRS": ["crop_type_class", "crop:code"], "RASTLINA": "rastlina", "CROP_LAT_E": "crop_lat_e", "COLOR": "color", @@ -48,6 +49,12 @@ class Converter(EuroCropsConverterMixin, FiboaBaseConverter): }, } + # four parcels carry no crop at all + column_filters = { + "SIFRA_KMRS": lambda col: col.notna() & (col.astype("string").str.strip() != "") + } + def add_hcat(self, gdf): - # the source carries HCAT and has no crop:code; only apply the supplement + # the source carries HCAT already; only apply the supplement and add the code list + gdf["crop:code_list"] = hcat_mapping_url(self.hcat_mapping_csv) return self.correct_resolved_hcat(gdf) diff --git a/tests/test_convert.py b/tests/test_convert.py index 86f5f2b3..1b3d1ffa 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -290,6 +290,10 @@ def _input_files(converter, *names): ) EC_V2_COLUMNS = ("determination:datetime", "metrics:area", "crop:code", "hcat:code", "id") expected_columns = { + # the crop extension requires both + "ec_lt": ("crop:code", "crop:code_list"), + "ec_lv": ("crop:code", "crop:code_list"), + "ec_si": ("crop:code", "crop:code_list"), # the province comes from the file name, the id from the row in each RM's file "ca_prairies": ( "id", From 94794f8d87e5ee7cb9717373e177d54f58b7b73a Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Tue, 27 Oct 2026 23:57:18 +0100 Subject: [PATCH 2/4] DE-MV: read the field block WFS in pages, all 71,537 instead of the first 20,000 Co-Authored-By: Claude Opus 5.5 --- fiboa_cli/datasets/de_mv.py | 11 +++++++---- tests/test_converter_wfs.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/fiboa_cli/datasets/de_mv.py b/fiboa_cli/datasets/de_mv.py index 456a6543..3e38215e 100644 --- a/fiboa_cli/datasets/de_mv.py +++ b/fiboa_cli/datasets/de_mv.py @@ -1,13 +1,16 @@ from vecorel_cli.conversion.admin import AdminConverterMixin +from ..conversion.converter_wfs import WFSConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter from .commons.hcat import CROP_EXTENSION -class Converter(AdminConverterMixin, FiboaBaseConverter): - sources = { - "https://www.geodaten-mv.de/dienste/gdimv_feldblock_wfs?SERVICE=WFS&VERSION=1.1.0&REQUEST=GetFeature&TYPENAME=mv:feldbloecke&OUTPUTFORMAT=shape-zip": "gdimv_feldblock_wfs.zip" - } +class Converter(WFSConverterMixin, AdminConverterMixin, FiboaBaseConverter): + # The service returns at most 20,000 features per request, so the layer is read in pages + wfs_url = "https://www.geodaten-mv.de/dienste/gdimv_feldblock_wfs" + wfs_params = {"typeNames": "mv:feldbloecke", "outputFormat": "shape-zip", "sortBy": "fbid"} + wfs_page_size = 20_000 + wfs_extension = "zip" id = "de_mv" admin_subdivision_code = "MV" short_name = "Germany, Mecklenburg-Western Pomerania" diff --git a/tests/test_converter_wfs.py b/tests/test_converter_wfs.py index 6daf7e63..69c228a8 100644 --- a/tests/test_converter_wfs.py +++ b/tests/test_converter_wfs.py @@ -147,3 +147,19 @@ def test_unknown_total_fails(monkeypatch): _serve(monkeypatch, _hits("unknown")) with pytest.raises(ValueError, match="get_wfs_total"): WFSConverter().get_urls() + + +def test_de_mv_reads_every_page(monkeypatch): + from fiboa_cli.datasets.de_mv import Converter as DEMVConverter + + service = _serve(monkeypatch, _hits(71_537)) + urls = DEMVConverter().get_urls() + + # the service caps a request at 20,000 features + assert len(urls) == 4 + query = parse_qs(urlsplit(list(urls)[-1]).query) + assert query["startIndex"] == ["60000"] + assert query["count"] == ["20000"] + assert query["sortBy"] == ["fbid"] + assert query["outputFormat"] == ["shape-zip"] + assert service.requests From 52eec5d70b5035c73d564bf54010f45607daa07f Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Tue, 27 Oct 2026 23:58:12 +0100 Subject: [PATCH 3/4] DE-MV: count the features without an output format, which the service rejects Co-Authored-By: Claude Opus 5.5 --- fiboa_cli/datasets/de_mv.py | 5 +++++ tests/test_converter_wfs.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/fiboa_cli/datasets/de_mv.py b/fiboa_cli/datasets/de_mv.py index 3e38215e..96d210a5 100644 --- a/fiboa_cli/datasets/de_mv.py +++ b/fiboa_cli/datasets/de_mv.py @@ -11,6 +11,11 @@ class Converter(WFSConverterMixin, AdminConverterMixin, FiboaBaseConverter): wfs_params = {"typeNames": "mv:feldbloecke", "outputFormat": "shape-zip", "sortBy": "fbid"} wfs_page_size = 20_000 wfs_extension = "zip" + + def get_wfs_total(self, params): + # the service answers a hits request that names an output format with an error + return super().get_wfs_total({k: v for k, v in params.items() if k != "outputFormat"}) + id = "de_mv" admin_subdivision_code = "MV" short_name = "Germany, Mecklenburg-Western Pomerania" diff --git a/tests/test_converter_wfs.py b/tests/test_converter_wfs.py index 69c228a8..6052d4c6 100644 --- a/tests/test_converter_wfs.py +++ b/tests/test_converter_wfs.py @@ -162,4 +162,5 @@ def test_de_mv_reads_every_page(monkeypatch): assert query["count"] == ["20000"] assert query["sortBy"] == ["fbid"] assert query["outputFormat"] == ["shape-zip"] - assert service.requests + # a hits request with an output format makes the service fail + assert "outputFormat" not in service.requests[0] From 7713ceaf8c8be29f8c21c9ef15e6ac8f24ccbda4 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Wed, 28 Oct 2026 01:01:11 +0100 Subject: [PATCH 4/4] CHANGELOG for DE-MV, DE-TH, EC-LT, EC-LV and EC-SI Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b39ed7d..3e8b48e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - CH: Removed the national `ch` converter, whose single licence could not cover the cantons' differing terms; the canton converters replace it, and a Swiss file is `fiboa merge` of their outputs. ### Fixed +- DE-MV: The field block WFS is read in pages, so all field blocks are converted instead of the first 20,000. +- DE-TH: A field block without an area last year has an empty `area_last_year` instead of 0, which the schema rejects. +- EC-LT, EC-LV, EC-SI: `crop:code` and `crop:code_list`, which the crop extension requires, are now written; EC-SI leaves out the parcels without a crop. - Added HCAT spelling fixes via `csv_supplements` for DE-BB, DE-NDS and EC-SI. - EC-FR, EC-NL-CROP: HCAT mappings for iris, Jerusalem artichoke and parsley now match the taxonomy. - EuroCrops converters whose source already carries HCAT (such as EC-FR and EC-SI) now apply their supplements too.