DE-MV, DE-TH, EC-LT, EC-LV, EC-SI: complete and valid output

CHANGELOG.md

@@ -104,6 +104,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - CH: Removed the national `ch` converter, whose single licence could not cover the cantons' differing terms; the canton converters replace it, and a Swiss file is `fiboa merge` of their outputs.
 
 ### Fixed
+- DE-MV: The field block WFS is read in pages, so all field blocks are converted instead of the first 20,000.
+- DE-TH: A field block without an area last year has an empty `area_last_year` instead of 0, which the schema rejects.
+- EC-LT, EC-LV, EC-SI: `crop:code` and `crop:code_list`, which the crop extension requires, are now written; EC-SI leaves out the parcels without a crop.
 - DE-BB: Landscape elements (hedges, tree rows, copses, wetlands) are left out, and codes under 100, such as the eco-scheme areas, now map to HCAT; they come zero-padded in the source.
 - Added HCAT spelling fixes via `csv_supplements` for DE-BB, DE-NDS and EC-SI.
 - EC-FR, EC-NL-CROP: HCAT mappings for iris, Jerusalem artichoke and parsley now match the taxonomy.

fiboa_cli/datasets/de_mv.py

@@ -1,13 +1,21 @@
 from vecorel_cli.conversion.admin import AdminConverterMixin
 
+from ..conversion.converter_wfs import WFSConverterMixin
 from ..conversion.fiboa_converter import FiboaBaseConverter
 from .commons.hcat import CROP_EXTENSION
 
 
-class Converter(AdminConverterMixin, FiboaBaseConverter):
-    sources = {
-        "https://www.geodaten-mv.de/dienste/gdimv_feldblock_wfs?SERVICE=WFS&VERSION=1.1.0&REQUEST=GetFeature&TYPENAME=mv:feldbloecke&OUTPUTFORMAT=shape-zip": "gdimv_feldblock_wfs.zip"
-    }
+class Converter(WFSConverterMixin, AdminConverterMixin, FiboaBaseConverter):
+    # The service returns at most 20,000 features per request, so the layer is read in pages
+    wfs_url = "https://www.geodaten-mv.de/dienste/gdimv_feldblock_wfs"
+    wfs_params = {"typeNames": "mv:feldbloecke", "outputFormat": "shape-zip", "sortBy": "fbid"}
+    wfs_page_size = 20_000
+    wfs_extension = "zip"
+
+    def get_wfs_total(self, params):
+        # the service answers a hits request that names an output format with an error
+        return super().get_wfs_total({k: v for k, v in params.items() if k != "outputFormat"})
+
     id = "de_mv"
     admin_subdivision_code = "MV"
     short_name = "Germany, Mecklenburg-Western Pomerania"

fiboa_cli/datasets/de_th.py

@@ -63,14 +63,16 @@ class Converter(AdminConverterMixin, FiboaBaseConverter):
         ),
         "FBI_VJ": lambda column: column.str.split(Converter.delim, regex=True),
         "GEO_UPDAT": lambda column: pd.to_datetime(column, format="%d.%m.%Y", utc=True),
+        # a field block that is new this year had no area last year, which the source writes as 0
+        "FB_FL_VJ": lambda column: column.where(column > 0),
     }
 
     column_filters = {"LF": lambda col: col == "LF"}
 
     # Schemas for the fields that are not defined in fiboa
     # Keys must be the values from the COLUMNS dict, not the keys
     missing_schemas = {
-        "required": ["valid_year", "area_last_year", "tk10", "bnk"],
+        "required": ["valid_year", "tk10", "bnk"],
         "properties": {
             "valid_year": {
                 # could also be uint16 or string

fiboa_cli/datasets/ec_lt.py

@@ -26,7 +26,7 @@ class Converter(EuroCropsConverterMixin, FiboaBaseConverter):
 
     columns = {
         "NMA_ID": "claimant_id",
-        "GRUPE": "crop:name",
+        "GRUPE": ["crop:name", "crop:code"],  # the code list is keyed on the name
         "Shape_Leng": "metrics:perimeter",
         "Shape_Area": "metrics:area",
         "geometry": "geometry",

fiboa_cli/datasets/ec_lv.py

@@ -2,6 +2,7 @@
 
 from ..conversion.fiboa_converter import FiboaBaseConverter
 from .commons.ec import EuroCropsConverterMixin
+from .commons.hcat import hcat_mapping_url
 
 
 class Converter(EuroCropsConverterMixin, FiboaBaseConverter):
@@ -54,5 +55,6 @@ class Converter(EuroCropsConverterMixin, FiboaBaseConverter):
     }
 
     def add_hcat(self, gdf):
-        # skip adding hcat
+        # the source carries HCAT already; only the code list is added
+        gdf["crop:code_list"] = hcat_mapping_url(self.hcat_mapping_csv)
         return gdf

fiboa_cli/datasets/ec_si.py

@@ -1,5 +1,6 @@
 from ..conversion.fiboa_converter import FiboaBaseConverter
 from .commons.ec import EuroCropsConverterMixin
+from .commons.hcat import hcat_mapping_url
 
 
 class Converter(EuroCropsConverterMixin, FiboaBaseConverter):
@@ -25,7 +26,7 @@ class Converter(EuroCropsConverterMixin, FiboaBaseConverter):
         "ID": "id",
         "AREA": "metrics:area",
         "GERK_PID": "gerk_pid",
-        "SIFRA_KMRS": "crop_type_class",
+        "SIFRA_KMRS": ["crop_type_class", "crop:code"],
         "RASTLINA": "rastlina",
         "CROP_LAT_E": "crop_lat_e",
         "COLOR": "color",
@@ -48,6 +49,12 @@ class Converter(EuroCropsConverterMixin, FiboaBaseConverter):
         },
     }
 
+    # four parcels carry no crop at all
+    column_filters = {
+        "SIFRA_KMRS": lambda col: col.notna() & (col.astype("string").str.strip() != "")
+    }
+
     def add_hcat(self, gdf):
-        # the source carries HCAT and has no crop:code; only apply the supplement
+        # the source carries HCAT already; only apply the supplement and add the code list
+        gdf["crop:code_list"] = hcat_mapping_url(self.hcat_mapping_csv)
         return self.correct_resolved_hcat(gdf)

tests/test_convert.py

@@ -290,6 +290,10 @@ def _input_files(converter, *names):
 )
 EC_V2_COLUMNS = ("determination:datetime", "metrics:area", "crop:code", "hcat:code", "id")
 expected_columns = {
+    # the crop extension requires both
+    "ec_lt": ("crop:code", "crop:code_list"),
+    "ec_lv": ("crop:code", "crop:code_list"),
+    "ec_si": ("crop:code", "crop:code_list"),
     # the province comes from the file name, the id from the row in each RM's file
     "ca_prairies": (
         "id",

tests/test_converter_wfs.py

@@ -147,3 +147,20 @@ def test_unknown_total_fails(monkeypatch):
     _serve(monkeypatch, _hits("unknown"))
     with pytest.raises(ValueError, match="get_wfs_total"):
         WFSConverter().get_urls()
+
+
+def test_de_mv_reads_every_page(monkeypatch):
+    from fiboa_cli.datasets.de_mv import Converter as DEMVConverter
+
+    service = _serve(monkeypatch, _hits(71_537))
+    urls = DEMVConverter().get_urls()
+
+    # the service caps a request at 20,000 features
+    assert len(urls) == 4
+    query = parse_qs(urlsplit(list(urls)[-1]).query)
+    assert query["startIndex"] == ["60000"]
+    assert query["count"] == ["20000"]
+    assert query["sortBy"] == ["fbid"]
+    assert query["outputFormat"] == ["shape-zip"]
+    # a hits request with an output format makes the service fail
+    assert "outputFormat" not in service.requests[0]