add optional parameter to treat None/NaN as clear the field (default is skip)

Author: Max Wang

.claude/skills/dataverse-sdk-use/SKILL.md

@@ -145,6 +145,10 @@ new_accounts["accountid"] = client.create_dataframe("account", new_accounts)
 new_accounts["telephone1"] = ["555-0199", "555-0299"]
 client.update_dataframe("account", new_accounts, id_column="accountid")
 
+# Clear a field by setting clear_nulls=True (by default, NaN/None fields are skipped)
+df = pd.DataFrame([{"accountid": "guid-1", "websiteurl": None}])
+client.update_dataframe("account", df, id_column="accountid", clear_nulls=True)
+
 # Delete records by passing a Series of GUIDs
 client.delete_dataframe("account", new_accounts["accountid"])
 ```

README.md

@@ -207,6 +207,10 @@ new_accounts["accountid"] = client.create_dataframe("account", new_accounts)
 new_accounts["telephone1"] = ["555-0199", "555-0299"]
 client.update_dataframe("account", new_accounts, id_column="accountid")
 
+# Clear a field by setting clear_nulls=True (by default, NaN/None fields are skipped)
+df = pd.DataFrame([{"accountid": ids[0], "websiteurl": None}])
+client.update_dataframe("account", df, id_column="accountid", clear_nulls=True)
+
 # Delete records by passing a Series of GUIDs
 client.delete_dataframe("account", new_accounts["accountid"])
 ```

examples/advanced/dataframe_operations.py

@@ -124,10 +124,19 @@ def main():
     verified = next(client.get_dataframe(table, select=select_cols, filter=test_filter))
     print(f"  Verified:\n{verified.to_string(index=False)}")
 
-    # Clear a field by updating to None
-    print("\n  Clearing websiteurl for Contoso by setting to None...")
+    # Default: NaN/None fields are skipped (not overridden on server)
+    print("\n  Updating with NaN values (default: clear_nulls=False, fields should stay unchanged)...")
+    sparse_df = pd.DataFrame([
+        {"accountid": new_accounts["accountid"].iloc[0], "telephone1": "555-9999", "websiteurl": None},
+    ])
+    client.update_dataframe(table, sparse_df, id_column="accountid")
+    verified = next(client.get_dataframe(table, select=select_cols, filter=test_filter))
+    print(f"  Verified (Contoso telephone1 updated, websiteurl unchanged):\n{verified.to_string(index=False)}")
+
+    # Opt-in: clear_nulls=True sends None as null to clear the field
+    print("\n  Clearing websiteurl for Contoso with clear_nulls=True...")
     clear_df = pd.DataFrame([{"accountid": new_accounts["accountid"].iloc[0], "websiteurl": None}])
-    client.update_dataframe(table, clear_df, id_column="accountid")
+    client.update_dataframe(table, clear_df, id_column="accountid", clear_nulls=True)
     verified = next(client.get_dataframe(table, select=select_cols, filter=test_filter))
     print(f"  Verified (Contoso websiteurl should be empty):\n{verified.to_string(index=False)}")
 

src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md

@@ -145,6 +145,10 @@ new_accounts["accountid"] = client.create_dataframe("account", new_accounts)
 new_accounts["telephone1"] = ["555-0199", "555-0299"]
 client.update_dataframe("account", new_accounts, id_column="accountid")
 
+# Clear a field by setting clear_nulls=True (by default, NaN/None fields are skipped)
+df = pd.DataFrame([{"accountid": "guid-1", "websiteurl": None}])
+client.update_dataframe("account", df, id_column="accountid", clear_nulls=True)
+
 # Delete records by passing a Series of GUIDs
 client.delete_dataframe("account", new_accounts["accountid"])
 ```

src/PowerPlatform/Dataverse/client.py

@@ -482,6 +482,7 @@ def update_dataframe(
         table_schema_name: str,
         records: pd.DataFrame,
         id_column: str,
+        clear_nulls: bool = False,
     ) -> None:
         """
         Update records from a pandas DataFrame.
@@ -495,6 +496,11 @@ def update_dataframe(
         :type records: ~pd.DataFrame
         :param id_column: Name of the DataFrame column containing record GUIDs.
         :type id_column: :class:`str`
+        :param clear_nulls: When ``False`` (default), missing values (NaN/None) are skipped
+            (the field is left unchanged on the server). When ``True``, missing values are sent
+            as ``null`` to Dataverse, clearing the field. Use ``True`` only when you intentionally
+            want NaN/None values to clear fields.
+        :type clear_nulls: :class:`bool`
 
         :raises TypeError: If ``records`` is not a pandas DataFrame.
         :raises ValueError: If ``id_column`` is not found in the DataFrame.
@@ -515,6 +521,11 @@ def update_dataframe(
                 df = pd.DataFrame({"accountid": ["guid-1", "guid-2", "guid-3"]})
                 df["websiteurl"] = "https://example.com"
                 client.update_dataframe("account", df, id_column="accountid")
+
+            Clear a field by setting clear_nulls=True::
+
+                df = pd.DataFrame([{"accountid": "guid-1", "websiteurl": None}])
+                client.update_dataframe("account", df, id_column="accountid", clear_nulls=True)
         """
         if not isinstance(records, pd.DataFrame):
             raise TypeError("records must be a pandas DataFrame")
@@ -523,7 +534,7 @@ def update_dataframe(
 
         ids = records[id_column].tolist()
         change_columns = [column for column in records.columns if column != id_column]
-        changes = dataframe_to_records(records[change_columns])
+        changes = dataframe_to_records(records[change_columns], na_as_null=clear_nulls)
 
         if len(ids) == 1:
             self.update(table_schema_name, ids[0], changes[0])

src/PowerPlatform/Dataverse/utils/_pandas.py

@@ -10,15 +10,20 @@
 import pandas as pd
 
 
-def dataframe_to_records(df: pd.DataFrame) -> List[Dict[str, Any]]:
-    """Convert a DataFrame to a list of dicts, converting missing values (e.g. NaN, None, NaT, pd.NA) to None and Timestamps to ISO strings."""
+def dataframe_to_records(df: pd.DataFrame, na_as_null: bool = False) -> List[Dict[str, Any]]:
+    """Convert a DataFrame to a list of dicts, converting Timestamps to ISO strings.
+
+    :param df: Input DataFrame.
+    :param na_as_null: When False (default), missing values are omitted from each dict.
+        When True, missing values are included as None (sends null to Dataverse, clearing the field).
+    """
     records = []
     for row in df.to_dict(orient="records"):
         clean = {}
         for k, v in row.items():
             if pd.notna(v):
                 clean[k] = v.isoformat() if isinstance(v, pd.Timestamp) else v
-            else:
+            elif na_as_null:
                 clean[k] = None
         records.append(clean)
     return records

tests/unit/test_client_dataframe.py

@@ -187,14 +187,12 @@ def test_create_empty_dataframe(self):
         self.assertIsInstance(ids, pd.Series)
         self.assertEqual(len(ids), 0)
 
-    def test_create_converts_nan_to_none(self):
-        """NaN values are converted to None in the payload."""
-        df = pd.DataFrame(
-            [
-                {"name": "Contoso", "telephone1": "555-0100"},
-                {"name": "Fabrikam", "telephone1": None},
-            ]
-        )
+    def test_create_drops_nan_values(self):
+        """NaN/None values are omitted from the create payload."""
+        df = pd.DataFrame([
+            {"name": "Contoso", "telephone1": "555-0100"},
+            {"name": "Fabrikam", "telephone1": None},
+        ])
         self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"]
         self.client._odata._entity_set_from_schema_name.return_value = "accounts"
 
@@ -203,7 +201,8 @@ def test_create_converts_nan_to_none(self):
         call_args = self.client._odata._create_multiple.call_args
         records_arg = call_args[0][2]
         self.assertEqual(records_arg[0], {"name": "Contoso", "telephone1": "555-0100"})
-        self.assertEqual(records_arg[1], {"name": "Fabrikam", "telephone1": None})
+        self.assertEqual(records_arg[1], {"name": "Fabrikam"})
+        self.assertNotIn("telephone1", records_arg[1])
 
     def test_create_converts_timestamps_to_iso(self):
         """Timestamp values are converted to ISO 8601 strings."""
@@ -277,17 +276,29 @@ def test_update_multiple_change_columns(self):
         self.assertIn("telephone1", changes)
         self.assertNotIn("accountid", changes)
 
-    def test_update_preserves_none_for_clearing_fields(self):
-        """None values in update are kept as None to allow clearing fields in Dataverse."""
-        df = pd.DataFrame(
-            [
-                {"accountid": "guid-1", "name": "New Name", "telephone1": None},
-                {"accountid": "guid-2", "name": None, "telephone1": "555-0200"},
-            ]
-        )
+    def test_update_skips_nan_by_default(self):
+        """NaN/None values are skipped by default (field left unchanged on server)."""
+        df = pd.DataFrame([
+            {"accountid": "guid-1", "name": "New Name", "telephone1": None},
+            {"accountid": "guid-2", "name": None, "telephone1": "555-0200"},
+        ])
 
         self.client.update_dataframe("account", df, id_column="accountid")
 
+        call_args = self.client._odata._update_by_ids.call_args[0]
+        changes = call_args[2]
+        self.assertEqual(changes[0], {"name": "New Name"})
+        self.assertEqual(changes[1], {"telephone1": "555-0200"})
+
+    def test_update_clear_nulls_sends_none(self):
+        """With clear_nulls=True, NaN/None values are sent as None to clear fields."""
+        df = pd.DataFrame([
+            {"accountid": "guid-1", "name": "New Name", "telephone1": None},
+            {"accountid": "guid-2", "name": None, "telephone1": "555-0200"},
+        ])
+
+        self.client.update_dataframe("account", df, id_column="accountid", clear_nulls=True)
+
         call_args = self.client._odata._update_by_ids.call_args[0]
         changes = call_args[2]
         self.assertEqual(changes[0], {"name": "New Name", "telephone1": None})