convert NaN to None instead of ignoring them

Author: Max Wang

examples/advanced/dataframe_operations.py

@@ -124,6 +124,13 @@ def main():
     verified = next(client.get_dataframe(table, select=select_cols, filter=test_filter))
     print(f"  Verified:\n{verified.to_string(index=False)}")
 
+    # Clear a field by updating to None
+    print("\n  Clearing websiteurl for Contoso by setting to None...")
+    clear_df = pd.DataFrame([{"accountid": new_accounts["accountid"].iloc[0], "websiteurl": None}])
+    client.update_dataframe(table, clear_df, id_column="accountid")
+    verified = next(client.get_dataframe(table, select=select_cols, filter=test_filter))
+    print(f"  Verified (Contoso websiteurl should be empty):\n{verified.to_string(index=False)}")
+
     # ── 7. Delete records by passing a Series of GUIDs ────────────
     print("\n" + "-" * 60)
     print("7. Delete records by passing a Series of GUIDs")

src/PowerPlatform/Dataverse/client.py

@@ -417,7 +417,7 @@ def get_dataframe(
 
             Collect all pages into one DataFrame::
 
-                all_data = pd.concat(client.get_dataframe("account", select=["name"]))
+                all_data = pd.concat(client.get_dataframe("account", select=["name"]), ignore_index=True)
         """
         if record_id is not None:
             result = self.get(

src/PowerPlatform/Dataverse/utils/_pandas.py

@@ -11,12 +11,14 @@
 
 
 def dataframe_to_records(df: pd.DataFrame) -> List[Dict[str, Any]]:
-    """Convert a DataFrame to a list of dicts, dropping NaN values and converting Timestamps to ISO strings."""
+    """Convert a DataFrame to a list of dicts, converting missing values (e.g. NaN, None, NaT, pd.NA) to None and Timestamps to ISO strings."""
     records = []
     for row in df.to_dict(orient="records"):
         clean = {}
         for k, v in row.items():
             if pd.notna(v):
                 clean[k] = v.isoformat() if isinstance(v, pd.Timestamp) else v
+            else:
+                clean[k] = None
         records.append(clean)
     return records

tests/unit/test_client_dataframe.py

@@ -187,14 +187,12 @@ def test_create_empty_dataframe(self):
         self.assertIsInstance(ids, pd.Series)
         self.assertEqual(len(ids), 0)
 
-    def test_create_drops_nan_values(self):
-        """NaN values are stripped from the payload (field omitted, not sent as NaN)."""
-        df = pd.DataFrame(
-            [
-                {"name": "Contoso", "telephone1": "555-0100"},
-                {"name": "Fabrikam", "telephone1": None},
-            ]
-        )
+    def test_create_converts_nan_to_none(self):
+        """NaN values are converted to None in the payload."""
+        df = pd.DataFrame([
+            {"name": "Contoso", "telephone1": "555-0100"},
+            {"name": "Fabrikam", "telephone1": None},
+        ])
         self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"]
         self.client._odata._entity_set_from_schema_name.return_value = "accounts"
 
@@ -203,8 +201,7 @@ def test_create_drops_nan_values(self):
         call_args = self.client._odata._create_multiple.call_args
         records_arg = call_args[0][2]
         self.assertEqual(records_arg[0], {"name": "Contoso", "telephone1": "555-0100"})
-        self.assertEqual(records_arg[1], {"name": "Fabrikam"})
-        self.assertNotIn("telephone1", records_arg[1])
+        self.assertEqual(records_arg[1], {"name": "Fabrikam", "telephone1": None})
 
     def test_create_converts_timestamps_to_iso(self):
         """Timestamp values are converted to ISO 8601 strings."""
@@ -278,21 +275,19 @@ def test_update_multiple_change_columns(self):
         self.assertIn("telephone1", changes)
         self.assertNotIn("accountid", changes)
 
-    def test_update_drops_nan_from_changes(self):
-        """NaN values in change columns are stripped from the payload."""
-        df = pd.DataFrame(
-            [
-                {"accountid": "guid-1", "name": "New Name", "telephone1": None},
-                {"accountid": "guid-2", "name": None, "telephone1": "555-0200"},
-            ]
-        )
+    def test_update_preserves_none_for_clearing_fields(self):
+        """None values in update are kept as None to allow clearing fields in Dataverse."""
+        df = pd.DataFrame([
+            {"accountid": "guid-1", "name": "New Name", "telephone1": None},
+            {"accountid": "guid-2", "name": None, "telephone1": "555-0200"},
+        ])
 
         self.client.update_dataframe("account", df, id_column="accountid")
 
         call_args = self.client._odata._update_by_ids.call_args[0]
         changes = call_args[2]
-        self.assertEqual(changes[0], {"name": "New Name"})
-        self.assertEqual(changes[1], {"telephone1": "555-0200"})
+        self.assertEqual(changes[0], {"name": "New Name", "telephone1": None})
+        self.assertEqual(changes[1], {"name": None, "telephone1": "555-0200"})
 
 
 class TestDataFrameDelete(unittest.TestCase):