refactor: return Pydantic objects instead of raw dicts

FrancescoSaverioZuppichini · claude · FrancescoSaverioZuppichini · commit 8c59f387a11c · 2026-04-14T23:41:11.000+02:00
- Add ResponseModel base class with camelCase alias generator
- Change all response models to inherit from ResponseModel
- Use TypeAdapter for proper generic type parsing
- Update all examples to use attribute access (res.data.results)
- Fix all test mocks with complete required fields

This follows industry standard SDK patterns where typed objects are
returned for IDE autocompletion and type safety.

Co-Authored-By: Claude Opus 4.5 &lt;noreply@anthropic.com&gt;
diff --git a/examples/crawl/crawl_basic.py b/examples/crawl/crawl_basic.py
@@ -11,10 +11,10 @@
 if start_res.status != "success" or not start_res.data:
     print("Failed to start:", start_res.error)
 else:
-    print("Crawl started:", start_res.data["id"])
-    print("Status:", start_res.data["status"])
+    print("Crawl started:", start_res.data.id)
+    print("Status:", start_res.data.status)
 
-    get_res = sgai.crawl.get(start_res.data["id"])
+    get_res = sgai.crawl.get(start_res.data.id)
     if get_res.status == "success":
-        print("\nProgress:", get_res.data["finished"], "/", get_res.data["total"])
+        print("\nProgress:", get_res.data.finished, "/", get_res.data.total)
         print("Pages:", [p["url"] for p in get_res.data.get("pages", [])])
diff --git a/examples/crawl/crawl_basic_async.py b/examples/crawl/crawl_basic_async.py
@@ -12,12 +12,12 @@ async def main():
         if start_res.status != "success" or not start_res.data:
             print("Failed to start:", start_res.error)
         else:
-            print("Crawl started:", start_res.data["id"])
-            print("Status:", start_res.data["status"])
+            print("Crawl started:", start_res.data.id)
+            print("Status:", start_res.data.status)
 
-            get_res = await sgai.crawl.get(start_res.data["id"])
+            get_res = await sgai.crawl.get(start_res.data.id)
             if get_res.status == "success":
-                print("\nProgress:", get_res.data["finished"], "/", get_res.data["total"])
+                print("\nProgress:", get_res.data.finished, "/", get_res.data.total)
                 print("Pages:", [p["url"] for p in get_res.data.get("pages", [])])
 
 asyncio.run(main())
diff --git a/examples/crawl/crawl_with_formats.py b/examples/crawl/crawl_with_formats.py
@@ -20,13 +20,13 @@
 if start_res.status != "success" or not start_res.data:
     print("Failed to start:", start_res.error)
 else:
-    crawl_id = start_res.data["id"]
+    crawl_id = start_res.data.id
     print("Crawl started:", crawl_id)
-    print("Status:", start_res.data["status"])
+    print("Status:", start_res.data.status)
 
     get_res = sgai.crawl.get(crawl_id)
     if get_res.status == "success":
-        print("\nProgress:", get_res.data["finished"], "/", get_res.data["total"])
+        print("\nProgress:", get_res.data.finished, "/", get_res.data.total)
 
         for page in get_res.data.get("pages", []):
             print(f"\n  Page: {page['url']}")
diff --git a/examples/crawl/crawl_with_formats_async.py b/examples/crawl/crawl_with_formats_async.py
@@ -21,13 +21,13 @@ async def main():
         if start_res.status != "success" or not start_res.data:
             print("Failed to start:", start_res.error)
         else:
-            crawl_id = start_res.data["id"]
+            crawl_id = start_res.data.id
             print("Crawl started:", crawl_id)
-            print("Status:", start_res.data["status"])
+            print("Status:", start_res.data.status)
 
             get_res = await sgai.crawl.get(crawl_id)
             if get_res.status == "success":
-                print("\nProgress:", get_res.data["finished"], "/", get_res.data["total"])
+                print("\nProgress:", get_res.data.finished, "/", get_res.data.total)
 
                 for page in get_res.data.get("pages", []):
                     print(f"\n  Page: {page['url']}")
diff --git a/examples/monitor/monitor_basic.py b/examples/monitor/monitor_basic.py
@@ -10,8 +10,8 @@
 ))
 
 if res.status == "success":
-    print("Monitor created:", res.data["cronId"])
-    print("Status:", res.data["status"])
-    print("Interval:", res.data["interval"])
+    print("Monitor created:", res.data.cron_id)
+    print("Status:", res.data.status)
+    print("Interval:", res.data.interval)
 else:
     print("Failed:", res.error)
diff --git a/examples/monitor/monitor_basic_async.py b/examples/monitor/monitor_basic_async.py
@@ -11,9 +11,9 @@ async def main():
         ))
 
         if res.status == "success":
-            print("Monitor created:", res.data["cronId"])
-            print("Status:", res.data["status"])
-            print("Interval:", res.data["interval"])
+            print("Monitor created:", res.data.cron_id)
+            print("Status:", res.data.status)
+            print("Interval:", res.data.interval)
         else:
             print("Failed:", res.error)
 
diff --git a/examples/monitor/monitor_with_webhook.py b/examples/monitor/monitor_with_webhook.py
@@ -11,9 +11,9 @@
 ))
 
 if res.status == "success":
-    print("Monitor created:", res.data["cronId"])
-    print("Status:", res.data["status"])
-    print("Interval:", res.data["interval"])
+    print("Monitor created:", res.data.cron_id)
+    print("Status:", res.data.status)
+    print("Interval:", res.data.interval)
     print("Webhook configured")
 else:
     print("Failed:", res.error)
diff --git a/examples/monitor/monitor_with_webhook_async.py b/examples/monitor/monitor_with_webhook_async.py
@@ -12,9 +12,9 @@ async def main():
         ))
 
         if res.status == "success":
-            print("Monitor created:", res.data["cronId"])
-            print("Status:", res.data["status"])
-            print("Interval:", res.data["interval"])
+            print("Monitor created:", res.data.cron_id)
+            print("Status:", res.data.status)
+            print("Interval:", res.data.interval)
             print("Webhook configured")
         else:
             print("Failed:", res.error)
diff --git a/examples/scrape/scrape_basic.py b/examples/scrape/scrape_basic.py
@@ -8,7 +8,7 @@
 ))
 
 if res.status == "success":
-    print("Markdown:", res.data["results"].get("markdown", {}).get("data"))
+    print("Markdown:", res.data.results.get("markdown", {}).get("data"))
     print(f"\nTook {res.elapsed_ms}ms")
 else:
     print("Failed:", res.error)
diff --git a/examples/scrape/scrape_basic_async.py b/examples/scrape/scrape_basic_async.py
@@ -9,7 +9,7 @@ async def main():
         ))
 
         if res.status == "success":
-            print("Markdown:", res.data["results"].get("markdown", {}).get("data"))
+            print("Markdown:", res.data.results.get("markdown", {}).get("data"))
             print(f"\nTook {res.elapsed_ms}ms")
         else:
             print("Failed:", res.error)
diff --git a/examples/scrape/scrape_json_extraction.py b/examples/scrape/scrape_json_extraction.py
@@ -25,7 +25,7 @@
 ))
 
 if res.status == "success":
-    json_result = res.data["results"].get("json", {})
+    json_result = res.data.results.get("json", {})
 
     print("=== JSON Extraction ===\n")
     print("Extracted data:")
diff --git a/examples/scrape/scrape_json_extraction_async.py b/examples/scrape/scrape_json_extraction_async.py
@@ -26,7 +26,7 @@ async def main():
         ))
 
         if res.status == "success":
-            json_result = res.data["results"].get("json", {})
+            json_result = res.data.results.get("json", {})
 
             print("=== JSON Extraction ===\n")
             print("Extracted data:")
diff --git a/examples/scrape/scrape_multi_format.py b/examples/scrape/scrape_multi_format.py
@@ -18,7 +18,7 @@
 ))
 
 if res.status == "success":
-    results = res.data["results"]
+    results = res.data.results
 
     print("=== Markdown ===")
     print(results.get("markdown", {}).get("data", [""])[0][:500], "...")
diff --git a/examples/scrape/scrape_multi_format_async.py b/examples/scrape/scrape_multi_format_async.py
@@ -19,7 +19,7 @@ async def main():
         ))
 
         if res.status == "success":
-            results = res.data["results"]
+            results = res.data.results
 
             print("=== Markdown ===")
             print(results.get("markdown", {}).get("data", [""])[0][:500], "...")
diff --git a/examples/scrape/scrape_pdf.py b/examples/scrape/scrape_pdf.py
@@ -9,7 +9,7 @@
 ))
 
 if res.status == "success":
-    print("Markdown:", res.data["results"].get("markdown", {}).get("data"))
+    print("Markdown:", res.data.results.get("markdown", {}).get("data"))
     print(f"\nTook {res.elapsed_ms}ms")
 else:
     print("Failed:", res.error)
diff --git a/examples/scrape/scrape_pdf_async.py b/examples/scrape/scrape_pdf_async.py
@@ -10,7 +10,7 @@ async def main():
         ))
 
         if res.status == "success":
-            print("Markdown:", res.data["results"].get("markdown", {}).get("data"))
+            print("Markdown:", res.data.results.get("markdown", {}).get("data"))
             print(f"\nTook {res.elapsed_ms}ms")
         else:
             print("Failed:", res.error)
diff --git a/examples/scrape/scrape_with_fetchconfig.py b/examples/scrape/scrape_with_fetchconfig.py
@@ -14,7 +14,7 @@
 ))
 
 if res.status == "success":
-    print("Markdown:", res.data["results"].get("markdown", {}).get("data"))
+    print("Markdown:", res.data.results.get("markdown", {}).get("data"))
     print(f"\nTook {res.elapsed_ms}ms")
 else:
     print("Failed:", res.error)
diff --git a/examples/scrape/scrape_with_fetchconfig_async.py b/examples/scrape/scrape_with_fetchconfig_async.py
@@ -15,7 +15,7 @@ async def main():
         ))
 
         if res.status == "success":
-            print("Markdown:", res.data["results"].get("markdown", {}).get("data"))
+            print("Markdown:", res.data.results.get("markdown", {}).get("data"))
             print(f"\nTook {res.elapsed_ms}ms")
         else:
             print("Failed:", res.error)
diff --git a/examples/utilities/credits.py b/examples/utilities/credits.py
@@ -5,12 +5,11 @@
 res = sgai.credits()
 
 if res.status == "success":
-    data = res.data
-    print("Plan:", data["plan"])
-    print("Remaining credits:", data["remaining"])
-    print("Used credits:", data["used"])
+    print("Plan:", res.data.plan)
+    print("Remaining credits:", res.data.remaining)
+    print("Used credits:", res.data.used)
     print("\nJob limits:")
-    print("  Crawl:", data["jobs"]["crawl"]["used"], "/", data["jobs"]["crawl"]["limit"])
-    print("  Monitor:", data["jobs"]["monitor"]["used"], "/", data["jobs"]["monitor"]["limit"])
+    print("  Crawl:", res.data.jobs.crawl.used, "/", res.data.jobs.crawl.limit)
+    print("  Monitor:", res.data.jobs.monitor.used, "/", res.data.jobs.monitor.limit)
 else:
     print("Failed:", res.error)
diff --git a/examples/utilities/credits_async.py b/examples/utilities/credits_async.py
@@ -6,13 +6,12 @@ async def main():
         res = await sgai.credits()
 
         if res.status == "success":
-            data = res.data
-            print("Plan:", data["plan"])
-            print("Remaining credits:", data["remaining"])
-            print("Used credits:", data["used"])
+            print("Plan:", res.data.plan)
+            print("Remaining credits:", res.data.remaining)
+            print("Used credits:", res.data.used)
             print("\nJob limits:")
-            print("  Crawl:", data["jobs"]["crawl"]["used"], "/", data["jobs"]["crawl"]["limit"])
-            print("  Monitor:", data["jobs"]["monitor"]["used"], "/", data["jobs"]["monitor"]["limit"])
+            print("  Crawl:", res.data.jobs.crawl.used, "/", res.data.jobs.crawl.limit)
+            print("  Monitor:", res.data.jobs.monitor.used, "/", res.data.jobs.monitor.limit)
         else:
             print("Failed:", res.error)
 
diff --git a/examples/utilities/health.py b/examples/utilities/health.py
@@ -5,12 +5,11 @@
 res = sgai.health()
 
 if res.status == "success":
-    data = res.data
-    print("Status:", data["status"])
-    print("Uptime:", data["uptime"], "seconds")
-    if data.get("services"):
+    print("Status:", res.data.status)
+    print("Uptime:", res.data.uptime, "seconds")
+    if res.data.services:
         print("Services:")
-        print("  Redis:", data["services"]["redis"])
-        print("  DB:", data["services"]["db"])
+        print("  Redis:", res.data.services.redis)
+        print("  DB:", res.data.services.db)
 else:
     print("Failed:", res.error)
diff --git a/examples/utilities/health_async.py b/examples/utilities/health_async.py
@@ -6,13 +6,12 @@ async def main():
         res = await sgai.health()
 
         if res.status == "success":
-            data = res.data
-            print("Status:", data["status"])
-            print("Uptime:", data["uptime"], "seconds")
-            if data.get("services"):
+            print("Status:", res.data.status)
+            print("Uptime:", res.data.uptime, "seconds")
+            if res.data.services:
                 print("Services:")
-                print("  Redis:", data["services"]["redis"])
-                print("  DB:", data["services"]["db"])
+                print("  Redis:", res.data.services.redis)
+                print("  DB:", res.data.services.db)
         else:
             print("Failed:", res.error)
 
diff --git a/src/scrapegraph_py/async_client.py b/src/scrapegraph_py/async_client.py
@@ -1,11 +1,14 @@
 from __future__ import annotations
 
+import json
 import os
 import re
+import sys
 import time
+from datetime import datetime
 
 import httpx
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
 
 from .env import env
 from .schemas import (
@@ -34,19 +37,11 @@
 def _debug(label: str, data: object = None) -> None:
     if not env.debug:
         return
-    from datetime import datetime
-
     ts = datetime.now().isoformat()
     if data is not None:
-        import json
-
-        print(
-            f"[{ts}] {label}",
-            json.dumps(data, indent=2, default=str),
-            file=__import__("sys").stderr,
-        )
+        print(f"[{ts}] {label}", json.dumps(data, indent=2, default=str), file=sys.stderr)
     else:
-        print(f"[{ts}] {label}", file=__import__("sys").stderr)
+        print(f"[{ts}] {label}", file=sys.stderr)
 
 
 def _map_http_error(status: int) -> str:
@@ -193,13 +188,14 @@ async def _request[T](
                         d = err_body["detail"]
                         detail = d if isinstance(d, str) else str(d)
                         error = f"{error}: {detail}"
-                except Exception:
-                    pass
+                except Exception as e:
+                    _debug(f"Failed to parse error response: {e}")
                 return ApiResult(status="error", data=None, error=error, elapsed_ms=elapsed_ms)
 
-            data = resp.json()
-            _debug(f"<- {resp.status_code} ({elapsed_ms}ms)", data)
-            return ApiResult(status="success", data=data, elapsed_ms=elapsed_ms)
+            raw = resp.json()
+            _debug(f"<- {resp.status_code} ({elapsed_ms}ms)", raw)
+            parsed = TypeAdapter(response_type).validate_python(raw)
+            return ApiResult(status="success", data=parsed, elapsed_ms=elapsed_ms)
 
         except httpx.TimeoutException:
             return ApiResult(status="error", data=None, error="Request timed out", elapsed_ms=0)
diff --git a/src/scrapegraph_py/client.py b/src/scrapegraph_py/client.py
@@ -1,11 +1,14 @@
 from __future__ import annotations
 
+import json
 import os
 import re
+import sys
 import time
+from datetime import datetime
 
 import httpx
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
 
 from .env import env
 from .schemas import (
@@ -34,19 +37,11 @@
 def _debug(label: str, data: object = None) -> None:
     if not env.debug:
         return
-    from datetime import datetime
-
     ts = datetime.now().isoformat()
     if data is not None:
-        import json
-
-        print(
-            f"[{ts}] {label}",
-            json.dumps(data, indent=2, default=str),
-            file=__import__("sys").stderr,
-        )
+        print(f"[{ts}] {label}", json.dumps(data, indent=2, default=str), file=sys.stderr)
     else:
-        print(f"[{ts}] {label}", file=__import__("sys").stderr)
+        print(f"[{ts}] {label}", file=sys.stderr)
 
 
 def _map_http_error(status: int) -> str:
@@ -193,13 +188,14 @@ def _request[T](
                         d = err_body["detail"]
                         detail = d if isinstance(d, str) else str(d)
                         error = f"{error}: {detail}"
-                except Exception:
-                    pass
+                except Exception as e:
+                    _debug(f"Failed to parse error response: {e}")
                 return ApiResult(status="error", data=None, error=error, elapsed_ms=elapsed_ms)
 
-            data = resp.json()
-            _debug(f"<- {resp.status_code} ({elapsed_ms}ms)", data)
-            return ApiResult(status="success", data=data, elapsed_ms=elapsed_ms)
+            raw = resp.json()
+            _debug(f"<- {resp.status_code} ({elapsed_ms}ms)", raw)
+            parsed = TypeAdapter(response_type).validate_python(raw)
+            return ApiResult(status="success", data=parsed, elapsed_ms=elapsed_ms)
 
         except httpx.TimeoutException:
             return ApiResult(status="error", data=None, error="Request timed out", elapsed_ms=0)
diff --git a/src/scrapegraph_py/schemas.py b/src/scrapegraph_py/schemas.py
diff --git a/tests/test_client.py b/tests/test_client.py
