Retry logic improvements + Fix Tests warnings

src/PowerPlatform/Dataverse/core/config.py

@@ -14,19 +14,28 @@ class DataverseConfig:
 
     :param language_code: LCID (Locale ID) for localized labels and messages. Default is 1033 (English - United States).
     :type language_code: int
-    :param http_retries: Optional maximum number of retry attempts for transient HTTP errors. Reserved for future use.
+    :param http_retries: Maximum number of retry attempts for HTTP requests (default: 5).
     :type http_retries: int or None
-    :param http_backoff: Optional backoff multiplier (in seconds) between retry attempts. Reserved for future use.
+    :param http_backoff: Base delay in seconds for exponential backoff (default: 0.5).
     :type http_backoff: float or None
-    :param http_timeout: Optional request timeout in seconds. Reserved for future use.
+    :param http_max_backoff: Maximum delay between retry attempts in seconds (default: 60.0).
+    :type http_max_backoff: float or None
+    :param http_timeout: Request timeout in seconds (default: method-dependent).
     :type http_timeout: float or None
+    :param http_jitter: Whether to add jitter to retry delays to prevent thundering herd (default: True).
+    :type http_jitter: bool or None
+    :param http_retry_transient_errors: Whether to retry transient HTTP errors like 429, 502, 503, 504 (default: True).
+    :type http_retry_transient_errors: bool or None
     """
     language_code: int = 1033
 
-    # Optional HTTP tuning (not yet wired everywhere; reserved for future use)
+    # HTTP retry and resilience configuration
     http_retries: Optional[int] = None
     http_backoff: Optional[float] = None
+    http_max_backoff: Optional[float] = None
     http_timeout: Optional[float] = None
+    http_jitter: Optional[bool] = None
+    http_retry_transient_errors: Optional[bool] = None
 
     @classmethod
     def from_env(cls) -> "DataverseConfig":
@@ -36,10 +45,13 @@ def from_env(cls) -> "DataverseConfig":
         :return: Configuration instance with default values.
         :rtype: ~PowerPlatform.Dataverse.core.config.DataverseConfig
         """
-        # Environment-free defaults
+        # Environment-free defaults with enhanced retry configuration
         return cls(
             language_code=1033,
-            http_retries=None,
-            http_backoff=None,
-            http_timeout=None,
+            http_retries=None,  # Will default to 5 in HttpClient
+            http_backoff=None,  # Will default to 0.5 in HttpClient
+            http_max_backoff=None,  # Will default to 60.0 in HttpClient
+            http_timeout=None,  # Will use method-dependent defaults in HttpClient
+            http_jitter=None,  # Will default to True in HttpClient
+            http_retry_transient_errors=None,  # Will default to True in HttpClient
         )

src/PowerPlatform/Dataverse/core/errors.py

@@ -43,7 +43,7 @@ def __init__(
         self.details = details or {}
         self.source = source or "client"
         self.is_transient = is_transient
-        self.timestamp = _dt.datetime.utcnow().isoformat() + "Z"
+        self.timestamp = _dt.datetime.now(_dt.UTC).isoformat() + "Z"
 
     def to_dict(self) -> Dict[str, Any]:
         """

src/PowerPlatform/Dataverse/core/http.py

@@ -11,6 +11,7 @@
 
 from __future__ import annotations
 
+import random
 import time
 from typing import Any, Optional
 
@@ -38,17 +39,26 @@ def __init__(
         retries: Optional[int] = None,
         backoff: Optional[float] = None,
         timeout: Optional[float] = None,
+        max_backoff: Optional[float] = None,
+        jitter: bool = True,
+        retry_transient_errors: bool = True,
     ) -> None:
         self.max_attempts = retries if retries is not None else 5
         self.base_delay = backoff if backoff is not None else 0.5
+        self.max_backoff = max_backoff if max_backoff is not None else 60.0
         self.default_timeout: Optional[float] = timeout
+        self.jitter = jitter
+        self.retry_transient_errors = retry_transient_errors
+
+        # Transient HTTP status codes that should be retried
+        self.transient_status_codes = {429, 502, 503, 504}
 
     def request(self, method: str, url: str, **kwargs: Any) -> requests.Response:
         """
         Execute an HTTP request with automatic retry logic and timeout management.
 
         Applies default timeouts based on HTTP method (120s for POST/DELETE, 10s for others)
-        and retries on network errors with exponential backoff.
+        and retries on transient network errors and HTTP status codes with exponential backoff.
 
         :param method: HTTP method (GET, POST, PUT, DELETE, etc.).
         :type method: str
@@ -68,13 +78,55 @@ def request(self, method: str, url: str, **kwargs: Any) -> requests.Response:
                 m = (method or "").lower()
                 kwargs["timeout"] = 120 if m in ("post", "delete") else 10
 
-        # Small backoff retry on network errors only
+        # Enhanced retry logic with exponential backoff, jitter, and HTTP status retries
         for attempt in range(self.max_attempts):
             try:
-                return requests.request(method, url, **kwargs)
+                response = requests.request(method, url, **kwargs)
+
+                # Check if we should retry based on HTTP status code
+                if (self.retry_transient_errors and 
+                    response.status_code in self.transient_status_codes and
+                    attempt < self.max_attempts - 1):
+
+                    delay = self._calculate_retry_delay(attempt, response)
+                    time.sleep(delay)
+                    continue
+
+                return response
+
             except requests.exceptions.RequestException:
                 if attempt == self.max_attempts - 1:
                     raise
-                delay = self.base_delay * (2 ** attempt)
+                delay = self._calculate_retry_delay(attempt)
                 time.sleep(delay)
                 continue
+
+        # This should never be reached due to the logic above
+        raise RuntimeError("Unexpected end of retry loop")
+
+    def _calculate_retry_delay(self, attempt: int, response: Optional[requests.Response] = None) -> float:
+        """Calculate retry delay with exponential backoff, optional jitter, and Retry-After header support."""
+
+        # Check for Retry-After header (RFC 7231)
+        if response and "Retry-After" in response.headers:
+            try:
+                retry_after = int(response.headers["Retry-After"])
+                # Respect Retry-After but cap it at max_backoff
+                return min(retry_after, self.max_backoff)
+            except (ValueError, TypeError):
+                # If Retry-After is not a valid integer, fall back to exponential backoff
+                pass
+
+        # Exponential backoff: base_delay * (2^attempt)
+        delay = self.base_delay * (2 ** attempt)
+
+        # Cap the delay at max_backoff
+        delay = min(delay, self.max_backoff)
+
+        # Add jitter to prevent thundering herd (±25% of delay)
+        if self.jitter:
+            jitter_range = delay * 0.25
+            jitter_offset = random.uniform(-jitter_range, jitter_range)
+            delay = max(0, delay + jitter_offset)
+
+        return delay

src/PowerPlatform/Dataverse/data/odata.py

@@ -47,7 +47,10 @@ def __init__(
         self._http = HttpClient(
             retries=self.config.http_retries,
             backoff=self.config.http_backoff,
+            max_backoff=getattr(self.config, 'http_max_backoff', None),
             timeout=self.config.http_timeout,
+            jitter=getattr(self.config, 'http_jitter', None) if getattr(self.config, 'http_jitter', None) is not None else True,
+            retry_transient_errors=getattr(self.config, 'http_retry_transient_errors', None) if getattr(self.config, 'http_retry_transient_errors', None) is not None else True,
         )
         # Cache: logical name -> entity set name (plural) resolved from metadata
         self._logical_to_entityset_cache: dict[str, str] = {}
@@ -903,6 +906,32 @@ def _normalize_picklist_label(self, label: str) -> str:
         norm = re.sub(r"\s+", " ", norm).strip().lower()
         return norm
 
+    def _request_metadata_with_retry(self, method: str, url: str, **kwargs) -> Any:
+        """
+        Make HTTP request with metadata-specific retry logic.
+
+        Retries 404 errors for metadata operations since attribute metadata
+        may not be immediately available after creation or schema changes.
+        """
+        import time
+
+        last_error = None
+        for attempt in range(3):  # 3 attempts for metadata operations
+            try:
+                return self._request(method, url, **kwargs)
+            except HttpError as err:
+                last_error = err
+                if getattr(err, "status_code", None) == 404 and attempt < 2:
+                    # Exponential backoff: 0.4s, 0.8s for metadata propagation delays
+                    delay = 0.4 * (2 ** attempt)
+                    time.sleep(delay)
+                    continue
+                raise
+
+        # This should never be reached due to logic above, but just in case
+        if last_error:
+            raise last_error
+
     def _optionset_map(self, logical_name: str, attr_logical: str) -> Optional[Dict[str, int]]:
         """Build or return cached mapping of normalized label -> value for a picklist attribute.
 
@@ -930,23 +959,14 @@ def _optionset_map(self, logical_name: str, attr_logical: str) -> Optional[Dict[
             f"?$filter=LogicalName eq '{attr_esc}'&$select=LogicalName,AttributeType"
         )
         # Retry up to 3 times on 404 (new or not-yet-published attribute metadata). If still 404, raise.
-        r_type = None
-        for attempt in range(3):
-            try:
-                r_type = self._request("get", url_type)
-                break
-            except HttpError as err:
-                if getattr(err, "status_code", None) == 404:
-                    if attempt < 2:
-                        # Exponential-ish backoff: 0.4s, 0.8s
-                        time.sleep(0.4 * (2 ** attempt))
-                        continue
-                    raise RuntimeError(
-                        f"Picklist attribute metadata not found after retries: entity='{logical_name}' attribute='{attr_logical}' (404)"
-                    ) from err
-                raise
-        if r_type is None:
-            raise RuntimeError("Failed to retrieve attribute metadata due to repeated request failures.")
+        try:
+            r_type = self._request_metadata_with_retry("get", url_type)
+        except HttpError as err:
+            if getattr(err, "status_code", None) == 404:
+                raise RuntimeError(
+                    f"Picklist attribute metadata not found after retries: entity='{logical_name}' attribute='{attr_logical}' (404)"
+                ) from err
+            raise
 
         body_type = r_type.json()
         items = body_type.get("value", []) if isinstance(body_type, dict) else []
@@ -964,22 +984,14 @@ def _optionset_map(self, logical_name: str, attr_logical: str) -> Optional[Dict[
             "Microsoft.Dynamics.CRM.PicklistAttributeMetadata?$select=LogicalName&$expand=OptionSet($select=Options)"
         )
         # Step 2 fetch with retries: expanded OptionSet (cast form first)
-        r_opts = None
-        for attempt in range(3):
-            try:
-                r_opts = self._request("get", cast_url)
-                break
-            except HttpError as err:
-                if getattr(err, "status_code", None) == 404:
-                    if attempt < 2:
-                        time.sleep(0.4 * (2 ** attempt))  # 0.4s, 0.8s
-                        continue
-                    raise RuntimeError(
-                        f"Picklist OptionSet metadata not found after retries: entity='{logical_name}' attribute='{attr_logical}' (404)"
-                    ) from err
-                raise
-        if r_opts is None:
-            raise RuntimeError("Failed to retrieve picklist OptionSet metadata due to repeated request failures.")
+        try:
+            r_opts = self._request_metadata_with_retry("get", cast_url)
+        except HttpError as err:
+            if getattr(err, "status_code", None) == 404:
+                raise RuntimeError(
+                    f"Picklist OptionSet metadata not found after retries: entity='{logical_name}' attribute='{attr_logical}' (404)"
+                ) from err
+            raise
 
         attr_full = {}
         try:

tests/unit/core/test_http_errors.py

@@ -34,7 +34,7 @@ def json_fail(): raise ValueError("non-json")
             r.json = json_fail
         return r
 
-class TestClient(ODataClient):
+class MockHttpClient(ODataClient):
     def __init__(self, responses):
         super().__init__(DummyAuth(), "https://org.example", None)
         self._http = DummyHTTP(responses)
@@ -47,7 +47,7 @@ def test_http_404_subcode_and_service_code():
         {"x-ms-correlation-request-id": "cid1"},
         {"error": {"code": "0x800404", "message": "Not found"}},
     )]
-    c = TestClient(responses)
+    c = MockHttpClient(responses)
     with pytest.raises(HttpError) as ei:
         c._request("get", c.api + "/accounts(abc)")
     err = ei.value.to_dict()
@@ -61,7 +61,7 @@ def test_http_429_transient_and_retry_after():
         {"Retry-After": "7"},
         {"error": {"message": "Throttle"}},
     )]
-    c = TestClient(responses)
+    c = MockHttpClient(responses)
     with pytest.raises(HttpError) as ei:
         c._request("get", c.api + "/accounts")
     err = ei.value.to_dict()
@@ -76,7 +76,7 @@ def test_http_500_body_excerpt():
         {},
         "Internal failure XYZ stack truncated",
     )]
-    c = TestClient(responses)
+    c = MockHttpClient(responses)
     with pytest.raises(HttpError) as ei:
         c._request("get", c.api + "/accounts")
     err = ei.value.to_dict()
@@ -90,7 +90,7 @@ def test_http_non_mapped_status_code_subcode_fallback():
         {},
         {"error": {"message": "Teapot"}},
     )]
-    c = TestClient(responses)
+    c = MockHttpClient(responses)
     with pytest.raises(HttpError) as ei:
         c._request("get", c.api + "/accounts")
     err = ei.value.to_dict()