Merge pull request #23 from microsoft/users/zhaodongwang/UserAgentHeaderPlusHeadersRefactor

add user agent to calls and consolidate headers calls into _request

Author: zhaodongwang-msft

examples/quickstart_file_upload.py

@@ -191,7 +191,7 @@ def ensure_file_attribute_generic(schema_name: str, label: str, key_prefix: str)
             f"{odata.api}/EntityDefinitions({meta_id})/Attributes?$select=SchemaName&$filter="
             f"SchemaName eq '{schema_name}'"
         )
-        r = odata._request("get", url, headers=odata._headers())
+        r = odata._request("get", url)
         val = []
         try:
             val = r.json().get("value", [])
@@ -215,7 +215,7 @@ def ensure_file_attribute_generic(schema_name: str, label: str, key_prefix: str)
     }
     try:
         url = f"{odata.api}/EntityDefinitions({meta_id})/Attributes"
-        r = odata._request("post", url, headers=odata._headers(), json=payload)
+        r = odata._request("post", url, json=payload)
         print({f"{key_prefix}_file_attribute_created": True})
         time.sleep(2)
         return True
@@ -288,7 +288,7 @@ def get_dataset_info(file_path: Path):
         print({"small_upload_completed": True, "small_source_size": small_file_size})
         odata = client._get_odata()
         dl_url_single = f"{odata.api}/{entity_set}({record_id})/{small_file_attr_logical}/$value"  # raw entity_set URL OK
-        resp_single = odata._request("get", dl_url_single, headers=odata._headers())
+        resp_single = odata._request("get", dl_url_single)
         content_single = resp_single.content or b""
         import hashlib  # noqa: WPS433
         downloaded_hash = hashlib.sha256(content_single).hexdigest() if content_single else None
@@ -313,7 +313,7 @@ def get_dataset_info(file_path: Path):
             mode="small",
         ))
         print({"small_replace_upload_completed": True, "small_replace_source_size": replace_size_small})
-        resp_single_replace = odata._request("get", dl_url_single, headers=odata._headers())
+        resp_single_replace = odata._request("get", dl_url_single)
         content_single_replace = resp_single_replace.content or b""
         downloaded_hash_replace = hashlib.sha256(content_single_replace).hexdigest() if content_single_replace else None
         hash_match_replace = (downloaded_hash_replace == replace_hash_small) if (downloaded_hash_replace and replace_hash_small) else None
@@ -343,7 +343,7 @@ def get_dataset_info(file_path: Path):
         print({"chunk_upload_completed": True})
         odata = client._get_odata()
         dl_url_chunk = f"{odata.api}/{entity_set}({record_id})/{chunk_file_attr_logical}/$value"  # raw entity_set for download
-        resp_chunk = odata._request("get", dl_url_chunk, headers=odata._headers())
+        resp_chunk = odata._request("get", dl_url_chunk)
         content_chunk = resp_chunk.content or b""
         import hashlib  # noqa: WPS433
         dst_hash_chunk = hashlib.sha256(content_chunk).hexdigest() if content_chunk else None
@@ -367,7 +367,7 @@ def get_dataset_info(file_path: Path):
             mode="chunk",
         ))
         print({"chunk_replace_upload_completed": True})
-        resp_chunk_replace = odata._request("get", dl_url_chunk, headers=odata._headers())
+        resp_chunk_replace = odata._request("get", dl_url_chunk)
         content_chunk_replace = resp_chunk_replace.content or b""
         dst_hash_chunk_replace = hashlib.sha256(content_chunk_replace).hexdigest() if content_chunk_replace else None
         hash_match_chunk_replace = (dst_hash_chunk_replace == replace_hash_chunk) if (dst_hash_chunk_replace and replace_hash_chunk) else None

src/dataverse_sdk/odata.py

@@ -54,14 +54,24 @@ def _headers(self) -> Dict[str, str]:
         """Build standard OData headers with bearer auth."""
         scope = f"{self.base_url}/.default"
         token = self.auth.acquire_token(scope).access_token
+        # TODO: add version to User-Agent
         return {
             "Authorization": f"Bearer {token}",
             "Accept": "application/json",
             "Content-Type": "application/json",
             "OData-MaxVersion": "4.0",
             "OData-Version": "4.0",
+            "User-Agent": "DataversePythonSDK",
         }
 
+    def _merge_headers(self, headers: Optional[Dict[str, str]] = None) -> Dict[str, str]:
+        base = self._headers()
+        if not headers:
+            return base
+        merged = base.copy()
+        merged.update(headers)
+        return merged
+
     def _raw_request(self, method: str, url: str, **kwargs):
         return self._http.request(method, url, **kwargs)
 
@@ -71,6 +81,8 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = (200, 2
         Returns the raw response for success codes; raises HttpError with extracted
         Dataverse error payload fields and correlation identifiers otherwise.
         """
+        headers = kwargs.pop("headers", None)
+        kwargs["headers"] = self._merge_headers(headers)
         r = self._raw_request(method, url, **kwargs)
         if r.status_code in expected:
             return r
@@ -149,8 +161,7 @@ def _create_single(self, entity_set: str, logical_name: str, record: Dict[str, A
         """
         record = self._convert_labels_to_ints(logical_name, record)
         url = f"{self.api}/{entity_set}"
-        headers = self._headers().copy()
-        r = self._request("post", url, headers=headers, json=record)
+        r = self._request("post", url, json=record)
 
         ent_loc = r.headers.get("OData-EntityId") or r.headers.get("OData-EntityID")
         if ent_loc:
@@ -184,8 +195,7 @@ def _create_multiple(self, entity_set: str, logical_name: str, records: List[Dic
         # Bound action form: POST {entity_set}/Microsoft.Dynamics.CRM.CreateMultiple
         url = f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.CreateMultiple"
         # The action currently returns only Ids; no need to request representation.
-        headers = self._headers().copy()
-        r = self._request("post", url, headers=headers, json=payload)
+        r = self._request("post", url, json=payload)
         try:
             body = r.json() if r.text else {}
         except ValueError:
@@ -302,9 +312,7 @@ def _update(self, logical_name: str, key: str, data: Dict[str, Any]) -> None:
         data = self._convert_labels_to_ints(logical_name, data)
         entity_set = self._entity_set_from_logical(logical_name)
         url = f"{self.api}/{entity_set}{self._format_key(key)}"
-        headers = self._headers().copy()
-        headers["If-Match"] = "*"
-        r = self._request("patch", url, headers=headers, json=data)
+        r = self._request("patch", url, headers={"If-Match": "*"}, json=data)
 
     def _update_multiple(self, entity_set: str, logical_name: str, records: List[Dict[str, Any]]) -> None:
         """Bulk update existing records via the collection-bound UpdateMultiple action.
@@ -353,18 +361,15 @@ def _update_multiple(self, entity_set: str, logical_name: str, records: List[Dic
 
         payload = {"Targets": enriched}
         url = f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.UpdateMultiple"
-        headers = self._headers().copy()
-        r = self._request("post", url, headers=headers, json=payload)
+        r = self._request("post", url, json=payload)
         # Intentionally ignore response content: no stable contract for IDs across environments.
         return None
 
     def _delete(self, logical_name: str, key: str) -> None:
         """Delete a record by GUID or alternate key."""
         entity_set = self._entity_set_from_logical(logical_name)
         url = f"{self.api}/{entity_set}{self._format_key(key)}"
-        headers = self._headers().copy()
-        headers["If-Match"] = "*"
-        self._request("delete", url, headers=headers)
+        self._request("delete", url, headers={"If-Match": "*"})
 
     def _get(self, logical_name: str, key: str, select: Optional[str] = None) -> Dict[str, Any]:
         """Retrieve a single record.
@@ -383,7 +388,7 @@ def _get(self, logical_name: str, key: str, select: Optional[str] = None) -> Dic
             params["$select"] = select
         entity_set = self._entity_set_from_logical(logical_name)
         url = f"{self.api}/{entity_set}{self._format_key(key)}"
-        r = self._request("get", url, headers=self._headers(), params=params)
+        r = self._request("get", url, params=params)
         return r.json()
 
     def _get_multiple(
@@ -421,13 +426,14 @@ def _get_multiple(
             A page of records from the Web API (the "value" array for each page).
         """
 
-        headers = self._headers().copy()
+        extra_headers: Dict[str, str] = {}
         if page_size is not None:
             ps = int(page_size)
             if ps > 0:
-                headers["Prefer"] = f"odata.maxpagesize={ps}"
+                extra_headers["Prefer"] = f"odata.maxpagesize={ps}"
 
         def _do_request(url: str, *, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+            headers = extra_headers if extra_headers else None
             r = self._request("get", url, headers=headers, params=params)
             try:
                 return r.json()
@@ -500,10 +506,9 @@ def _query_sql(self, sql: str) -> list[dict[str, Any]]:
 
         entity_set = self._entity_set_from_logical(logical)
         # Issue GET /{entity_set}?sql=<query>
-        headers = self._headers().copy()
         url = f"{self.api}/{entity_set}"
         params = {"sql": sql}
-        r = self._request("get", url, headers=headers, params=params)
+        r = self._request("get", url, params=params)
         try:
             body = r.json()
         except ValueError:
@@ -554,7 +559,7 @@ def _entity_set_from_logical(self, logical: str) -> str:
             "$select": "LogicalName,EntitySetName,PrimaryIdAttribute",
             "$filter": f"LogicalName eq '{logical_escaped}'",
         }
-        r = self._request("get", url, headers=self._headers(), params=params)
+        r = self._request("get", url, params=params)
         try:
             body = r.json()
             items = body.get("value", []) if isinstance(body, dict) else []
@@ -599,7 +604,7 @@ def _get_entity_by_schema(self, schema_name: str) -> Optional[Dict[str, Any]]:
             "$select": "MetadataId,LogicalName,SchemaName,EntitySetName",
             "$filter": f"SchemaName eq '{schema_escaped}'",
         }
-        r = self._request("get", url, headers=self._headers(), params=params)
+        r = self._request("get", url, params=params)
         items = r.json().get("value", [])
         return items[0] if items else None
 
@@ -617,8 +622,7 @@ def _create_entity(self, schema_name: str, display_name: str, attributes: List[D
             "IsActivity": False,
             "Attributes": attributes,
         }
-        headers = self._headers()
-        r = self._request("post", url, headers=headers, json=payload)
+        r = self._request("post", url, json=payload)
         ent = self._wait_for_entity_ready(schema_name)
         if not ent or not ent.get("EntitySetName"):
             raise RuntimeError(
@@ -791,20 +795,21 @@ def _optionset_map(self, logical_name: str, attr_logical: str) -> Optional[Dict[
         # Retry up to 3 times on 404 (new or not-yet-published attribute metadata). If still 404, raise.
         r_type = None
         for attempt in range(3):
-            r_type = self._raw_request("get", url_type, headers=self._headers())
-            if r_type.status_code != 404:
+            try:
+                r_type = self._request("get", url_type)
                 break
-            if attempt < 2:
-                # Exponential-ish backoff: 0.4s, 0.8s
-                time.sleep(0.4 * (2 ** attempt))
-        if r_type.status_code == 404:
-            # After retries we still cannot find the attribute definition – treat as fatal so caller sees a clear error.
-            raise RuntimeError(
-                f"Picklist attribute metadata not found after retries: entity='{logical_name}' attribute='{attr_logical}' (404)"
-            )
-        if not (200 <= r_type.status_code < 300):
-            # Re-issue via _send to raise structured HttpError (rare path)
-            self._request("get", url_type, headers=self._headers())
+            except HttpError as err:
+                if getattr(err, "status_code", None) == 404:
+                    if attempt < 2:
+                        # Exponential-ish backoff: 0.4s, 0.8s
+                        time.sleep(0.4 * (2 ** attempt))
+                        continue
+                    raise RuntimeError(
+                        f"Picklist attribute metadata not found after retries: entity='{logical_name}' attribute='{attr_logical}' (404)"
+                    ) from err
+                raise
+        if r_type is None:
+            raise RuntimeError("Failed to retrieve attribute metadata due to repeated request failures.")
 
         body_type = r_type.json()
         items = body_type.get("value", []) if isinstance(body_type, dict) else []
@@ -824,15 +829,20 @@ def _optionset_map(self, logical_name: str, attr_logical: str) -> Optional[Dict[
         # Step 2 fetch with retries: expanded OptionSet (cast form first)
         r_opts = None
         for attempt in range(3):
-            r_opts = self._raw_request("get", cast_url, headers=self._headers())
-            if r_opts.status_code != 404:
+            try:
+                r_opts = self._request("get", cast_url)
                 break
-            if attempt < 2:
-                time.sleep(0.4 * (2 ** attempt))  # 0.4s, 0.8s
-        if r_opts.status_code == 404:
-            raise RuntimeError(f"Picklist OptionSet metadata not found after retries: entity='{logical_name}' attribute='{attr_logical}' (404)")
-        if not (200 <= r_opts.status_code < 300):
-            self._request("get", cast_url, headers=self._headers())
+            except HttpError as err:
+                if getattr(err, "status_code", None) == 404:
+                    if attempt < 2:
+                        time.sleep(0.4 * (2 ** attempt))  # 0.4s, 0.8s
+                        continue
+                    raise RuntimeError(
+                        f"Picklist OptionSet metadata not found after retries: entity='{logical_name}' attribute='{attr_logical}' (404)"
+                    ) from err
+                raise
+        if r_opts is None:
+            raise RuntimeError("Failed to retrieve picklist OptionSet metadata due to repeated request failures.")
 
         attr_full = {}
         try:
@@ -993,7 +1003,7 @@ def _list_tables(self) -> List[Dict[str, Any]]:
         params = {
             "$filter": "IsPrivate eq false"
         }
-        r = self._request("get", url, headers=self._headers(), params=params)
+        r = self._request("get", url, params=params)
         return r.json().get("value", [])
 
     def _delete_table(self, tablename: str) -> None:
@@ -1004,8 +1014,7 @@ def _delete_table(self, tablename: str) -> None:
             raise RuntimeError(f"Table '{entity_schema}' not found.")
         metadata_id = ent["MetadataId"]
         url = f"{self.api}/EntityDefinitions({metadata_id})"
-        headers = self._headers()
-        r = self._request("delete", url, headers=headers)
+        r = self._request("delete", url)
 
     def _create_table(self, tablename: str, schema: Dict[str, Any]) -> Dict[str, Any]:
         # Accept a friendly name and construct a default schema under 'new_'.

src/dataverse_sdk/odata_upload_files.py

@@ -94,16 +94,16 @@ def _upload_file_small(
         fname = os.path.basename(path)
         key = self._format_key(record_id)
         url = f"{self.api}/{entity_set}{key}/{file_name_attribute}"
-        headers = self._headers().copy()
-        headers["Content-Type"] = content_type or "application/octet-stream"
-        headers["Accept"] = "application/json"
-        headers["x-ms-file-name"] = fname
+        headers = {
+            "Content-Type": content_type or "application/octet-stream",
+            "x-ms-file-name": fname,
+        }
         if if_none_match:
             headers["If-None-Match"] = "null"
         else:
             headers["If-Match"] = "*"
         # Single PATCH upload; allow default success codes (includes 204)
-        self._send("patch", url, headers=headers, data=data)
+        self._request("patch", url, headers=headers, data=data)
         return None
 
     def _upload_file_chunk(
@@ -146,14 +146,14 @@ def _upload_file_chunk(
         fname = os.path.basename(path)
         key = self._format_key(record_id)
         init_url = f"{self.api}/{entity_set}{key}/{file_name_attribute}?x-ms-file-name={quote(fname)}"
-        headers = self._headers().copy()
-        headers["x-ms-transfer-mode"] = "chunked"
-        headers["Accept"] = "application/json"
+        headers = {
+            "x-ms-transfer-mode": "chunked",
+        }
         if if_none_match:
             headers["If-None-Match"] = "null"
         else:
             headers["If-Match"] = "*"
-        r_init = self._send("patch", init_url, headers=headers, data=b"")
+        r_init = self._request("patch", init_url, headers=headers, data=b"")
         location = r_init.headers.get("Location") or r_init.headers.get("location")
         if not location:
             raise RuntimeError("Missing Location header with sessiontoken for chunked upload")
@@ -174,13 +174,13 @@ def _upload_file_chunk(
                     break
                 start = uploaded_bytes
                 end = start + len(chunk) - 1
-                c_headers = self._headers().copy()
-                c_headers["x-ms-file-name"] = fname
-                c_headers["Content-Type"] = "application/octet-stream"
-                c_headers["Accept"] = "application/json"
-                c_headers["Content-Range"] = f"bytes {start}-{end}/{total_size}"
-                c_headers["Content-Length"] = str(len(chunk))
+                c_headers = {
+                    "x-ms-file-name": fname,
+                    "Content-Type": "application/octet-stream",
+                    "Content-Range": f"bytes {start}-{end}/{total_size}",
+                    "Content-Length": str(len(chunk)),
+                }
                 # Each chunk returns 206 (partial) or 204 (final). Accept both.
-                self._send("patch", location, headers=c_headers, data=chunk, expected=(206, 204))
+                self._request("patch", location, headers=c_headers, data=chunk, expected=(206, 204))
                 uploaded_bytes += len(chunk)
         return None