fix: Add retry logic to GitHub and GitLab tag providers

Add application-level retry logic to GitHubTagProvider and
GitLabTagProvider to handle transient API failures, particularly
common with gitlab.com. The implementation uses the existing
retry_on_exception decorator with 5 attempts and exponential backoff.

Changes:
- Apply @retry_on_exception decorator to GitHubTagProvider._find_tags()
- Apply @retry_on_exception decorator to GitLabTagProvider._find_tags()
- Add HTTP 408 (Request Timeout) to retry status codes
- Remove redundant try-except blocks that immediately re-raised
- Update documentation to reflect 408 status code handling

The two-layer retry strategy (HTTP-level + application-level)
significantly improves build reliability when GitHub/GitLab APIs
experience intermittent failures.

Fixes: #810

Co-authored-by: Claude <noreply@anthropic.com>

Signed-off-by: Ioannis Angelakopoulos <iangelak@redhat.com>

Author: iangelak

docs/http-retry.md

@@ -43,6 +43,7 @@ The retry mechanism specifically handles these error conditions:
 
 ### Server Errors
 
+- `408 Request Timeout` - Request took too long to complete
 - `504 Gateway Timeout` - Server overwhelmed or upstream timeout
 - `502 Bad Gateway` - Proxy/gateway errors
 - `503 Service Unavailable` - Temporary server overload

src/fromager/http_retry.py

@@ -2,6 +2,7 @@
 
 This module provides a robust retry mechanism for HTTP requests with exponential
 backoff, jitter, and specific handling for common network failures including:
+- Request timeouts (408)
 - Server timeouts (502, 503, 504)
 - Rate limiting (429, GitHub API rate limits)
 - Connection errors and incomplete reads
@@ -35,7 +36,7 @@
 DEFAULT_RETRY_CONFIG = {
     "total": 5,
     "backoff_factor": 1.0,
-    "status_forcelist": [429, 500, 502, 503, 504],
+    "status_forcelist": [408, 429, 500, 502, 503, 504],
     "allowed_methods": ["GET", "PUT", "POST", "HEAD", "OPTIONS"],
     "raise_on_status": False,
 }

src/fromager/request_session.py

@@ -6,7 +6,7 @@
 FROMAGER_RETRY_CONFIG = {
     "total": int(os.environ.get("FROMAGER_HTTP_RETRIES", "8")),
     "backoff_factor": float(os.environ.get("FROMAGER_HTTP_BACKOFF_FACTOR", "1.5")),
-    "status_forcelist": [429, 500, 502, 503, 504],
+    "status_forcelist": [408, 429, 500, 502, 503, 504],
     "allowed_methods": ["GET", "PUT", "POST", "HEAD", "OPTIONS"],
     "raise_on_status": False,
 }

src/fromager/resolver.py

@@ -33,6 +33,7 @@
 from .candidate import Candidate
 from .constraints import Constraints
 from .extras_provider import ExtrasProvider
+from .http_retry import RETRYABLE_EXCEPTIONS, retry_on_exception
 from .request_session import session
 from .requirements_file import RequirementType
 
@@ -760,6 +761,12 @@ def __init__(
     def cache_key(self) -> str:
         return f"{self.organization}/{self.repo}"
 
+    @retry_on_exception(
+        exceptions=RETRYABLE_EXCEPTIONS,
+        max_attempts=5,
+        backoff_factor=1.5,
+        max_backoff=120.0,
+    )
     def _find_tags(
         self,
         identifier: str,
@@ -773,17 +780,8 @@ def _find_tags(
 
         nexturl = self.api_url.format(self=self)
         while nexturl:
-            try:
-                resp = session.get(nexturl, headers=headers)
-                resp.raise_for_status()
-            except Exception as e:
-                logger.error(
-                    "%s: Failed to fetch GitHub tags from %s: %s",
-                    identifier,
-                    nexturl,
-                    e,
-                )
-                raise
+            resp = session.get(nexturl, headers=headers)
+            resp.raise_for_status()
 
             for entry in resp.json():
                 name = entry["name"]
@@ -834,23 +832,20 @@ def __init__(
     def cache_key(self) -> str:
         return f"{self.server_url}/{self.project_path}"
 
+    @retry_on_exception(
+        exceptions=RETRYABLE_EXCEPTIONS,
+        max_attempts=5,
+        backoff_factor=1.5,
+        max_backoff=120.0,
+    )
     def _find_tags(
         self,
         identifier: str,
     ) -> Iterable[tuple[str, Version]]:
         nexturl: str = self.api_url
         while nexturl:
-            try:
-                resp: Response = session.get(nexturl)
-                resp.raise_for_status()
-            except Exception as e:
-                logger.error(
-                    "%s: Failed to fetch GitLab tags from %s: %s",
-                    identifier,
-                    nexturl,
-                    e,
-                )
-                raise
+            resp: Response = session.get(nexturl)
+            resp.raise_for_status()
             for entry in resp.json():
                 name = entry["name"]
                 version = self._match_function(identifier, name)