HTTP downloads: no overall time limit

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Fixed
+
+- Longer HTTP download limit, 5 minute can be too short
+
 ## [v0.4.1] - 2026-10-25
 
 ### Added

vecorel_cli/conversion/base.py

@@ -22,6 +22,7 @@
 import rarfile
 import shapely
 from fsspec import AbstractFileSystem
+from fsspec.exceptions import FSTimeoutError
 from fsspec.implementations.local import LocalFileSystem
 from geopandas import GeoDataFrame
 
@@ -443,11 +444,13 @@ def _download_file(source_fs, uri, cache_fs, cache_file):
         try:
             with cache_fs.open(part_file, mode="wb") as file:
                 stream_file(source_fs, uri, file)
-        except BaseException:
+        except BaseException as e:
             try:
                 cache_fs.rm(part_file)
             except FileNotFoundError:
                 pass
+            if isinstance(e, FSTimeoutError):
+                raise TimeoutError(f"Download timed out: {uri}") from e
             raise
         cache_fs.mv(part_file, cache_file)
 

vecorel_cli/vecorel/util.py

@@ -5,6 +5,7 @@
 from typing import Optional, Union
 from urllib.parse import urlparse
 
+import aiohttp
 import yaml
 from fsspec import AbstractFileSystem
 from fsspec.implementations.http import HTTPFileSystem
@@ -131,6 +132,10 @@ def merge_headers(*headers: dict) -> dict:
     return merged
 
 
+HTTP_CONNECT_TIMEOUT = 60
+HTTP_READ_TIMEOUT = 300
+
+
 def get_fs(url_or_path: Union[str, Path, URL], **kwargs) -> AbstractFileSystem:
     """Choose fsspec filesystem by sniffing input url"""
     if isinstance(url_or_path, Path):
@@ -142,7 +147,12 @@ def get_fs(url_or_path: Union[str, Path, URL], **kwargs) -> AbstractFileSystem:
     if parsed.scheme in ("http", "https"):
         client_kwargs = kwargs.pop("client_kwargs", {})
         headers = merge_headers({"User-Agent": USER_AGENT}, client_kwargs.get("headers", {}))
-        return HTTPFileSystem(client_kwargs={**client_kwargs, "headers": headers}, **kwargs)
+        timeout = client_kwargs.get("timeout") or aiohttp.ClientTimeout(
+            total=None, sock_connect=HTTP_CONNECT_TIMEOUT, sock_read=HTTP_READ_TIMEOUT
+        )
+        return HTTPFileSystem(
+            client_kwargs={**client_kwargs, "headers": headers, "timeout": timeout}, **kwargs
+        )
 
     if parsed.scheme == "s3":
         from s3fs import S3FileSystem