Feature/storage provider ingestion (Pipelex#118)

Author: Abdujabbar

docs/pages/advanced-customization/storage-provider-ingestion.md

@@ -0,0 +1,3 @@
+# Storage Provider Injection
+
+⚠️ Under construction

pipelex/hub.py

@@ -4,12 +4,16 @@
 from kajson.class_registry_abstract import ClassRegistryAbstract
 
 from pipelex import log
-from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol
+from pipelex.cogt.content_generation.content_generator_protocol import (
+    ContentGeneratorProtocol,
+)
 from pipelex.cogt.imgg.imgg_worker_abstract import ImggWorkerAbstract
 from pipelex.cogt.inference.inference_manager_protocol import InferenceManagerProtocol
 from pipelex.cogt.llm.llm_models.llm_deck_abstract import LLMDeckAbstract
 from pipelex.cogt.llm.llm_models.llm_engine_blueprint import LLMEngineBlueprint
-from pipelex.cogt.llm.llm_models.llm_model_provider_abstract import LLMModelProviderAbstract
+from pipelex.cogt.llm.llm_models.llm_model_provider_abstract import (
+    LLMModelProviderAbstract,
+)
 from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
 from pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstract
 from pipelex.cogt.plugin_manager import PluginManager
@@ -28,6 +32,7 @@
 from pipelex.tools.config.manager import config_manager
 from pipelex.tools.config.models import ConfigRoot
 from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
+from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract
 from pipelex.tools.templating.template_provider_abstract import TemplateProviderAbstract
 
 
@@ -46,6 +51,7 @@ def __init__(self):
         self._secrets_provider: Optional[SecretsProviderAbstract] = None
         self._template_provider: Optional[TemplateProviderAbstract] = None
         self._class_registry: Optional[ClassRegistryAbstract] = None
+        self._storage_provider: Optional[StorageProviderAbstract] = None
         # cogt
         self._llm_models_provider: Optional[LLMModelProviderAbstract] = None
         self._llm_deck_provider: Optional[LLMDeckAbstract] = None
@@ -112,6 +118,9 @@ def reset_config(self) -> None:
     def set_secrets_provider(self, secrets_provider: SecretsProviderAbstract):
         self._secrets_provider = secrets_provider
 
+    def set_storage_provider(self, storage_provider: StorageProviderAbstract | None):
+        self._storage_provider = storage_provider
+
     def set_template_provider(self, template_provider: TemplateProviderAbstract):
         self._template_provider = template_provider
 
@@ -198,6 +207,11 @@ def get_required_class_registry(self) -> ClassRegistryAbstract:
             raise RuntimeError("ClassRegistry is not initialized")
         return self._class_registry
 
+    def get_storage_provider(self) -> StorageProviderAbstract:
+        if self._storage_provider is None:
+            raise RuntimeError("StorageProvider is not initialized")
+        return self._storage_provider
+
     # cogt
 
     def get_required_llm_models_provider(self) -> LLMModelProviderAbstract:
@@ -297,6 +311,10 @@ def get_secrets_provider() -> SecretsProviderAbstract:
     return get_pipelex_hub().get_required_secrets_provider()
 
 
+def get_storage_provider() -> StorageProviderAbstract:
+    return get_pipelex_hub().get_storage_provider()
+
+
 def get_template_provider() -> TemplateProviderAbstract:
     return get_pipelex_hub().get_required_template_provider()
 
@@ -391,7 +409,9 @@ def get_pipe_provider() -> PipeProviderAbstract:
     return get_pipelex_hub().get_required_pipe_provider()
 
 
-def get_pipes_by_domain(excluded_domains: Optional[List[str]] = None) -> Dict[str, List[str]]:
+def get_pipes_by_domain(
+    excluded_domains: Optional[List[str]] = None,
+) -> Dict[str, List[str]]:
     pipes = get_pipe_provider().get_pipes()
     pipes_by_domain: Dict[str, List[str]] = defaultdict(list)
     for pipe in pipes:

pipelex/pipelex.py

@@ -11,7 +11,9 @@
 
 from pipelex import log
 from pipelex.cogt.content_generation.content_generator import ContentGenerator
-from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol
+from pipelex.cogt.content_generation.content_generator_protocol import (
+    ContentGeneratorProtocol,
+)
 from pipelex.cogt.inference.inference_manager import InferenceManager
 from pipelex.cogt.llm.llm_models.llm_model import LATEST_VERSION_NAME
 from pipelex.cogt.llm.llm_models.llm_model_library import LLMModelLibrary
@@ -24,10 +26,16 @@
 from pipelex.pipe_works.pipe_router import PipeRouter
 from pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol
 from pipelex.pipeline.activity.activity_manager import ActivityManager
-from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerNoOp, ActivityManagerProtocol
+from pipelex.pipeline.activity.activity_manager_protocol import (
+    ActivityManagerNoOp,
+    ActivityManagerProtocol,
+)
 from pipelex.pipeline.pipeline_manager import PipelineManager
 from pipelex.pipeline.track.pipeline_tracker import PipelineTracker
-from pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerNoOp, PipelineTrackerProtocol
+from pipelex.pipeline.track.pipeline_tracker_protocol import (
+    PipelineTrackerNoOp,
+    PipelineTrackerProtocol,
+)
 from pipelex.reporting.reporting_manager import ReportingManager
 from pipelex.reporting.reporting_protocol import ReportingNoOp, ReportingProtocol
 from pipelex.test_extras.registry_test_models import PipelexTestModels
@@ -36,6 +44,7 @@
 from pipelex.tools.runtime_manager import runtime_manager
 from pipelex.tools.secrets.env_secrets_provider import EnvSecretsProvider
 from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
+from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract
 from pipelex.tools.templating.template_library import TemplateLibrary
 from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
 
@@ -172,10 +181,11 @@ def setup(
         content_generator: Optional[ContentGeneratorProtocol] = None,
         pipe_router: Optional[PipeRouterProtocol] = None,
         structure_classes: Optional[List[Type[Any]]] = None,
+        storage_provider: Optional[StorageProviderAbstract] = None,
     ):
         # tools
         self.pipelex_hub.set_secrets_provider(secrets_provider or EnvSecretsProvider())
-
+        self.pipelex_hub.set_storage_provider(storage_provider)
         # cogt
         self.pipelex_hub.set_content_generator(content_generator or ContentGenerator())
         self.reporting_delegate.setup()

pipelex/tools/storage/__init__.py

@@ -0,0 +1,11 @@
+from abc import ABC, abstractmethod
+
+
+class StorageProviderAbstract(ABC):
+    @abstractmethod
+    def load(self, uri: str) -> bytes:
+        pass
+
+    @abstractmethod
+    def store(self, data: bytes) -> str:
+        pass