fix/Pipelex config fixed path(Pipelex#151)

Author: thomashebrard

.github/workflows/manual-trigger-tests-check.yml

@@ -0,0 +1,66 @@
+name: Manual trigger tests check
+
+on:
+  pull_request:
+    branches:
+      - dev
+      - "release/v[0-9]+.[0-9]+.[0-9]+"
+  workflow_dispatch:
+
+jobs:
+  trigger-tests:
+    runs-on: ubuntu-latest
+    environment:
+      name: manual-trigger-tests-check
+      url: ${{ github.server_url }}/${{ github.repository }}/deployments/activity_log?environment=manual-trigger-tests-check
+    steps:
+      - name: Initialize check
+        run: |
+          echo "Initializing manual trigger tests check..."
+
+  matrix-test:
+    name: Manual trigger tests check
+    needs: [trigger-tests]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    permissions:
+      contents: read
+      id-token: write
+      packages: read
+      actions: read
+      repository-projects: read
+    env:
+      VIRTUAL_ENV: ${{ github.workspace }}/.venv
+      ENV: dev
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::975050286103:role/GitHubActionsRole
+          aws-region: eu-west-3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Check UV installation
+        run: make check-uv
+
+      - name: Verify UV installation
+        run: uv --version
+
+      - name: Install dependencies
+        run: PYTHON_VERSION=${{ matrix.python-version }} make install
+
+      - name: Boot test
+        run: make tp TEST=TestFundamentals
+
+      - name: Run tests
+        run: make run-manual-trigger-gha-tests

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [v0.6.0] - 2025-07-12
+
+### Changed 
+- The `Pipelex.make()` now takes a `relative_config_folder_path` argument, which is considered to be a relative path to the current working directory, pointing to the config folder path of pipelex. The argument `from_file` indicates if the path is relative to the caller's file path. If False, the path is considered to be absolute from where the process was started.
+
+### Added
+- Added github action for inference tests
+- `load_json_list_from_path` function in `pipelex.tools.misc.file_utils`: Loads a JSON file and ensures it contains a list.
+
 ## [v0.5.2] - 2025-07-11
 
 - log a warning when dry running a `PipeFunc`

pipelex/cli/_cli.py

@@ -47,7 +47,7 @@ def init_libraries(
     """
     try:
         # TODO: Have a more proper print message regarding the overwrited files (e.g. list of files that were overwritten or not)
-        LibraryConfig.export_libraries(overwrite=overwrite)
+        LibraryConfig().export_libraries(overwrite=overwrite)
         if overwrite:
             typer.echo("Successfully initialized pipelex libraries (all files overwritten)")
         else:
@@ -76,10 +76,15 @@ def init_config(
 
 
 @app.command()
-def validate() -> None:
+def validate(
+    relative_config_folder_path: Annotated[
+        str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+    ] = "pipelex_libraries",
+) -> None:
     """Run the setup sequence."""
-    LibraryConfig.export_libraries()
-    pipelex_instance = Pipelex.make()
+    config_folder_path = os.path.join(os.getcwd(), relative_config_folder_path)
+    LibraryConfig(config_folder_path=config_folder_path).export_libraries()
+    pipelex_instance = Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
     pipelex_instance.validate_libraries()
     asyncio.run(dry_run_all_pipes())
     log.info("Setup sequence passed OK, config and pipelines are validated.")
@@ -96,9 +101,13 @@ def show_config() -> None:
 
 
 @app.command()
-def list_pipes() -> None:
+def list_pipes(
+    relative_config_folder_path: Annotated[
+        str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+    ] = "pipelex_libraries",
+) -> None:
     """List all available pipes."""
-    Pipelex.make()
+    Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
 
     try:
         get_pipe_provider().pretty_list_pipes()

pipelex/cogt/llm/llm_models/llm_model_library.py

@@ -1,5 +1,5 @@
 import os
-from typing import Any, Dict, List, Optional
+from typing import Any, ClassVar, Dict, List, Optional
 
 from pydantic import Field, RootModel
 from typing_extensions import override
@@ -32,6 +32,12 @@ class LLMModelLibraryError(ToolException):
 
 class LLMModelLibrary(LLMModelProviderAbstract, RootModel[LLMModelLibraryRoot]):
     root: LLMModelLibraryRoot = Field(default_factory=list)
+    library_config: ClassVar[LibraryConfig]
+
+    @classmethod
+    def make_empty(cls, config_folder_path: str) -> "LLMModelLibrary":
+        cls.library_config = LibraryConfig(config_folder_path=config_folder_path)
+        return cls()
 
     @override
     def setup(self):
@@ -63,7 +69,7 @@ def desc(self) -> str:
 
     @classmethod
     def load_llm_model_library_dict(cls) -> LLMModelLibraryDict:
-        libraries_path = LibraryConfig.exported_llm_integrations_path
+        libraries_path = cls.library_config.llm_integrations_path
         if not os.path.exists(libraries_path):
             raise LLMModelLibraryError(f"LLM model library path `{libraries_path}` not found. Please run `pipelex init-libraries` to create it.")
         llm_library: LLMModelLibraryDict = {}

pipelex/libraries/library_config.py

@@ -3,99 +3,117 @@
 from pipelex.tools.config.models import ConfigModel
 from pipelex.tools.misc.file_utils import copy_file_from_package, copy_folder_from_package, find_files_in_dir
 
+PIPELEX_LIBRARIES_PATH = "libraries"
+
 
 class LibraryConfig(ConfigModel):
-    # Class variables
     package_name: ClassVar[str] = "pipelex"
-    internal_library_root: ClassVar[str] = "libraries"
-    exported_library_root: ClassVar[str] = "pipelex_libraries"
-    internal_base_pipelines_path: ClassVar[str] = f"{internal_library_root}/pipelines"
-    exported_pipelines_path: ClassVar[str] = f"{exported_library_root}/pipelines"
-    exported_base_pipelines_path: ClassVar[str] = f"{exported_library_root}/pipelines/base_library"
-    loaded_pipelines_path: ClassVar[str] = f"{exported_library_root}/pipelines"
-    test_pipelines_path: ClassVar[str] = "tests/test_pipelines"
-    internal_llm_integrations_path: ClassVar[str] = f"{internal_library_root}/llm_integrations"
-    exported_llm_integrations_path: ClassVar[str] = f"{exported_library_root}/llm_integrations"
-    internal_llm_deck_path: ClassVar[str] = f"{internal_library_root}/llm_deck"
-    exported_llm_deck_path: ClassVar[str] = f"{exported_library_root}/llm_deck"
-    internal_templates_path: ClassVar[str] = f"{internal_library_root}/templates"
-    exported_templates_path: ClassVar[str] = f"{exported_library_root}/templates"
-    internal_plugins_path: ClassVar[str] = f"{internal_library_root}/plugins"
-    exported_plugins_path: ClassVar[str] = f"{exported_library_root}/plugins"
-    failing_pipelines_path: ClassVar[str] = "tests/test_pipelines/failing_pipelines.toml"
-
-    @classmethod
-    def get_llm_deck_paths(cls) -> List[str]:
-        llm_deck_paths = [str(path) for path in find_files_in_dir(dir_path=cls.exported_llm_deck_path, pattern="*.toml", is_recursive=True)]
+    config_folder_path: str = "pipelex_libraries"
+
+    @property
+    def config_file(self) -> str:
+        return f"{self.config_folder_path}/config.toml"
+
+    @property
+    def pipelines_path(self) -> str:
+        return f"{self.config_folder_path}/pipelines"
+
+    @property
+    def base_pipelines_path(self) -> str:
+        return f"{self.config_folder_path}/pipelines/base_library"
+
+    @property
+    def llm_integrations_path(self) -> str:
+        return f"{self.config_folder_path}/llm_integrations"
+
+    @property
+    def llm_deck_path(self) -> str:
+        return f"{self.config_folder_path}/llm_deck"
+
+    @property
+    def templates_path(self) -> str:
+        return f"{self.config_folder_path}/templates"
+
+    @property
+    def plugins_path(self) -> str:
+        return f"{self.config_folder_path}/plugins"
+
+    @property
+    def test_pipelines_path(self) -> str:
+        return "tests/test_pipelines"
+
+    @property
+    def failing_pipelines_path(self) -> str:
+        return "tests/test_pipelines/failing_pipelines.toml"
+
+    def get_llm_deck_paths(self) -> List[str]:
+        llm_deck_paths = [str(path) for path in find_files_in_dir(dir_path=self.llm_deck_path, pattern="*.toml", is_recursive=True)]
         llm_deck_paths.sort()
         return llm_deck_paths
 
-    @classmethod
-    def get_templates_paths(cls) -> List[str]:
-        return [str(path) for path in find_files_in_dir(dir_path=cls.exported_templates_path, pattern="*.toml", is_recursive=True)]
+    def get_templates_paths(self) -> List[str]:
+        return [str(path) for path in find_files_in_dir(dir_path=self.templates_path, pattern="*.toml", is_recursive=True)]
 
-    @classmethod
-    def get_plugin_config_path(cls) -> str:
-        return f"{cls.exported_plugins_path}/plugin_config.toml"
+    def get_default_plugin_config_path(self) -> str:
+        """Get the default plugin config path."""
+        return f"{self.plugins_path}/plugin_config.toml"
 
-    @classmethod
-    def export_libraries(cls, overwrite: bool = False) -> None:
+    def export_libraries(self, overwrite: bool = False) -> None:
         """Duplicate pipelex libraries files in the client project, preserving directory structure."""
         # pipelines
         copy_folder_from_package(
-            package_name=cls.package_name,
-            folder_path_in_package=cls.internal_base_pipelines_path,
-            target_dir=cls.exported_base_pipelines_path,
+            package_name=self.package_name,
+            folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines",
+            target_dir=self.base_pipelines_path,
             overwrite=overwrite,
         )
         copy_file_from_package(
-            package_name=cls.package_name,
-            file_path_in_package=f"{cls.internal_library_root}/__init__.py",
-            target_path=f"{cls.exported_library_root}/__init__.py",
+            package_name=self.package_name,
+            file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/__init__.py",
+            target_path=f"{self.config_folder_path}/__init__.py",
             overwrite=overwrite,
         )
         copy_file_from_package(
-            package_name=cls.package_name,
-            file_path_in_package=f"{cls.internal_base_pipelines_path}/__init__.py",
-            target_path=f"{cls.exported_pipelines_path}/__init__.py",
+            package_name=self.package_name,
+            file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines/__init__.py",
+            target_path=f"{self.pipelines_path}/__init__.py",
             overwrite=overwrite,
         )
         copy_file_from_package(
-            package_name=cls.package_name,
-            file_path_in_package=f"{cls.internal_base_pipelines_path}/__init__.py",
-            target_path=f"{cls.exported_base_pipelines_path}/__init__.py",
+            package_name=self.package_name,
+            file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines/__init__.py",
+            target_path=f"{self.base_pipelines_path}/__init__.py",
             overwrite=overwrite,
         )
-
         # llm_integrations
         copy_folder_from_package(
-            package_name=cls.package_name,
-            folder_path_in_package=cls.internal_llm_integrations_path,
-            target_dir=cls.exported_llm_integrations_path,
+            package_name=self.package_name,
+            folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/llm_integrations",
+            target_dir=self.llm_integrations_path,
             overwrite=overwrite,
         )
 
         # llm_deck
         copy_folder_from_package(
-            package_name=cls.package_name,
-            folder_path_in_package=cls.internal_llm_deck_path,
-            target_dir=cls.exported_llm_deck_path,
+            package_name=self.package_name,
+            folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/llm_deck",
+            target_dir=self.llm_deck_path,
             overwrite=overwrite,
             non_overwrite_files=["overrides.toml"],
         )
 
         # templates
         copy_folder_from_package(
-            package_name=cls.package_name,
-            folder_path_in_package=cls.internal_templates_path,
-            target_dir=cls.exported_templates_path,
+            package_name=self.package_name,
+            folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/templates",
+            target_dir=self.templates_path,
             overwrite=overwrite,
         )
 
         # plugins
         copy_folder_from_package(
-            package_name=cls.package_name,
-            folder_path_in_package=cls.internal_plugins_path,
-            target_dir=cls.exported_plugins_path,
+            package_name=self.package_name,
+            folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/plugins",
+            target_dir=self.plugins_path,
             overwrite=overwrite,
         )