Merge pull request Pipelex#179 from Pipelex/release/v0.6.2

Release/v0.6.2

Author: thomashebrard

.github/workflows/manual-trigger-tests-check.yml

@@ -3,10 +3,11 @@ name: Manual trigger tests check
 on:
   pull_request:
     branches:
-      - main
-      - dev
-      - "release/v[0-9]+.[0-9]+.[0-9]+"
-  workflow_dispatch:
+      - manual-trigger-tests-check
+      # - main
+      # - dev
+      # - "release/v[0-9]+.[0-9]+.[0-9]+"
+  # workflow_dispatch:
 
 jobs:
   trigger-tests:

.gitignore

@@ -44,7 +44,7 @@ reports/
 results/
 
 # Pipelex libraries, duplicated from pipelex/libraries/ by `pipelex init-libraries`
-pipelex_libraries/
+/pipelex_libraries
 
 # personnal pipelex config file that overrides the default one
 pipelex_super.toml

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## [v0.6.2] - 2025-07-18
+
+### Added
+- New `dry-run-pipe` cli command to dry run a single pipe by its code
+- New `show-pipe` cli command to display pipe definitions from the pipe library
+- New `dry_run_single_pipe()` function for running individual pipe dry runs
+
+### Changed
+- Updated `init-libraries` command to accept a directory argument and create `pipelex_libraries` folder in specified location
+- Updated `validate` command to use `-c` flag for the config folder path
+
 ## [v0.6.1] - 2025-07-16
 
 - Can execute pipelines with `input_memory`: It is a `CompactMemory: Dict[str, Dict[str, Any]]`

Makefile

@@ -137,18 +137,11 @@ env: check-uv
 	fi
 	@echo "Using Python: $$($(VENV_PYTHON) --version) from $$(which $$(readlink -f $(VENV_PYTHON)))"
 
-init: env
-	$(call PRINT_TITLE,"Running pipelex init-libraries and init-config")
-	$(VENV_PIPELEX) init-libraries
-	$(VENV_PIPELEX) init-config
-
 install: env
 	$(call PRINT_TITLE,"Installing dependencies")
 	@. $(VIRTUAL_ENV)/bin/activate && \
 	uv sync --all-extras && \
-	$(VENV_PIPELEX) init-libraries && \
-	$(VENV_PIPELEX) init-config && \
-	echo "Installed Pipelex dependencies in ${VIRTUAL_ENV} with all extras and initialized Pipelex";
+	echo "Installed Pipelex dependencies in ${VIRTUAL_ENV} with all extras.";
 
 lock: env
 	$(call PRINT_TITLE,"Resolving dependencies without update")
@@ -161,9 +154,14 @@ update: env
 	uv sync --all-extras && \
 	echo "Updated dependencies in ${VIRTUAL_ENV}";
 
+init: env
+	$(call PRINT_TITLE,"Running pipelex init-libraries and init-config")
+	$(VENV_PIPELEX) init-libraries
+	$(VENV_PIPELEX) init-config
+
 validate: env
 	$(call PRINT_TITLE,"Running setup sequence")
-	$(VENV_PIPELEX) validate
+	$(VENV_PIPELEX) validate -c pipelex/libraries
 
 build: env
 	$(call PRINT_TITLE,"Building the wheels")
@@ -439,16 +437,16 @@ check-unused-imports: env
 	$(call PRINT_TITLE,"Checking for unused imports without fixing")
 	$(VENV_RUFF) check --select=F401 --no-fix .
 
-c: init format lint pyright mypy
+c: format lint pyright mypy
 	@echo "> done: c = check"
 
-cc: init cleanderived c
-	@echo "> done: cc = init cleanderived init format lint pyright mypy"
+cc: cleanderived c
+	@echo "> done: cc = cleanderived format lint pyright mypy"
 
 check: cc check-unused-imports
 	@echo "> done: check"
 
-v: init validate
+v: validate
 	@echo "> done: v = validate"
 
 li: lock install

pipelex/cli/_cli.py

@@ -10,13 +10,42 @@
 
 from pipelex import log, pretty_print
 from pipelex.exceptions import PipelexCLIError, PipelexConfigError
-from pipelex.hub import get_pipe_provider
+from pipelex.hub import get_pipe_provider, get_pipeline_tracker, get_required_pipe
 from pipelex.libraries.library_config import LibraryConfig
-from pipelex.pipe_works.pipe_dry import dry_run_all_pipes
+from pipelex.pipe_works.pipe_dry import dry_run_all_pipes, dry_run_single_pipe
 from pipelex.pipelex import Pipelex
 from pipelex.tools.config.manager import config_manager
 
 
+def is_pipelex_libraries_folder(folder_path: str) -> bool:
+    """Check if the given folder path contains a valid pipelex libraries structure.
+
+    A valid pipelex libraries folder should contain the following subdirectories:
+    - pipelines
+    - llm_deck
+    - llm_integrations
+    - plugins
+    - templates
+
+    Args:
+        folder_path: Path to the folder to check
+
+    Returns:
+        True if the folder contains all required subdirectories, False otherwise
+    """
+    if not os.path.exists(folder_path) or not os.path.isdir(folder_path):
+        return False
+
+    required_subdirs = ["pipelines", "llm_deck", "llm_integrations", "plugins", "templates"]
+
+    for subdir in required_subdirs:
+        subdir_path = os.path.join(folder_path, subdir)
+        if not os.path.exists(subdir_path) or not os.path.isdir(subdir_path):
+            return False
+
+    return True
+
+
 class PipelexCLI(TyperGroup):
     @override
     def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]:
@@ -38,22 +67,31 @@ def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]:
 
 @app.command("init-libraries")
 def init_libraries(
+    directory: Annotated[str, typer.Argument(help="Directory where to create the pipelex_libraries folder")] = ".",
     overwrite: Annotated[bool, typer.Option("--overwrite", "-o", help="Warning: If set, existing files will be overwritten.")] = False,
 ) -> None:
-    """Initialize pipelex libraries in the current directory.
+    """Initialize pipelex libraries in a pipelex_libraries folder in the specified directory.
 
     If overwrite is False, only create files that don't exist yet.
     If overwrite is True, all files will be overwritten even if they exist.
     """
     try:
-        # TODO: Have a more proper print message regarding the overwrited files (e.g. list of files that were overwritten or not)
-        LibraryConfig().export_libraries(overwrite=overwrite)
+        # Always create a pipelex_libraries folder in the specified directory
+        target_path = os.path.join(directory, "pipelex_libraries")
+
+        # Create the target directory if it doesn't exist
+        os.makedirs(directory, exist_ok=True)
+
+        # Create a LibraryConfig instance with the target path
+        library_config = LibraryConfig(config_folder_path=target_path)
+        library_config.export_libraries(overwrite=overwrite)
+
         if overwrite:
-            typer.echo("Successfully initialized pipelex libraries (all files overwritten)")
+            typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_path}' (all files overwritten)")
         else:
-            typer.echo("Successfully initialized pipelex libraries (only created non-existing files)")
+            typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_path}' (only created non-existing files)")
     except Exception as e:
-        raise PipelexCLIError(f"Failed to initialize libraries: {e}")
+        raise PipelexCLIError(f"Failed to initialize libraries at '{directory}': {e}")
 
 
 @app.command("init-config")
@@ -79,17 +117,49 @@ def init_config(
 def validate(
     relative_config_folder_path: Annotated[
         str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
-    ] = "pipelex_libraries",
+    ] = "./pipelex_libraries",
 ) -> None:
     """Run the setup sequence."""
-    config_folder_path = os.path.join(os.getcwd(), relative_config_folder_path)
-    LibraryConfig(config_folder_path=config_folder_path).export_libraries()
+    # Check if pipelex libraries folder exists
+    if not is_pipelex_libraries_folder(relative_config_folder_path):
+        typer.echo(f"❌ No pipelex libraries folder found at '{relative_config_folder_path}'")
+        typer.echo("To create a pipelex libraries folder, run: pipelex init-libraries")
+        raise typer.Exit(1)
+
     pipelex_instance = Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
     pipelex_instance.validate_libraries()
     asyncio.run(dry_run_all_pipes())
     log.info("Setup sequence passed OK, config and pipelines are validated.")
 
 
+@app.command()
+def dry_run_pipe(
+    pipe_code: Annotated[str, typer.Argument(help="The pipe code to dry run")],
+    relative_config_folder_path: Annotated[
+        str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+    ] = "./pipelex_libraries",
+) -> None:
+    """Dry run a single pipe."""
+    # Check if pipelex libraries folder exists
+    if not is_pipelex_libraries_folder(relative_config_folder_path):
+        typer.echo(f"❌ No pipelex libraries folder found at '{relative_config_folder_path}'")
+        typer.echo("To create a pipelex libraries folder, run: pipelex init-libraries")
+        raise typer.Exit(1)
+
+    try:
+        # Initialize Pipelex
+        pipelex_instance = Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
+        pipelex_instance.validate_libraries()
+
+        # Run the single pipe dry run
+        asyncio.run(dry_run_single_pipe(pipe_code))
+        get_pipeline_tracker().output_flowchart()
+
+    except Exception as e:
+        typer.echo(f"❌ Error running dry run for pipe '{pipe_code}': {e}")
+        raise typer.Exit(1)
+
+
 @app.command()
 def show_config() -> None:
     """Show the pipelex configuration."""
@@ -116,6 +186,22 @@ def list_pipes(
         raise PipelexCLIError(f"Failed to list pipes: {e}")
 
 
+@app.command("show-pipe")
+def show_pipe(
+    pipe_code: Annotated[
+        str,
+        typer.Argument(help="Pipeline code to show definition for"),
+    ],
+    relative_config_folder_path: Annotated[
+        str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+    ] = "./pipelex_libraries",
+) -> None:
+    """Show pipe from the pipe library."""
+    Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
+    pipe = get_required_pipe(pipe_code=pipe_code)
+    pretty_print(pipe, title=f"Pipe '{pipe_code}'")
+
+
 def main() -> None:
     """Entry point for the pipelex CLI."""
     app()

pipelex/pipe_works/pipe_dry.py

@@ -21,6 +21,29 @@ async def dry_run_all_pipes():
     await dry_run_pipes(pipes=all_pipes)
 
 
+async def dry_run_single_pipe(pipe_code: str) -> str:
+    """
+    Dry run a single pipe by its code.
+
+    Args:
+        pipe_code: The code of the pipe to dry run
+
+    Returns:
+        Status string: "SUCCESS" or error message
+    """
+    try:
+        # Get the pipe using the hub function
+        pipe = get_pipe_provider().get_optional_pipe(pipe_code=pipe_code)
+        if not pipe:
+            return f"FAILED: Pipe '{pipe_code}' not found"
+
+        # Run the single pipe
+        result = await dry_run_pipes(pipes=[pipe])
+        return result.get(pipe_code, f"FAILED: No result for pipe '{pipe_code}'")
+    except Exception as e:
+        return f"FAILED: {str(e)}"
+
+
 # TODO: add a function to dry run a single pipe, make it callable as a param of `pipelex validate`
 async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:
     """

pipelex/pipelex_template.toml

@@ -1,3 +1,10 @@
+[pipelex]
+[pipelex.feature_config]
+# WIP/Experimental feature flags
+is_pipeline_tracking_enabled = true
+is_activity_tracking_enabled = true
+is_reporting_enabled = true
+
 [pipelex.aws_config]
 api_key_method = "env"
 # The possible values are "env" and "secret_provider".

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pipelex"
-version = "0.6.1"
+version = "0.6.2"
 description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines."
 authors = [{ name = "Evotis S.A.S.", email = "evotis@pipelex.com" }]
 maintainers = [{ name = "Pipelex staff", email = "oss@pipelex.com" }]