Merge pull request Pipelex#50 from Pipelex/release/v0.2.12

Release/v0.2.12

Author: thomashebrard

.gitignore

@@ -40,7 +40,7 @@ Thumbs.db
 reports/
 results/
 
-# Pipelex libraries, duplicated from pipelex/libraries/ by `pipelex init`
+# Pipelex libraries, duplicated from pipelex/libraries/ by `pipelex init-libraries`
 pipelex_libraries/
 
 # personnal pipelex config file that overrides the default one

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## [v0.2.12] - 2025-06-03
+
+- pipelex cli: Split `pipelex init` into 2 separate functions: `pipelex init-libraries` and `pipelex init-config`
+- Fixed the inheritance config manager method
+- Rename Mission to Pipeline
+- Enable to start a pipeline and let in run in the background, getting it's run id, but not waiting for the output
+- `Makefile`: avoid defaulting pytest to verbose. Setup target `make test-xdist` = Run unit tests with `xdist`, make it the default for shorthand `make t`. The old `make t` is now `make tp` (test-with-prints)
+- Added `mistral-small-3.1` and `qwen3:8b`
+- Fix template pre-processor: don't try and substitute a dollar numerical like $10 or @25
+- Refactor with less "OpenAI" naming for non-openai stuff that just uses the OpenAI SDK
+
 ## [v0.2.11] - 2025-06-02
 
 - HotFix for v0.2.10 👇 regarding the new pipelex/pipelex_init.toml`
@@ -13,7 +24,7 @@
 **Major Model Additions** - Claude 4 (Opus & Sonnet), Grok-3, and GPT-4 image generation are now in the house.
 
 ### Pipeline Base Library update
-- **New pipe** - `extract_page_contents_and_views_from_pdf` transfered from cookbook to base library (congrats on the promotion!). This pipe extracts text, linked images, **AND** page_view images (rendered pages) - it's very useful if you want to use Vision in follow-up pipes
+- **New pipe** - `extract_page_contents_and_views_from_pdf` transferred from cookbook to base library (congrats on the promotion!). This pipe extracts text, linked images, **AND** page_view images (rendered pages) - it's very useful if you want to use Vision in follow-up pipes
 
 ### Added
 
@@ -56,16 +67,16 @@
 
 - Include `pyproject.toml` inside the project build.
 - Fix `ImggEngineFactory`: image generation (imgg) handle required format is `platform/model_name`
-- pipelex cli: Added `list-pipes` method that can list all the availables pipes with descriptions.
+- pipelex cli: Added `list-pipes` method that can list all the available pipes along with their descriptions.
 - Use a minimum version for `uv` instead of a fixed version
 - Implement `AGENTS.md` for Codex
 - Add tests for some of the `tools.misc`
 - pipelex cli: Rename `pipelex run-setup` to `pipelex validate`
 
 ## [v0.2.8] - 2025-05-28
 
-- Defaults to `uv`
-- Simplify llm provider config: All the API keys, urls, and regions are to be set in `.env`.
+- Replaced `poetry` by `uv` for dependency management.
+- Simplify llm provider config: All the API keys, urls, and regions now live in the `.env`.
 - Added logging level `OFF`, prevents any log from hitting the console
 
 ## [v0.2.7] - 2025-05-26

Makefile

@@ -61,14 +61,16 @@ make merge-check-pyright	  - Run pyright merge check without updating files
 
 make rl                       - Shorthand -> reinitlibraries
 make v                        - Shorthand -> validate
-make init                     - Run pipelex init
+make init                     - Run `pipelex init-libraries` and `pipelex init-config`
 make codex-tests              - Run tests for Codex (exit on first failure) (no inference, no codex_disabled)
 make gha-tests		          - Run tests for github actions (exit on first failure) (no inference, no gha_disabled)
 make test                     - Run unit tests (no inference)
+make test-xdist               - Run unit tests with xdist (no inference)
+make t                        - Shorthand -> test-xdist
 make test-quiet               - Run unit tests without prints (no inference)
 make tq                       - Shorthand -> test-quiet
 make test-with-prints         - Run tests with prints (no inference)
-make t                        - Shorthand -> test-with-prints
+make tp                       - Shorthand -> test-with-prints
 make test-inference           - Run unit tests only for inference (with prints)
 make ti                       - Shorthand -> test-inference
 make test-ocr                 - Run unit tests only for ocr (with prints)
@@ -90,7 +92,7 @@ export HELP
 	all help env lock install update build \
 	format lint pyright mypy \
 	cleanderived cleanenv cleanlibraries cleanall \
-	test test-quiet tq test-with-prints t test-inference ti \
+	test test-xdist t test-quiet tq test-with-prints tp test-inference ti \
 	test-imgg tg test-ocr to codex-tests gha-tests \
 	run-all-tests run-manual-trigger-gha-tests run-gha_disabled-tests \
 	validate v check c cc \
@@ -125,14 +127,16 @@ env: check-uv
 	@echo "Using Python: $$($(VENV_PYTHON) --version) from $$(which $$(readlink -f $(VENV_PYTHON)))"
 
 init: env
-	$(call PRINT_TITLE,"Running pipelex init")
-	$(VENV_PIPELEX) init
+	$(call PRINT_TITLE,"Running pipelex init-libraries and init-config")
+	$(VENV_PIPELEX) init-libraries
+	$(VENV_PIPELEX) init-config
 
 install: env
 	$(call PRINT_TITLE,"Installing dependencies")
 	@. $(VIRTUAL_ENV)/bin/activate && \
 	uv sync --all-extras && \
-	$(VENV_PIPELEX) init && \
+	$(VENV_PIPELEX) init-libraries && \
+	$(VENV_PIPELEX) init-config && \
 	echo "Installed Pipelex dependencies in ${VIRTUAL_ENV} with all extras and initialized Pipelex";
 
 lock: env
@@ -225,18 +229,30 @@ test: env
 	$(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")
 	@echo "• Running unit tests"
 	@if [ -n "$(TEST)" ]; then \
-		$(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	else \
-		$(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	fi
 
+test-xdist: env
+	$(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")
+	@echo "• Running unit tests"
+	@if [ -n "$(TEST)" ]; then \
+		$(VENV_PYTEST) -n auto -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+	else \
+		$(VENV_PYTEST) -n auto -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+	fi
+
+t: test-xdist
+	@echo "> done: t = test-xdist"
+
 test-quiet: env
 	$(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")
 	@echo "• Running unit tests"
 	@if [ -n "$(TEST)" ]; then \
-		$(VENV_PYTEST) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	else \
-		$(VENV_PYTEST) -o log_cli=true -o log_level=WARNING $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	fi
 
 tq: test-quiet
@@ -246,20 +262,20 @@ test-with-prints: env
 	$(call PRINT_TITLE,"Unit testing with prints and our rich logs")
 	@echo "• Running unit tests"
 	@if [ -n "$(TEST)" ]; then \
-		$(VENV_PYTEST) -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	else \
-		$(VENV_PYTEST) -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	fi
 
-t: test-with-prints
-	@echo "> done: t = test-with-prints"
+tp: test-with-prints
+	@echo "> done: tp = test-with-prints"
 
 test-inference: env
 	$(call PRINT_TITLE,"Unit testing")
 	@if [ -n "$(TEST)" ]; then \
-		$(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	else \
-		$(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	fi
 
 ti: test-inference
@@ -268,9 +284,9 @@ ti: test-inference
 test-ocr: env
 	$(call PRINT_TITLE,"Unit testing ocr")
 	@if [ -n "$(TEST)" ]; then \
-		$(VENV_PYTEST) --exitfirst -m "ocr" -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) --exitfirst -m "ocr" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	else \
-		$(VENV_PYTEST) --exitfirst -m "ocr" -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) --exitfirst -m "ocr" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	fi
 
 to: test-ocr
@@ -279,9 +295,9 @@ to: test-ocr
 test-imgg: env
 	$(call PRINT_TITLE,"Unit testing")
 	@if [ -n "$(TEST)" ]; then \
-		$(VENV_PYTEST) --exitfirst -m "imgg" -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) --exitfirst -m "imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	else \
-		$(VENV_PYTEST) --exitfirst -m "imgg" -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+		$(VENV_PYTEST) --exitfirst -m "imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
 	fi
 
 tg: test-imgg

pipelex/_bootstrap_user_libs.py

@@ -16,7 +16,7 @@ def activate() -> None:
     # 2) If a pipelex_libraries folder already exists, also touch __init__.py
     #    (helps IDE & static type-checkers) and ensure the *exact* folder is
     #    on sys.path.  When it does **not** exist yet, nothing else to do —
-    #    as soon as the user runs `pipelex init`, the directory will be
+    #    as soon as the user runs `pipelex init-libraries`, the directory will be
     #    created *inside* <root>, which is already import-able thanks to (1).
     for parent in (root, *root.parents):
         lib_dir = parent / "pipelex_libraries"

pipelex/cli/_cli.py

@@ -23,6 +23,7 @@ class PipelexCLI(TyperGroup):
     def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]:
         cmd = super().get_command(ctx, cmd_name)
         if cmd is None:
+            typer.echo(f"Unknown command: {cmd_name}")
             typer.echo(ctx.get_help())
             ctx.exit(1)
         return cmd
@@ -36,22 +37,40 @@ def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]:
 )
 
 
-@app.command()
-def init(
+@app.command("init-libraries")
+def init_libraries(
     overwrite: Annotated[bool, typer.Option("--overwrite", "-o", help="Warning: If set, existing files will be overwritten.")] = False,
 ) -> None:
-    """Initialize pipelex configuration in the current directory."""
-    LibraryConfig.export_libraries(overwrite=overwrite)
+    """Initialize pipelex libraries in the current directory.
+
+    If overwrite is False, only create files that don't exist yet.
+    If overwrite is True, all files will be overwritten even if they exist.
+    """
+    try:
+        # TODO: Have a more proper print message regarding the overwrited files (e.g. list of files that were overwritten or not)
+        LibraryConfig.export_libraries(overwrite=overwrite)
+        if overwrite:
+            typer.echo("Successfully initialized pipelex libraries (all files overwritten)")
+        else:
+            typer.echo("Successfully initialized pipelex libraries (only created non-existing files)")
+    except Exception as e:
+        raise PipelexCLIError(f"Failed to initialize libraries: {e}")
+
 
-    pipelex_init_path = os.path.join(config_manager.pipelex_root_dir, "pipelex_init.toml")
+@app.command("init-config")
+def init_config(
+    reset: Annotated[bool, typer.Option("--reset", "-r", help="Warning: If set, existing files will be overwritten.")] = False,
+) -> None:
+    """Initialize pipelex configuration in the current directory."""
+    pipelex_template_path = os.path.join(config_manager.pipelex_root_dir, "pipelex_template.toml")
     target_config_path = os.path.join(config_manager.local_root_dir, "pipelex.toml")
 
-    if os.path.exists(target_config_path) and not overwrite:
-        typer.echo("Warning: pipelex.toml already exists. Use --overwrite to force creation.")
+    if os.path.exists(target_config_path) and not reset:
+        typer.echo("Warning: pipelex.toml already exists. Use --reset to force creation.")
         return
 
     try:
-        shutil.copy2(pipelex_init_path, target_config_path)
+        shutil.copy2(pipelex_template_path, target_config_path)
         typer.echo(f"Created pipelex.toml at {target_config_path}")
     except Exception as e:
         raise PipelexCLIError(f"Failed to create pipelex.toml: {e}")

pipelex/cogt/content_generation/assignment_models.py

@@ -16,7 +16,7 @@
 from pipelex.cogt.ocr.ocr_handle import OcrHandle
 from pipelex.cogt.ocr.ocr_input import OcrInput
 from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
 from pipelex.tools.templating.jinja2_environment import Jinja2TemplateCategory
 from pipelex.tools.templating.templating_models import PromptingStyle
 

pipelex/cogt/content_generation/content_generator.py

@@ -30,7 +30,7 @@
 from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams
 from pipelex.cogt.ocr.ocr_output import OcrOutput
 from pipelex.config import get_config
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
 from pipelex.tools.templating.jinja2_environment import Jinja2TemplateCategory
 from pipelex.tools.templating.templating_models import PromptingStyle
 from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar

pipelex/cogt/content_generation/content_generator_protocol.py

@@ -12,7 +12,7 @@
 from pipelex.cogt.ocr.ocr_input import OcrInput
 from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams
 from pipelex.cogt.ocr.ocr_output import OcrOutput
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
 from pipelex.tools.templating.jinja2_environment import Jinja2TemplateCategory
 from pipelex.tools.templating.templating_models import PromptingStyle
 from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar

pipelex/cogt/imgg/imgg_job_factory.py

@@ -4,7 +4,7 @@
 from pipelex.cogt.imgg.imgg_job_components import ImggJobConfig, ImggJobParams, ImggJobReport
 from pipelex.cogt.imgg.imgg_prompt import ImggPrompt
 from pipelex.config import get_config
-from pipelex.mission.job_metadata import JobCategory, JobMetadata
+from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
 
 
 class ImggJobFactory:

pipelex/cogt/imgg/imgg_worker_abstract.py

@@ -10,7 +10,7 @@
 from pipelex.cogt.imgg.imgg_job import ImggJob
 from pipelex.cogt.inference.inference_report_delegate import InferenceReportDelegate
 from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract
-from pipelex.mission.job_metadata import UnitJobId
+from pipelex.pipeline.job_metadata import UnitJobId
 
 F = TypeVar("F", bound=Callable[..., Awaitable[Any]])