Various improvements

* Allow override of defaults (for library use, incl. our tests)
* Improved logging for all log levels (incl. template rendering preview for --trace)
* Teardown of top-level build directory
* Minor tidy-ups/clarifications

Author: dannyadair

src/pdfbaker/__main__.py

@@ -30,12 +30,10 @@ def cli() -> None:
     "-t",
     "--trace",
     is_flag=True,
-    help="Show trace information (even more detailed than debug)",
+    help="Show trace information (even more detailed than --verbose)",
 )
 @click.option("--keep-build", is_flag=True, help="Keep build artifacts")
-@click.option(
-    "--debug", is_flag=True, help="Debug mode (implies --verbose and --keep-build)"
-)
+@click.option("--debug", is_flag=True, help="Debug mode (--verbose and --keep-build)")
 # pylint: disable=too-many-arguments,too-many-positional-arguments
 def bake(
     config_file: Path,

src/pdfbaker/baker.py

@@ -11,7 +11,7 @@
 from pathlib import Path
 from typing import Any
 
-from .config import PDFBakerConfiguration
+from .config import PDFBakerConfiguration, deep_merge
 from .document import PDFBakerDocument
 from .errors import ConfigurationError
 from .logging import TRACE, LoggingMixin
@@ -38,12 +38,15 @@ class PDFBakerOptions:
         verbose: Show debug information
         trace: Show trace information (even more detailed than debug)
         keep_build: Keep build artifacts after processing
+        default_config_overrides: Dictionary of values to override the built-in defaults
+            before loading the main configuration
     """
 
     quiet: bool = False
     verbose: bool = False
     trace: bool = False
     keep_build: bool = False
+    default_config_overrides: dict[str, Any] | None = None
 
 
 class PDFBaker(LoggingMixin):
@@ -57,13 +60,14 @@ def __init__(
         ) -> None:
             """Initialize baker configuration (needs documents)."""
             self.baker = baker
+            self.baker.log_debug_section("Loading main configuration: %s", config_file)
             super().__init__(base_config, config_file)
-            self.baker.log_trace_section("Main configuration: %s", config_file)
             self.baker.log_trace(self.pretty())
             if "documents" not in self:
                 raise ConfigurationError(
                     'Key "documents" missing - is this the main configuration file?'
                 )
+            self.build_dir = self["directories"]["build"]
             self.documents = [
                 self.resolve_path(doc_spec, directory=self["directories"]["documents"])
                 for doc_spec in self["documents"]
@@ -93,7 +97,13 @@ def __init__(
         else:
             logging.getLogger().setLevel(logging.INFO)
         self.keep_build = options.keep_build
+
+        # Start with defaults and apply any overrides
         base_config = DEFAULT_BAKER_CONFIG.copy()
+        if options and options.default_config_overrides:
+            base_config = deep_merge(base_config, options.default_config_overrides)
+
+        # Set config directory and initialize
         base_config["directories"]["config"] = config_file.parent.resolve()
         self.config = self.Configuration(
             baker=self,
@@ -102,7 +112,7 @@ def __init__(
         )
 
     def bake(self) -> None:
-        """Generate PDFs from documents."""
+        """Create PDFs for all documents."""
         pdfs_created: list[Path] = []
         failed_docs: list[tuple[str, str]] = []
 
@@ -115,7 +125,7 @@ def bake(self) -> None:
                 config_path=doc_config,
             )
             pdf_files, error_message = doc.process_document()
-            if pdf_files is None:
+            if error_message:
                 self.log_error(
                     "Failed to process document '%s': %s",
                     doc.config.name,
@@ -130,7 +140,7 @@ def bake(self) -> None:
                     doc.teardown()
 
         if pdfs_created:
-            self.log_info("Created PDFs:")
+            self.log_info("Successfully created PDFs:")
             for pdf in pdfs_created:
                 self.log_info("  %s", pdf)
         else:
@@ -144,3 +154,18 @@ def bake(self) -> None:
             )
             for doc_name, error in failed_docs:
                 self.log_error("  %s: %s", doc_name, error)
+
+        if not self.keep_build:
+            self.teardown()
+
+    def teardown(self) -> None:
+        """Clean up (top-level) build directory after processing."""
+        self.log_debug_subsection(
+            "Tearing down top-level build directory: %s", self.config.build_dir
+        )
+        if self.config.build_dir.exists():
+            try:
+                self.log_debug("Removing top-level build directory...")
+                self.config.build_dir.rmdir()
+            except OSError:
+                self.log_warning("Top-level build directory not empty - not removing")

src/pdfbaker/config.py

@@ -9,6 +9,7 @@
 from jinja2 import Template
 
 from .errors import ConfigurationError
+from .logging import truncate_strings
 from .types import PathSpec
 
 __all__ = ["PDFBakerConfiguration", "deep_merge", "render_config"]
@@ -27,24 +28,6 @@ def deep_merge(base: dict[str, Any], update: dict[str, Any]) -> dict[str, Any]:
     return result
 
 
-def _truncate_strings(obj, max_length: int) -> Any:
-    """Recursively truncate strings in nested structures."""
-    if isinstance(obj, str):
-        return obj if len(obj) <= max_length else obj[:max_length] + "…"
-    if isinstance(obj, dict):
-        return {
-            _truncate_strings(k, max_length): _truncate_strings(v, max_length)
-            for k, v in obj.items()
-        }
-    if isinstance(obj, list):
-        return [_truncate_strings(item, max_length) for item in obj]
-    if isinstance(obj, tuple):
-        return tuple(_truncate_strings(item, max_length) for item in obj)
-    if isinstance(obj, set):
-        return {_truncate_strings(item, max_length) for item in obj}
-    return obj
-
-
 class PDFBakerConfiguration(dict):
     """Base  class for handling config loading/merging/parsing."""
 
@@ -110,9 +93,9 @@ def resolve_path(self, spec: PathSpec, directory: Path | None = None) -> Path:
 
         return directory / spec["name"]
 
-    def pretty(self, max_string=60) -> str:
+    def pretty(self, max_chars: int = 60) -> str:
         """Return readable presentation (for debugging)."""
-        truncated = _truncate_strings(self, max_string)
+        truncated = truncate_strings(self, max_chars=max_chars)
         return pprint.pformat(truncated, indent=2)
 
 
@@ -162,6 +145,13 @@ def render_config(config: dict[str, Any]) -> dict[str, Any]:
         resolved_yaml = config_yaml.render(**current_config)
         new_config = yaml.safe_load(resolved_yaml)
 
+        # Check for direct self-references
+        for key, value in new_config.items():
+            if isinstance(value, str) and f"{{{{ {key} }}}}" in value:
+                raise ConfigurationError(
+                    f"Circular reference detected: {key} references itself"
+                )
+
         if new_config == current_config:  # No more changes
             return new_config
         current_config = new_config

src/pdfbaker/document.py

@@ -71,8 +71,10 @@ def __init__(
             base_config = deep_merge(base_config, DEFAULT_DOCUMENT_CONFIG)
             base_config["directories"]["config"] = config_path.parent.resolve()
 
+            self.document.log_trace_section(
+                "Loading document configuration: %s", config_path
+            )
             super().__init__(base_config, config_path)
-            self.document.log_trace_section("Document configuration: %s", config_path)
             self.document.log_trace(self.pretty())
 
             self.bake_path = self["directories"]["config"] / "bake.py"
@@ -168,15 +170,13 @@ def process(self) -> Path | list[Path]:
                     variant_config["variant"] = variant
                     variant_config = render_config(variant_config)
                     page_pdfs = self._process_pages(variant_config)
-                    pdf_files.append(
-                        self._combine_and_compress(page_pdfs, variant_config)
-                    )
+                    pdf_files.append(self._finalize(page_pdfs, variant_config))
                 return pdf_files
 
             # Single PDF document
             doc_config = render_config(self.config)
             page_pdfs = self._process_pages(doc_config)
-            return self._combine_and_compress(page_pdfs, doc_config)
+            return self._finalize(page_pdfs, doc_config)
         except Exception:
             # Ensure build directory is cleaned up if processing fails
             if not self.baker.keep_build:
@@ -189,7 +189,6 @@ def _process_pages(self, config: dict[str, Any]) -> list[Path]:
         self.log_debug_subsection("Pages to process:")
         self.log_debug(self.config.pages)
         for page_num, page_config in enumerate(self.config.pages, start=1):
-            # FIXME: just call with config - already merged
             page = PDFBakerPage(
                 document=self,
                 page_number=page_num,
@@ -200,10 +199,10 @@ def _process_pages(self, config: dict[str, Any]) -> list[Path]:
 
         return pdf_files
 
-    def _combine_and_compress(
-        self, pdf_files: list[Path], doc_config: dict[str, Any]
-    ) -> Path:
+    def _finalize(self, pdf_files: list[Path], doc_config: dict[str, Any]) -> Path:
         """Combine PDF pages and optionally compress."""
+        self.log_debug_subsection("Finalizing document...")
+        self.log_debug("Combining PDF pages...")
         try:
             combined_pdf = combine_pdfs(
                 pdf_files,
@@ -215,6 +214,7 @@ def _combine_and_compress(
         output_path = self.config.dist_dir / f"{doc_config['filename']}.pdf"
 
         if doc_config.get("compress_pdf", False):
+            self.log_debug("Compressing PDF document...")
             try:
                 compress_pdf(combined_pdf, output_path)
                 self.log_info("PDF compressed successfully")
@@ -227,23 +227,22 @@ def _combine_and_compress(
         else:
             os.rename(combined_pdf, output_path)
 
-        self.log_info("Created PDF: %s", output_path)
+        self.log_info("Created %s", output_path.name)
         return output_path
 
     def teardown(self) -> None:
-        """Clean up build directory after successful processing."""
+        """Clean up build directory after processing."""
         self.log_debug_subsection(
             "Tearing down build directory: %s", self.config.build_dir
         )
         if self.config.build_dir.exists():
-            # Remove all files in the build directory
+            self.log_debug("Removing files in build directory...")
             for file_path in self.config.build_dir.iterdir():
                 if file_path.is_file():
                     file_path.unlink()
 
-            # Try to remove the build directory
             try:
+                self.log_debug("Removing build directory...")
                 self.config.build_dir.rmdir()
             except OSError:
-                # Directory not empty - this is expected if we have subdirectories
-                pass
+                self.log_warning("Build directory not empty - not removing")

src/pdfbaker/logging.py

@@ -18,6 +18,14 @@ def log_trace(self, msg: str, *args: Any, **kwargs: Any) -> None:
         """Log a trace message (more detailed than debug)."""
         self.logger.log(TRACE, msg, *args, **kwargs)
 
+    def log_trace_preview(
+        self, msg: str, *args: Any, max_chars: int = 500, **kwargs: Any
+    ) -> None:
+        """Log a trace preview of a potentially large message, truncating if needed."""
+        self.logger.log(
+            TRACE, truncate_strings(msg, max_chars=max_chars), *args, **kwargs
+        )
+
     def log_trace_section(self, msg: str, *args: Any, **kwargs: Any) -> None:
         """Log a trace message as a main section header."""
         self.logger.log(TRACE, f"──── {msg} ────", *args, **kwargs)
@@ -61,3 +69,21 @@ def log_error(self, msg: str, *args: Any, **kwargs: Any) -> None:
     def log_critical(self, msg: str, *args: Any, **kwargs: Any) -> None:
         """Log a critical message."""
         self.logger.critical(msg, *args, **kwargs)
+
+
+def truncate_strings(obj, max_chars: int) -> Any:
+    """Recursively truncate strings in nested structures."""
+    if isinstance(obj, str):
+        return obj if len(obj) <= max_chars else obj[:max_chars] + "…"
+    if isinstance(obj, dict):
+        return {
+            truncate_strings(k, max_chars): truncate_strings(v, max_chars)
+            for k, v in obj.items()
+        }
+    if isinstance(obj, list):
+        return [truncate_strings(item, max_chars) for item in obj]
+    if isinstance(obj, tuple):
+        return tuple(truncate_strings(item, max_chars) for item in obj)
+    if isinstance(obj, set):
+        return {truncate_strings(item, max_chars) for item in obj}
+    return obj

src/pdfbaker/page.py

@@ -13,7 +13,7 @@
 
 from .config import PDFBakerConfiguration
 from .errors import ConfigurationError, SVGConversionError, SVGTemplateError
-from .logging import LoggingMixin
+from .logging import TRACE, LoggingMixin
 from .pdf import convert_svg_to_pdf
 from .render import create_env, prepare_template_context
 
@@ -39,8 +39,8 @@ def __init__(
             self.name = config_path.stem
             base_config["directories"]["config"] = config_path.parent.resolve()
 
+            self.page.log_trace_section("Loading page configuration: %s", config_path)
             super().__init__(base_config, config_path)
-            self.page.log_trace_section("Page configuration: %s", config_path)
             self.page.log_trace(self.pretty())
 
             self.templates_dir = self["directories"]["templates"]
@@ -83,9 +83,14 @@ def __init__(
 
     def process(self) -> Path:
         """Render SVG template and convert to PDF."""
-        self.config.build_dir.mkdir(parents=True, exist_ok=True)
-        output_svg = self.config.build_dir / f"{self.config.name}_{self.number:03}.svg"
-        output_pdf = self.config.build_dir / f"{self.config.name}_{self.number:03}.pdf"
+        self.log_debug_subsection(
+            "Processing page %d: %s", self.number, self.config.name
+        )
+
+        self.log_debug("Loading template: %s", self.config.template)
+        if self.logger.isEnabledFor(TRACE):
+            with open(self.config.template, encoding="utf-8") as f:
+                self.log_trace_preview(f.read())
 
         try:
             jinja_env = create_env(self.config.template.parent)
@@ -101,14 +106,22 @@ def process(self) -> Path:
             self.config.images_dir,
         )
 
+        self.config.build_dir.mkdir(parents=True, exist_ok=True)
+        output_svg = self.config.build_dir / f"{self.config.name}_{self.number:03}.svg"
+        output_pdf = self.config.build_dir / f"{self.config.name}_{self.number:03}.pdf"
+
+        self.log_debug("Rendering template...")
         try:
+            rendered_template = template.render(**template_context)
             with open(output_svg, "w", encoding="utf-8") as f:
-                f.write(template.render(**template_context))
+                f.write(rendered_template)
         except TemplateError as exc:
             raise SVGTemplateError(
                 f"Failed to render page {self.number} ({self.config.name}): {exc}"
             ) from exc
+        self.log_trace_preview(rendered_template)
 
+        self.log_debug("Converting SVG to PDF: %s", output_svg)
         svg2pdf_backend = self.config.get("svg2pdf_backend", "cairosvg")
         try:
             return convert_svg_to_pdf(