From b83e4e1a3fbd83c3d030a2186d8f5d6039715f3a Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 13:38:35 -0400 Subject: [PATCH 01/12] Generate shared Python serialization export keys from interfaces --- CMakeLists.txt | 3 +- DOCS.md | 36 ++++++ cmake/PybindWrap.cmake | 29 +++++ cmake/gtwrapConfig.cmake.in | 1 + gtwrap/pybind_wrapper.py | 54 ++++++++- scripts/pybind_serialization.py | 39 ++++++ tests/fixtures/serialization_shared.h | 29 +++++ tests/fixtures/serialization_shared.i | 16 +++ tests/test_serialization_support.py | 163 ++++++++++++++++++++++++++ 9 files changed, 366 insertions(+), 4 deletions(-) create mode 100644 scripts/pybind_serialization.py create mode 100644 tests/fixtures/serialization_shared.h create mode 100644 tests/fixtures/serialization_shared.i create mode 100644 tests/test_serialization_support.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c8b33a..4a1eeaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,7 +75,8 @@ endif() # Install wrapping scripts as binaries to `CMAKE_INSTALL_PREFIX/bin` so they can # be invoked for wrapping. We use DESTINATION (instead of TYPE) so we can # support older CMake versions. -install(PROGRAMS scripts/pybind_wrap.py scripts/matlab_wrap.py +install(PROGRAMS scripts/pybind_wrap.py scripts/pybind_serialization.py + scripts/matlab_wrap.py DESTINATION "${GTWRAP_BIN_INSTALL_DIR}") # Install the matlab.h file to `CMAKE_INSTALL_PREFIX/lib/gtwrap/matlab.h`. diff --git a/DOCS.md b/DOCS.md index 8303045..c16beb7 100644 --- a/DOCS.md +++ b/DOCS.md @@ -252,6 +252,42 @@ The python wrapper supports keyword arguments for functions/methods. Hence, the - Unfortunately, this means that aliases can no longer be used. - Similarly, there can be multiple `preamble.h` and `specializations.h` files. Each of these should match the module file name. +### Shared Python Boost export keys + +When one binding translation unit serializes a polymorphic type exported by +another, both must see the same `BOOST_CLASS_EXPORT_KEY` specialization before +archive instantiation. `pybind_serialization.py` derives keys and optional +implementation macros from the same interface markers used by the ordinary +pybind generator. Select a narrow namespace to avoid including unrelated +types in every binding translation unit: + +```cmake +pybind_generate_serialization_support( + "${PROJECT_SOURCE_DIR}/gtsam/linear/linear.i" + "gtsam::noiseModel::" + "${CMAKE_CURRENT_BINARY_DIR}/noise_model_keys.h" + "${CMAKE_CURRENT_BINARY_DIR}/noise_model_implementations.inc" + "gtsam/linear/NoiseModel.h") +add_custom_target(noise_model_keys DEPENDS + "${CMAKE_CURRENT_BINARY_DIR}/noise_model_keys.h" + "${CMAKE_CURRENT_BINARY_DIR}/noise_model_implementations.inc") +add_dependencies(stable_python_module noise_model_keys) +add_dependencies(unstable_python_module noise_model_keys) +``` + +Add the build directory to the modules' include paths. Include the generated +key header in each serialization-enabled binding translation unit, after its +normal preamble and before `{boost_class_export}` or other serialization uses. +The translation unit whose generated `BOOST_CLASS_EXPORT` macros already own +these types must **skip** the key header: that macro declares the keys again. +In a second extension, include the key header followed by the generated +implementation fragment exactly once, after the Boost archive headers. This +retains one set of serializer implementations per extension and the existing +archive names, without an additional library or link dependency. Keep these +files private to Python bindings when C++ clients use different archive IDs. +Generate and include them only when Boost serialization is enabled. MATLAB +generation is independent of this Python helper. + ## Pybind Callable Adapters The Python generator uses explicit full-signature C++ callable-pointer casts for diff --git a/cmake/PybindWrap.cmake b/cmake/PybindWrap.cmake index 2c98b69..a530dd7 100644 --- a/cmake/PybindWrap.cmake +++ b/cmake/PybindWrap.cmake @@ -5,6 +5,35 @@ else() set(GTWRAP_PACKAGE_DIR ${CMAKE_CURRENT_LIST_DIR}/..) endif() +# Generate one shared export-key header and one implementation fragment from a +# selected interface namespace. Callers include the header in every binding +# translation unit that may serialize these types, and include the fragment +# once in each extension that lacks its own BOOST_CLASS_EXPORT owner. +# ARGN contains C++ headers defining the selected types. +function(pybind_generate_serialization_support interface namespace_prefix + output_header output_implementations) + if(UNIX) + set(_serialization_path_separator ":") + else() + set(_serialization_path_separator ";") + endif() + set(_serialization_includes "") + foreach(_serialization_include IN LISTS ARGN) + list(APPEND _serialization_includes --include "${_serialization_include}") + endforeach() + add_custom_command( + OUTPUT "${output_header}" "${output_implementations}" + COMMAND ${CMAKE_COMMAND} -E env + "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${_serialization_path_separator}$ENV{PYTHONPATH}" + ${PYTHON_EXECUTABLE} ${PYBIND_SERIALIZATION_SCRIPT} + --src "${interface}" --namespace-prefix "${namespace_prefix}" + --out-header "${output_header}" + --out-implementations "${output_implementations}" + ${_serialization_includes} + DEPENDS "${interface}" "${PYBIND_SERIALIZATION_SCRIPT}" + VERBATIM) +endfunction() + # Get the Python version include(GtwrapUtils) message(STATUS "Checking Python Version") diff --git a/cmake/gtwrapConfig.cmake.in b/cmake/gtwrapConfig.cmake.in index 182fa7e..c3b1bf4 100644 --- a/cmake/gtwrapConfig.cmake.in +++ b/cmake/gtwrapConfig.cmake.in @@ -16,6 +16,7 @@ include(@PACKAGE_INSTALL_CMAKE_DIR@/GtwrapUtils.cmake) # Set the variables for the wrapping scripts to be used in the build. set_and_check(PYBIND_WRAP_SCRIPT "@PACKAGE_INSTALL_BIN_DIR@/pybind_wrap.py") +set_and_check(PYBIND_SERIALIZATION_SCRIPT "@PACKAGE_INSTALL_BIN_DIR@/pybind_serialization.py") set_and_check(MATLAB_WRAP_SCRIPT "@PACKAGE_INSTALL_BIN_DIR@/matlab_wrap.py") # Load the same pybind11 provider selected when gtwrap was configured. diff --git a/gtwrap/pybind_wrapper.py b/gtwrap/pybind_wrapper.py index 7b6dbf5..b411008 100755 --- a/gtwrap/pybind_wrapper.py +++ b/gtwrap/pybind_wrapper.py @@ -916,10 +916,9 @@ def wrap_file(self, # Export classes for serialization. boost_class_export = "" for cpp_class in self._serializing_classes: - new_name = cpp_class + new_name = self.serialization_export_name(cpp_class) # The boost's macro doesn't like commas, so we have to typedef. - if ',' in cpp_class: - new_name = re.sub("[,:<> ]", "", cpp_class) + if new_name != cpp_class: boost_class_export += "typedef {cpp_class} {new_name};\n".format( # noqa cpp_class=cpp_class, new_name=new_name) @@ -987,6 +986,55 @@ def wrap_file(self, submodules_init="", ) + def serialization_export_types(self, content, namespace_prefix): + """Find serializable types in one interface using the normal wrapper path. + + A prefix selects the shared polymorphic family. In particular, this + avoids exporting unrelated classes from a large interface such as + GTSAM's linear.i. The caller owns the export implementations; this + method only discovers the types and does not change normal wrapping. + """ + if not namespace_prefix or not namespace_prefix.endswith('::'): + raise ValueError('namespace_prefix must end in ::') + module = parser.Module.parse_string(content) + module = instantiator.instantiate_namespace(module) + self._serializing_classes = [] + try: + self.wrap_namespace(module) + return [cpp_class for cpp_class in self._serializing_classes + if cpp_class.startswith(namespace_prefix)] + finally: + self._serializing_classes = [] + + @staticmethod + def serialization_export_name(cpp_class): + """Return the existing Boost export name, including comma-safe alias.""" + if ',' in cpp_class: + return re.sub('[,:<> ]', '', cpp_class) + return cpp_class + + def serialization_support(self, content, namespace_prefix, includes=()): + """Generate shared export keys and a single-owner implementation list. + + Include the key header in every serialization translation unit before + archive instantiation. Include the implementation fragment exactly once + per extension, after its archive headers. The ordinary generated full + exports in that owner must not also declare the same keys. + """ + classes = self.serialization_export_types(content, namespace_prefix) + header = ['#pragma once', '', '#include '] + for include in includes: + header.append(f'#include <{include}>') + header.append('') + implementations = [] + for cpp_class in classes: + name = self.serialization_export_name(cpp_class) + if name != cpp_class: + header.append(f'typedef {cpp_class} {name};') + header.append(f'BOOST_CLASS_EXPORT_KEY({name})') + implementations.append(f'BOOST_CLASS_EXPORT_IMPLEMENT({name})') + return '\n'.join(header) + '\n', '\n'.join(implementations) + '\n' + def wrap_submodule(self, source): """ Wrap a list of submodule files, i.e. a set of interface files which are diff --git a/scripts/pybind_serialization.py b/scripts/pybind_serialization.py new file mode 100644 index 0000000..b11d2e7 --- /dev/null +++ b/scripts/pybind_serialization.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Generate shared Boost export keys from a pybind interface. + +The implementation fragment is for an extension that does not already own +full BOOST_CLASS_EXPORT registrations for these types. It must be included +once, after that extension's archive headers and the generated key header. +""" + +import argparse +from pathlib import Path + +from gtwrap.pybind_wrapper import PybindWrapper + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--src', required=True, type=Path) + parser.add_argument('--namespace-prefix', required=True) + parser.add_argument('--include', action='append', default=[], + help='C++ header defining the selected types; repeatable') + parser.add_argument('--out-header', required=True, type=Path) + parser.add_argument('--out-implementations', required=True, type=Path) + args = parser.parse_args() + + wrapper = PybindWrapper(module_name='', top_module_namespaces=[''], + use_boost_serialization=True) + keys, implementations = wrapper.serialization_support( + args.src.read_text(encoding='utf-8'), args.namespace_prefix, + args.include) + if not implementations.strip(): + parser.error('no serializable types matched --namespace-prefix') + args.out_header.parent.mkdir(parents=True, exist_ok=True) + args.out_implementations.parent.mkdir(parents=True, exist_ok=True) + args.out_header.write_text(keys, encoding='utf-8') + args.out_implementations.write_text(implementations, encoding='utf-8') + + +if __name__ == '__main__': + main() diff --git a/tests/fixtures/serialization_shared.h b/tests/fixtures/serialization_shared.h new file mode 100644 index 0000000..fadd51d --- /dev/null +++ b/tests/fixtures/serialization_shared.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +namespace example { +struct Base { + virtual ~Base() = default; + template + void serialize(Archive&, unsigned int) {} +}; + +struct First : Base { + int value = 42; + template + void serialize(Archive& archive, unsigned int) { + archive & boost::serialization::base_object(*this); + archive & value; + } +}; + +struct Second : Base { + int value = 7; + template + void serialize(Archive& archive, unsigned int) { + archive & boost::serialization::base_object(*this); + archive & value; + } +}; +} diff --git a/tests/fixtures/serialization_shared.i b/tests/fixtures/serialization_shared.i new file mode 100644 index 0000000..e145d98 --- /dev/null +++ b/tests/fixtures/serialization_shared.i @@ -0,0 +1,16 @@ +namespace example { +#include +virtual class Base {}; +virtual class First : example::Base { + void serializable() const; +}; +virtual class Second : example::Base { + void serializable() const; +}; +} + +namespace unrelated { +class Third { + void serializable() const; +}; +} diff --git a/tests/test_serialization_support.py b/tests/test_serialization_support.py new file mode 100644 index 0000000..b86393f --- /dev/null +++ b/tests/test_serialization_support.py @@ -0,0 +1,163 @@ +"""Shared Boost export generation and cross-translation-unit registration.""" + +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +from gtwrap.pybind_wrapper import PybindWrapper + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = ROOT / 'tests' / 'fixtures' + + +class TestSerializationSupport(unittest.TestCase): + + def test_selected_keys_and_implementations(self): + source = (FIXTURES / 'serialization_shared.i').read_text() + wrapper = PybindWrapper('', top_module_namespaces=[''], + use_boost_serialization=True) + keys, implementations = wrapper.serialization_support( + source, 'example::', ['serialization_shared.h']) + self.assertIn('#include ', keys) + self.assertEqual(keys.count('BOOST_CLASS_EXPORT_KEY('), 2) + self.assertEqual(implementations.count('BOOST_CLASS_EXPORT_IMPLEMENT('), 2) + self.assertIn('BOOST_CLASS_EXPORT_KEY(example::First)', keys) + self.assertIn('BOOST_CLASS_EXPORT_KEY(example::Second)', keys) + self.assertNotIn('unrelated::Third', keys + implementations) + self.assertEqual(wrapper._serializing_classes, []) + + def test_comma_containing_type_keeps_existing_archive_name(self): + source = '''namespace example { +template class Pair {}; +@serializable +typedef example::Pair PairID; +}''' + wrapper = PybindWrapper('', top_module_namespaces=[''], + use_boost_serialization=True) + keys, implementations = wrapper.serialization_support( + source, 'example::') + self.assertIn('typedef example::Pair examplePairintdouble;', + keys) + self.assertIn('BOOST_CLASS_EXPORT_KEY(examplePairintdouble)', keys) + self.assertEqual(implementations, + 'BOOST_CLASS_EXPORT_IMPLEMENT(examplePairintdouble)\n') + + def test_cli_and_cross_translation_unit_round_trip(self): + compiler = shutil.which('c++') + boost_lib = next((path for path in + ('/opt/homebrew/lib', '/usr/local/lib', + '/usr/lib/x86_64-linux-gnu', '/usr/lib') + if list(Path(path).glob('libboost_serialization.*'))), + None) + if not compiler or not boost_lib: + self.skipTest('C++ compiler or Boost.Serialization library unavailable') + + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + subprocess.run([ + sys.executable, str(ROOT / 'scripts' / 'pybind_serialization.py'), + '--src', str(FIXTURES / 'serialization_shared.i'), + '--namespace-prefix', 'example::', + '--include', 'serialization_shared.h', + '--out-header', str(directory / 'keys.h'), + '--out-implementations', str(directory / 'implementations.inc'), + ], check=True, env={**os.environ, 'PYTHONPATH': str(ROOT)}) + + # The first owner mirrors stable's existing full exports. The + # second mirrors unstable's keys plus implementation fragment. + # Each is compiled separately with a consumer translation unit. + consumers = r''' +#include "keys.h" +#include +#include +#include +#include +#include +int main() { + if (std::string(boost::serialization::guid()) != + "example::First") return 1; + std::shared_ptr value = std::make_shared(); + std::stringstream stream; + { boost::archive::text_oarchive archive(stream); archive << value; } + value.reset(); + { boost::archive::text_iarchive archive(stream); archive >> value; } + auto result = std::dynamic_pointer_cast(value); + return result && result->value == 42 ? 0 : 2; +} +''' + (directory / 'consumer.cpp').write_text(consumers) + archive_headers = '''#include +#include +''' + owners = { + 'full': archive_headers + '''#include "serialization_shared.h" +#include +BOOST_CLASS_EXPORT(example::First) +BOOST_CLASS_EXPORT(example::Second) +''', + 'split': archive_headers + '''#include "keys.h" +#include "implementations.inc" +''', + } + libraries = {} + for name, owner in owners.items(): + with self.subTest(owner=name): + (directory / 'owner.cpp').write_text(owner) + executable = directory / name + build = subprocess.run([ + compiler, '-std=c++17', '-I', str(directory), + '-I', str(Path(boost_lib).parent / 'include'), + '-I', str(FIXTURES), str(directory / 'owner.cpp'), + str(directory / 'consumer.cpp'), '-L', boost_lib, + '-lboost_serialization', '-o', str(executable), + ], capture_output=True, text=True) + self.assertEqual(build.returncode, 0, build.stderr) + subprocess.run([str(executable)], check=True, + env={**os.environ, + 'DYLD_LIBRARY_PATH': boost_lib, + 'LD_LIBRARY_PATH': boost_lib}) + + # Two extensions each own their implementations. Loading + # either first must not change the archive identifier or + # prevent the other from round-tripping the derived type. + (directory / 'consumer.cpp').write_text( + consumers.replace('int main()', + 'extern "C" int roundtrip()')) + library = directory / f'lib{name}{".dylib" if sys.platform == "darwin" else ".so"}' + shared = subprocess.run([ + compiler, '-std=c++17', '-shared', '-fPIC', + '-I', str(directory), + '-I', str(Path(boost_lib).parent / 'include'), + '-I', str(FIXTURES), str(directory / 'owner.cpp'), + str(directory / 'consumer.cpp'), '-L', boost_lib, + f'-Wl,-rpath,{boost_lib}', '-lboost_serialization', + '-o', str(library), + ], capture_output=True, text=True) + self.assertEqual(shared.returncode, 0, shared.stderr) + libraries[name] = library + (directory / 'consumer.cpp').write_text(consumers) + + load_script = '''import ctypes, sys +for path in sys.argv[1:]: + library = ctypes.CDLL(path) + assert library.roundtrip() == 0 +''' + for order in (('full',), ('full', 'split'), ('split', 'full')): + with self.subTest(import_order=order): + run = subprocess.run([ + sys.executable, '-c', load_script, + *(str(libraries[name]) for name in order), + ], capture_output=True, text=True, + env={**os.environ, 'DYLD_LIBRARY_PATH': boost_lib, + 'LD_LIBRARY_PATH': boost_lib}) + self.assertEqual(run.returncode, 0, + run.stdout + run.stderr) + + +if __name__ == '__main__': + unittest.main() From 2745b0c1df9c0c2f58d260f68ab8f7596f8922c0 Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 13:56:00 -0400 Subject: [PATCH 02/12] Extract generic serialization export discovery --- gtwrap/pybind_wrapper.py | 19 ++------ gtwrap/serialization.py | 68 +++++++++++++++++++++++++++ tests/test_serialization_discovery.py | 35 ++++++++++++++ 3 files changed, 107 insertions(+), 15 deletions(-) create mode 100644 gtwrap/serialization.py create mode 100644 tests/test_serialization_discovery.py diff --git a/gtwrap/pybind_wrapper.py b/gtwrap/pybind_wrapper.py index b411008..bb66ad0 100755 --- a/gtwrap/pybind_wrapper.py +++ b/gtwrap/pybind_wrapper.py @@ -13,6 +13,7 @@ # pylint: disable=too-many-arguments, too-many-instance-attributes, no-self-use, no-else-return, too-many-arguments, unused-format-string-argument, line-too-long, consider-using-f-string import re +from gtwrap.serialization import discover, export_name from pathlib import Path from typing import List @@ -996,22 +997,10 @@ def serialization_export_types(self, content, namespace_prefix): """ if not namespace_prefix or not namespace_prefix.endswith('::'): raise ValueError('namespace_prefix must end in ::') - module = parser.Module.parse_string(content) - module = instantiator.instantiate_namespace(module) - self._serializing_classes = [] - try: - self.wrap_namespace(module) - return [cpp_class for cpp_class in self._serializing_classes - if cpp_class.startswith(namespace_prefix)] - finally: - self._serializing_classes = [] + return [entry.cpp_type for entry in discover( + [('', content)], [namespace_prefix], self.ignore_classes)] - @staticmethod - def serialization_export_name(cpp_class): - """Return the existing Boost export name, including comma-safe alias.""" - if ',' in cpp_class: - return re.sub('[,:<> ]', '', cpp_class) - return cpp_class + serialization_export_name = staticmethod(export_name) def serialization_support(self, content, namespace_prefix, includes=()): """Generate shared export keys and a single-owner implementation list. diff --git a/gtwrap/serialization.py b/gtwrap/serialization.py new file mode 100644 index 0000000..ed89c8f --- /dev/null +++ b/gtwrap/serialization.py @@ -0,0 +1,68 @@ +"""Discover and generate centrally owned Boost.Serialization support.""" + +from dataclasses import dataclass +import re + +from gtwrap import interface_parser as parser +from gtwrap import template_instantiator as instantiator + + +def export_name(cpp_type): + """Preserve the identifiers historically emitted by the Python wrapper.""" + return re.sub('[,:<> ]', '', cpp_type) if ',' in cpp_type else cpp_type + + +def is_serializable(cls): + """Use the same instantiated method markers as ordinary Python wrapping.""" + return any(method.to_cpp() in ('serialize', 'serializable') + for method in list(cls.methods) + list(cls.static_methods)) + + +@dataclass(frozen=True) +class Export: + cpp_type: str + name: str + source: str + + +def discover(sources, namespace_prefixes=(), ignore_classes=()): + """Collect exports from ``(source name, interface text)`` pairs in order. + + Selection happens after template instantiation, so aliases and explicit + instantiations have exactly the names used by the ordinary wrapper. + """ + for prefix in namespace_prefixes: + if not prefix or not prefix.endswith('::'): + raise ValueError('namespace prefixes must be nonempty and end in ::') + ignored = set(ignore_classes) + exports = {} + names = {} + + def visit(namespace, source): + for element in namespace.content: + if isinstance(element, parser.Namespace): + visit(element, source) + elif isinstance(element, instantiator.InstantiatedClass): + cpp_type = element.to_cpp() + if (cpp_type in ignored or not is_serializable(element) + or (namespace_prefixes and not any( + cpp_type.startswith(p) for p in namespace_prefixes))): + continue + name = export_name(cpp_type) + if name in names and names[name].cpp_type != cpp_type: + previous = names[name] + raise ValueError( + f'{source}: export identifier {name!r} for {cpp_type} ' + f'conflicts with {previous.cpp_type} in {previous.source}') + entry = Export(cpp_type, name, source) + exports.setdefault(cpp_type, entry) + names.setdefault(name, entry) + + source_names = [] + for source, content in sources: + source_names.append(source) + module = parser.Module.parse_string(content, source_name=source) + visit(instantiator.instantiate_namespace(module), source) + if not exports: + raise ValueError(f'{", ".join(source_names)}: no serializable types matched') + return list(exports.values()) diff --git a/tests/test_serialization_discovery.py b/tests/test_serialization_discovery.py new file mode 100644 index 0000000..395e6b5 --- /dev/null +++ b/tests/test_serialization_discovery.py @@ -0,0 +1,35 @@ +"""Selection uses instantiated declarations, without generating bindings.""" +import unittest + +from gtwrap.serialization import discover + + +class TestDiscovery(unittest.TestCase): + SOURCE = '''namespace example { +class First { void serializable() const; }; +class Second { void serialize() const; }; +class Plain {}; +}''' + + def test_sources_filters_and_exclusions(self): + sources = [('one.i', self.SOURCE), ('two.i', self.SOURCE)] + self.assertEqual([e.cpp_type for e in discover(sources)], + ['example::First', 'example::Second']) + self.assertEqual([e.cpp_type for e in discover( + sources, ['example::'], ['example::First'])], ['example::Second']) + with self.assertRaisesRegex(ValueError, 'one.i.*no serializable'): + discover(sources, ['other::']) + with self.assertRaisesRegex(ValueError, 'end in ::'): + discover(sources, ['example']) + + def test_template_alias_and_collision(self): + source = '''namespace example { +template class Pair {}; +@serializable typedef example::Pair PairID; +}''' + entry, = discover([('alias.i', source)]) + self.assertEqual(entry.cpp_type, 'example::Pair') + self.assertEqual(entry.name, 'examplePairintdouble') + collision = 'class examplePairintdouble { void serialize() const; };' + with self.assertRaisesRegex(ValueError, 'collision.i.*alias.i'): + discover([('alias.i', source), ('collision.i', collision)]) From 24f52d2353169a1d1c3e49e5b755b32bd41408ba Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 13:58:02 -0400 Subject: [PATCH 03/12] Generate a central text archive runtime with a typed consumer ABI --- gtwrap/serialization_library.py | 105 ++++++++++++++++++++++++++++ tests/test_serialization_library.py | 25 +++++++ 2 files changed, 130 insertions(+) create mode 100644 gtwrap/serialization_library.py create mode 100644 tests/test_serialization_library.py diff --git a/gtwrap/serialization_library.py b/gtwrap/serialization_library.py new file mode 100644 index 0000000..95fe3ac --- /dev/null +++ b/gtwrap/serialization_library.py @@ -0,0 +1,105 @@ +"""Generate a private archive runtime and its typed consumer API.""" + +import hashlib +import json +from pathlib import Path + +from gtwrap.serialization import discover + + +def generate_library(name, sources, headers, namespace_prefixes=(), + ignore_classes=()): + """Return generated files indexed by basename; callers choose placement. + + C linkage gives the linker an exact export list; parameters still require + the same C++ ABI and runtime in consumers and the library. + """ + if not name or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-' for c in name): + raise ValueError('library name must contain only letters, digits, _ or -') + if not headers or any('\n' in h or '\r' in h or '>' in h for h in headers): + raise ValueError('supply C++ headers defining the managed types') + exports = discover(sources, namespace_prefixes, ignore_classes) + library_id = hashlib.sha256(name.encode()).hexdigest()[:16] + macro = f'GTWRAP_SERIALIZATION_{library_id.upper()}_API' + building = macro + '_BUILDING' + api_name = name + '.h' + keys_name = name + '_keys.h' + entries = [] + for export in exports: + type_id = hashlib.sha256(export.cpp_type.encode()).hexdigest()[:16] + symbol = f'gtwrap_{library_id}_{type_id}' + entries.append(dict(cpp_type=export.cpp_type, export_name=export.name, + save=symbol + '_save', load=symbol + '_load')) + + api = ['// Generated by gtwrap. Do not edit.', '#pragma once', + '#include '] + api += [f'#include <{h}>' for h in headers] + api += [f'#if defined(_WIN32) && defined({building})', + f'#define {macro} __declspec(dllexport)', + '#elif defined(_WIN32)', f'#define {macro} __declspec(dllimport)', + '#else', f'#define {macro} __attribute__((visibility("default")))', + '#endif', ''] + keys = ['// Private registration declarations.', '#pragma once', + '#include ', f'#include "{api_name}"'] + source = ['// Generated by gtwrap. Do not edit.', + '// Only the typed bridge is part of this library\'s public ABI.', + '#if defined(_WIN32)', '#include ', + '// Keep Boost singleton optimizer annotations private.', + '#undef BOOST_DLLEXPORT', '#define BOOST_DLLEXPORT', '#endif', + f'#define {building}', + '#include ', + '#include ', + '#include ', + f'#include "{keys_name}"', '#include ', + '#include ', ''] + symbols = [] + for entry in entries: + cpp_type, export = entry['cpp_type'], entry['export_name'] + save, load = entry['save'], entry['load'] + symbols += [save, load] + api += [f'extern "C" {macro} void {save}(const {cpp_type}* value, std::string* output);', + f'extern "C" {macro} void {load}(const std::string* input, {cpp_type}* value);'] + if export != cpp_type: + keys.append(f'typedef {cpp_type} {export};') + keys.append(f'BOOST_CLASS_EXPORT_KEY({export})') + source.append(f'BOOST_CLASS_EXPORT_IMPLEMENT({export})') + for entry in entries: + cpp_type, save, load = entry['cpp_type'], entry['save'], entry['load'] + source.append(f''' +extern "C" {macro} void {save}(const {cpp_type}* value, std::string* output) {{ + try {{ + std::ostringstream stream; + {{ boost::archive::text_oarchive archive(stream); archive << *value; }} + *output = stream.str(); + }} catch (const boost::archive::archive_exception& error) {{ + throw std::runtime_error(error.what()); + }} +}} + +extern "C" {macro} void {load}(const std::string* input, {cpp_type}* value) {{ + try {{ + std::istringstream stream(*input); + boost::archive::text_iarchive archive(stream); + archive >> *value; + }} catch (const boost::archive::archive_exception& error) {{ + throw std::runtime_error(error.what()); + }} +}}''') + return { + api_name: '\n'.join(api) + '\n', + keys_name: '\n'.join(keys) + '\n', + name + '.cpp': '\n'.join(source) + '\n', + name + '.json': json.dumps(dict(version=1, header=api_name, + entries=entries), indent=2) + '\n', + name + '.map': '{ global:\n' + ''.join(f' {s};\n' for s in symbols) + + 'local: *;\n};\n', + name + '.exports': ''.join(f'_{s}\n' for s in symbols), + } + + +def write_library(directory, files): + """Write every output, including unchanged ones, to satisfy build timestamps.""" + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + for name, content in files.items(): + (directory / name).write_text(content, encoding='utf-8') diff --git a/tests/test_serialization_library.py b/tests/test_serialization_library.py new file mode 100644 index 0000000..077c18d --- /dev/null +++ b/tests/test_serialization_library.py @@ -0,0 +1,25 @@ +"""The generated ABI keeps archive implementation details in one owner.""" +import json +import unittest + +from gtwrap.serialization_library import generate_library + + +class TestLibraryGeneration(unittest.TestCase): + def test_private_exports_and_stable_abi(self): + source = '''namespace example { +template class Pair {}; +@serializable typedef example::Pair PairID; +}''' + files = generate_library('registry', [('types.i', source)], ['types.h']) + manifest = json.loads(files['registry.json']) + entry, = manifest['entries'] + self.assertEqual(entry['export_name'], 'examplePairintdouble') + self.assertNotIn('BOOST_CLASS_EXPORT', files['registry.h']) + self.assertNotIn('boost/archive', files['registry.h']) + self.assertIn('BOOST_CLASS_EXPORT_KEY(examplePairintdouble)', files['registry_keys.h']) + self.assertIn('BOOST_CLASS_EXPORT_IMPLEMENT(examplePairintdouble)', files['registry.cpp']) + self.assertIn('throw std::runtime_error(error.what())', files['registry.cpp']) + self.assertIn(entry['save'] + ';', files['registry.map']) + self.assertIn('_' + entry['load'] + '\n', files['registry.exports']) + self.assertEqual(files, generate_library('registry', [('types.i', source)], ['types.h'])) From 33993257cfc0ea23e122962f038307d20c51299f Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 14:04:59 -0400 Subject: [PATCH 04/12] Connect managed archives to pybind bindings and CMake targets --- cmake/PybindWrap.cmake | 135 +++++++++++++++++----- cmake/gtwrapConfig.cmake.in | 10 +- gtwrap/pybind_wrapper.py | 70 ++++++------ scripts/pybind_serialization.py | 41 +++---- scripts/pybind_wrap.py | 3 + tests/test_serialization_bindings.py | 44 ++++++++ tests/test_serialization_support.py | 163 --------------------------- 7 files changed, 210 insertions(+), 256 deletions(-) create mode 100644 tests/test_serialization_bindings.py delete mode 100644 tests/test_serialization_support.py diff --git a/cmake/PybindWrap.cmake b/cmake/PybindWrap.cmake index a530dd7..cb8ef1d 100644 --- a/cmake/PybindWrap.cmake +++ b/cmake/PybindWrap.cmake @@ -5,33 +5,89 @@ else() set(GTWRAP_PACKAGE_DIR ${CMAKE_CURRENT_LIST_DIR}/..) endif() -# Generate one shared export-key header and one implementation fragment from a -# selected interface namespace. Callers include the header in every binding -# translation unit that may serialize these types, and include the fragment -# once in each extension that lacks its own BOOST_CLASS_EXPORT owner. -# ARGN contains C++ headers defining the selected types. -function(pybind_generate_serialization_support interface namespace_prefix - output_header output_implementations) +# Installed packages set these before including this module. Source-tree users +# need no separate script assignments. +if(NOT PYBIND_WRAP_SCRIPT) + set(PYBIND_WRAP_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/../scripts/pybind_wrap.py") +endif() +if(NOT PYBIND_SERIALIZATION_SCRIPT) + set(PYBIND_SERIALIZATION_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/../scripts/pybind_serialization.py") +endif() +file(GLOB_RECURSE GTWRAP_GENERATOR_SOURCES CONFIGURE_DEPENDS + "${GTWRAP_PACKAGE_DIR}/gtwrap/*.py") + +# Create one shared archive owner. Callers supply the complete polymorphic +# type family and use SERIALIZATION_LIBRARY in every participating pybind_wrap. +function(pybind_add_serialization_library target) + cmake_parse_arguments(REG "" "BOOST_SERIALIZATION_TARGET" + "INTERFACES;HEADERS;LINK_LIBRARIES;NAMESPACE_PREFIXES;IGNORE_CLASSES" ${ARGN}) + if(REG_UNPARSED_ARGUMENTS OR REG_KEYWORDS_MISSING_VALUES) + message(FATAL_ERROR "Invalid pybind_add_serialization_library arguments") + endif() + if(NOT REG_INTERFACES OR NOT REG_HEADERS) + message(FATAL_ERROR "A serialization library requires INTERFACES and HEADERS") + endif() + if(NOT target MATCHES "^[A-Za-z0-9_-]+$") + message(FATAL_ERROR "Serialization target names may contain letters, digits, _ and -") + endif() + if(NOT REG_BOOST_SERIALIZATION_TARGET) + set(REG_BOOST_SERIALIZATION_TARGET Boost::serialization) + endif() + if(NOT TARGET ${REG_BOOST_SERIALIZATION_TARGET}) + message(FATAL_ERROR "Provide an existing Boost.Serialization target") + endif() + set(directory "${CMAKE_CURRENT_BINARY_DIR}/gtwrap_serialization/${target}") + set(arguments --name "${target}" --out-dir "${directory}") + set(interfaces "") + foreach(interface IN LISTS REG_INTERFACES) + get_filename_component(interface "${interface}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") + list(APPEND interfaces "${interface}") + list(APPEND arguments --src "${interface}") + endforeach() + foreach(header IN LISTS REG_HEADERS) + list(APPEND arguments --include "${header}") + endforeach() + foreach(prefix IN LISTS REG_NAMESPACE_PREFIXES) + list(APPEND arguments --namespace-prefix "${prefix}") + endforeach() + foreach(type IN LISTS REG_IGNORE_CLASSES) + list(APPEND arguments --ignore "${type}") + endforeach() if(UNIX) - set(_serialization_path_separator ":") + set(separator ":") else() - set(_serialization_path_separator ";") + set(separator ";") endif() - set(_serialization_includes "") - foreach(_serialization_include IN LISTS ARGN) - list(APPEND _serialization_includes --include "${_serialization_include}") + set(outputs "") + foreach(suffix .h _keys.h .cpp .json .map .exports) + list(APPEND outputs "${directory}/${target}${suffix}") endforeach() - add_custom_command( - OUTPUT "${output_header}" "${output_implementations}" + add_custom_command(OUTPUT ${outputs} COMMAND ${CMAKE_COMMAND} -E env - "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${_serialization_path_separator}$ENV{PYTHONPATH}" - ${PYTHON_EXECUTABLE} ${PYBIND_SERIALIZATION_SCRIPT} - --src "${interface}" --namespace-prefix "${namespace_prefix}" - --out-header "${output_header}" - --out-implementations "${output_implementations}" - ${_serialization_includes} - DEPENDS "${interface}" "${PYBIND_SERIALIZATION_SCRIPT}" + "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${separator}$ENV{PYTHONPATH}" + "${PYTHON_EXECUTABLE}" "${PYBIND_SERIALIZATION_SCRIPT}" ${arguments} + DEPENDS ${interfaces} "${PYBIND_SERIALIZATION_SCRIPT}" ${GTWRAP_GENERATOR_SOURCES} VERBATIM) + add_custom_target(${target}_generate DEPENDS ${outputs}) + add_library(${target} SHARED "${directory}/${target}.cpp") + add_dependencies(${target} ${target}_generate) + target_compile_features(${target} PUBLIC cxx_std_17) + target_link_libraries(${target} PRIVATE ${REG_BOOST_SERIALIZATION_TARGET} + PUBLIC ${REG_LINK_LIBRARIES}) + target_include_directories(${target} PUBLIC "$") + set_target_properties(${target} PROPERTIES + CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON + WINDOWS_EXPORT_ALL_SYMBOLS OFF + GTWRAP_SERIALIZATION_MANIFEST "${directory}/${target}.json") + if(APPLE) + # Match pybind_wrap's cross-DSO RTTI behavior for header-defined types. + target_compile_options(${target} PRIVATE -fvisibility-ms-compat) + target_link_options(${target} PRIVATE "LINKER:-exported_symbols_list,${directory}/${target}.exports") + set_property(TARGET ${target} APPEND PROPERTY LINK_DEPENDS "${directory}/${target}.exports") + elseif(UNIX) + target_link_options(${target} PRIVATE "LINKER:--version-script=${directory}/${target}.map") + set_property(TARGET ${target} APPEND PROPERTY LINK_DEPENDS "${directory}/${target}.map") + endif() endfunction() # Get the Python version @@ -79,7 +135,30 @@ function( libs dependencies) set(ExtraMacroArgs ${ARGN}) - list(GET ExtraMacroArgs 0 USE_BOOST_SERIALIZATION) + set(USE_BOOST_SERIALIZATION OFF) + if(ExtraMacroArgs) + list(POP_FRONT ExtraMacroArgs USE_BOOST_SERIALIZATION) + endif() + cmake_parse_arguments(WRAP "" "SERIALIZATION_LIBRARY" "" ${ExtraMacroArgs}) + if(WRAP_UNPARSED_ARGUMENTS OR WRAP_KEYWORDS_MISSING_VALUES) + message(FATAL_ERROR "Invalid pybind_wrap optional arguments") + endif() + set(_serialization_args "") + set(_serialization_depends "") + if(WRAP_SERIALIZATION_LIBRARY) + if(NOT USE_BOOST_SERIALIZATION) + message(FATAL_ERROR "SERIALIZATION_LIBRARY requires serialization enabled") + endif() + if(NOT TARGET ${WRAP_SERIALIZATION_LIBRARY}) + message(FATAL_ERROR "Unknown serialization library ${WRAP_SERIALIZATION_LIBRARY}") + endif() + get_target_property(_manifest ${WRAP_SERIALIZATION_LIBRARY} GTWRAP_SERIALIZATION_MANIFEST) + if(NOT _manifest) + message(FATAL_ERROR "Target was not created by pybind_add_serialization_library") + endif() + set(_serialization_args --serialization-manifest "${_manifest}") + set(_serialization_depends "${_manifest}" ${WRAP_SERIALIZATION_LIBRARY}_generate) + endif() if(USE_BOOST_SERIALIZATION) set(_WRAP_BOOST_ARG "--use-boost-serialization") else(USE_BOOST_SERIALIZATION) @@ -119,8 +198,8 @@ function( --out "${cpp_file}" --module_name ${module_name} --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes} --template ${module_template} --is_submodule ${_WRAP_BOOST_ARG} - --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}" - DEPENDS "${interface_file}" ${module_template} "${module_name}/specializations/${interface}.h" "${module_name}/preamble/${interface}.h" + --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}" ${_serialization_args} + DEPENDS ${_serialization_depends} ${GTWRAP_GENERATOR_SOURCES} "${PYBIND_WRAP_SCRIPT}" "${interface_file}" ${module_template} "${module_name}/specializations/${interface}.h" "${module_name}/preamble/${interface}.h" VERBATIM) endforeach() @@ -138,8 +217,8 @@ function( --out "${generated_cpp}" --module_name ${module_name} --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes} --template ${module_template} ${_WRAP_BOOST_ARG} - --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}" - DEPENDS "${main_interface}" ${module_template} "${module_name}/specializations/${main_interface_name}.h" "${module_name}/specializations/${main_interface_name}.h" + --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}" ${_serialization_args} + DEPENDS ${_serialization_depends} ${GTWRAP_GENERATOR_SOURCES} "${PYBIND_WRAP_SCRIPT}" "${main_interface}" ${module_template} "${module_name}/specializations/${main_interface_name}.h" "${module_name}/specializations/${main_interface_name}.h" VERBATIM) add_custom_target(pybind_wrap_${module_name} DEPENDS ${cpp_files}) @@ -159,6 +238,10 @@ function( endif() add_dependencies(${target} pybind_wrap_${module_name}) + if(WRAP_SERIALIZATION_LIBRARY) + target_link_libraries(${target} PRIVATE ${WRAP_SERIALIZATION_LIBRARY}) + add_dependencies(${target} ${WRAP_SERIALIZATION_LIBRARY}_generate) + endif() if(NOT "${libs}" STREQUAL "") target_link_libraries(${target} PRIVATE "${libs}") diff --git a/cmake/gtwrapConfig.cmake.in b/cmake/gtwrapConfig.cmake.in index c3b1bf4..0374ec2 100644 --- a/cmake/gtwrapConfig.cmake.in +++ b/cmake/gtwrapConfig.cmake.in @@ -9,16 +9,16 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") # Used in the *Wrap.cmake files. set_and_check(GTWRAP_PYTHON_PACKAGE_DIR @PACKAGE_INSTALL_LIB_DIR@) -# Load all the CMake scripts from the standard location -include(@PACKAGE_INSTALL_CMAKE_DIR@/PybindWrap.cmake) -include(@PACKAGE_INSTALL_CMAKE_DIR@/MatlabWrap.cmake) -include(@PACKAGE_INSTALL_CMAKE_DIR@/GtwrapUtils.cmake) - # Set the variables for the wrapping scripts to be used in the build. set_and_check(PYBIND_WRAP_SCRIPT "@PACKAGE_INSTALL_BIN_DIR@/pybind_wrap.py") set_and_check(PYBIND_SERIALIZATION_SCRIPT "@PACKAGE_INSTALL_BIN_DIR@/pybind_serialization.py") set_and_check(MATLAB_WRAP_SCRIPT "@PACKAGE_INSTALL_BIN_DIR@/matlab_wrap.py") +# Load all the CMake scripts from the standard location +include(@PACKAGE_INSTALL_CMAKE_DIR@/PybindWrap.cmake) +include(@PACKAGE_INSTALL_CMAKE_DIR@/MatlabWrap.cmake) +include(@PACKAGE_INSTALL_CMAKE_DIR@/GtwrapUtils.cmake) + # Load the same pybind11 provider selected when gtwrap was configured. if(@GTWRAP_USE_SYSTEM_PYBIND@) find_package(pybind11 REQUIRED) diff --git a/gtwrap/pybind_wrapper.py b/gtwrap/pybind_wrapper.py index bb66ad0..8c1e167 100755 --- a/gtwrap/pybind_wrapper.py +++ b/gtwrap/pybind_wrapper.py @@ -13,7 +13,8 @@ # pylint: disable=too-many-arguments, too-many-instance-attributes, no-self-use, no-else-return, too-many-arguments, unused-format-string-argument, line-too-long, consider-using-f-string import re -from gtwrap.serialization import discover, export_name +import json +from gtwrap.serialization import export_name from pathlib import Path from typing import List @@ -83,12 +84,24 @@ def __init__(self, use_boost_serialization=False, ignore_classes=(), module_template="", - xml_source=""): + xml_source="", + serialization_manifest=None): self.module_name = module_name self.top_module_namespaces = top_module_namespaces self.use_boost_serialization = use_boost_serialization self.ignore_classes = ignore_classes self._serializing_classes = [] + self._managed_serialization = {} + self._serialization_header = None + if serialization_manifest is not None: + if not use_boost_serialization: + raise ValueError('a serialization library requires serialization enabled') + manifest = json.loads(Path(serialization_manifest).read_text(encoding='utf-8')) + if manifest.get('version') != 1: + raise ValueError('unsupported serialization manifest version') + self._serialization_header = manifest['header'] + self._managed_serialization = { + entry['cpp_type']: entry for entry in manifest['entries']} self.module_template = module_template self.python_keywords = [ 'lambda', 'False', 'def', 'if', 'raise', 'None', 'del', 'import', @@ -185,6 +198,21 @@ def wrap_ctors(self, my_class): def _wrap_serialization(self, cpp_class): """Helper method to add serialize, deserialize and pickle methods to the wrapped class.""" + if cpp_class in self._managed_serialization: + entry = self._managed_serialization[cpp_class] + save, load = entry['save'], entry['load'] + indent = self.method_indent + return ( + f'{indent}.def("serialize", [](const {cpp_class}* self){{ ' + f'std::string result; {save}(self, &result); return result; }})' + f'{indent}.def("deserialize", []({cpp_class}* self, const std::string& serialized){{ ' + f'{load}(&serialized, self); }}, py::arg("serialized"))' + f'{indent}.def(py::pickle(' + f'[](const {cpp_class}& value){{ std::string result; ' + f'{save}(&value, &result); return py::make_tuple(result); }}, ' + f'[](py::tuple state){{ {cpp_class} value; ' + f'auto input = state[0].cast(); ' + f'{load}(&input, &value); return value; }}))') if not cpp_class in self._serializing_classes: self._serializing_classes.append(cpp_class) @@ -911,6 +939,9 @@ def wrap_file(self, wrapped_declarations, wrapped_bindings, includes = self.wrap_namespace( module) + if self._serialization_header: + includes = f'#include "{self._serialization_header}"\n' + includes + if self.use_boost_serialization: includes += "#include " @@ -987,43 +1018,8 @@ def wrap_file(self, submodules_init="", ) - def serialization_export_types(self, content, namespace_prefix): - """Find serializable types in one interface using the normal wrapper path. - - A prefix selects the shared polymorphic family. In particular, this - avoids exporting unrelated classes from a large interface such as - GTSAM's linear.i. The caller owns the export implementations; this - method only discovers the types and does not change normal wrapping. - """ - if not namespace_prefix or not namespace_prefix.endswith('::'): - raise ValueError('namespace_prefix must end in ::') - return [entry.cpp_type for entry in discover( - [('', content)], [namespace_prefix], self.ignore_classes)] - serialization_export_name = staticmethod(export_name) - def serialization_support(self, content, namespace_prefix, includes=()): - """Generate shared export keys and a single-owner implementation list. - - Include the key header in every serialization translation unit before - archive instantiation. Include the implementation fragment exactly once - per extension, after its archive headers. The ordinary generated full - exports in that owner must not also declare the same keys. - """ - classes = self.serialization_export_types(content, namespace_prefix) - header = ['#pragma once', '', '#include '] - for include in includes: - header.append(f'#include <{include}>') - header.append('') - implementations = [] - for cpp_class in classes: - name = self.serialization_export_name(cpp_class) - if name != cpp_class: - header.append(f'typedef {cpp_class} {name};') - header.append(f'BOOST_CLASS_EXPORT_KEY({name})') - implementations.append(f'BOOST_CLASS_EXPORT_IMPLEMENT({name})') - return '\n'.join(header) + '\n', '\n'.join(implementations) + '\n' - def wrap_submodule(self, source): """ Wrap a list of submodule files, i.e. a set of interface files which are diff --git a/scripts/pybind_serialization.py b/scripts/pybind_serialization.py index b11d2e7..39eea14 100644 --- a/scripts/pybind_serialization.py +++ b/scripts/pybind_serialization.py @@ -1,38 +1,29 @@ #!/usr/bin/env python3 -"""Generate shared Boost export keys from a pybind interface. - -The implementation fragment is for an extension that does not already own -full BOOST_CLASS_EXPORT registrations for these types. It must be included -once, after that extension's archive headers and the generated key header. -""" +"""Generate a central Boost.Serialization library from interface declarations.""" import argparse from pathlib import Path -from gtwrap.pybind_wrapper import PybindWrapper +from gtwrap.interface_parser import InterfaceParseError +from gtwrap.serialization_library import generate_library, write_library def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--src', required=True, type=Path) - parser.add_argument('--namespace-prefix', required=True) - parser.add_argument('--include', action='append', default=[], - help='C++ header defining the selected types; repeatable') - parser.add_argument('--out-header', required=True, type=Path) - parser.add_argument('--out-implementations', required=True, type=Path) + parser.add_argument('--name', required=True) + parser.add_argument('--src', required=True, action='append', type=Path) + parser.add_argument('--namespace-prefix', action='append', default=[]) + parser.add_argument('--ignore', action='append', default=[]) + parser.add_argument('--include', required=True, action='append') + parser.add_argument('--out-dir', required=True, type=Path) args = parser.parse_args() - - wrapper = PybindWrapper(module_name='', top_module_namespaces=[''], - use_boost_serialization=True) - keys, implementations = wrapper.serialization_support( - args.src.read_text(encoding='utf-8'), args.namespace_prefix, - args.include) - if not implementations.strip(): - parser.error('no serializable types matched --namespace-prefix') - args.out_header.parent.mkdir(parents=True, exist_ok=True) - args.out_implementations.parent.mkdir(parents=True, exist_ok=True) - args.out_header.write_text(keys, encoding='utf-8') - args.out_implementations.write_text(implementations, encoding='utf-8') + try: + sources = [(str(p), p.read_text(encoding='utf-8')) for p in args.src] + files = generate_library(args.name, sources, args.include, + args.namespace_prefix, args.ignore) + write_library(args.out_dir, files) + except (ValueError, OSError, InterfaceParseError) as error: + parser.error(str(error)) if __name__ == '__main__': diff --git a/scripts/pybind_wrap.py b/scripts/pybind_wrap.py index 4de783d..43d9b05 100644 --- a/scripts/pybind_wrap.py +++ b/scripts/pybind_wrap.py @@ -70,6 +70,8 @@ def main(): type=str, default="", help="The path to the Doxygen-generated XML documentation") + arg_parser.add_argument('--serialization-manifest', + help='Manifest generated by a central serialization library') args = arg_parser.parse_args() top_module_namespaces = args.top_module_namespaces.split("::") @@ -86,6 +88,7 @@ def main(): ignore_classes=args.ignore, module_template=template_content, xml_source=args.xml_source, + serialization_manifest=args.serialization_manifest, ) try: diff --git a/tests/test_serialization_bindings.py b/tests/test_serialization_bindings.py new file mode 100644 index 0000000..2a4aded --- /dev/null +++ b/tests/test_serialization_bindings.py @@ -0,0 +1,44 @@ +"""Managed methods call the archive owner; legacy generation remains opt-in.""" +import json +from pathlib import Path +import tempfile +import unittest + +from gtwrap.pybind_wrapper import PybindWrapper +from gtwrap.serialization_library import generate_library, write_library + + +class TestManagedBindings(unittest.TestCase): + def test_managed_and_unselected_types(self): + source = '''namespace example { +class Managed { Managed(); void serializable() const; }; +class Local { Local(); void serialize() const; }; +}''' + template = Path(__file__).with_name('pybind_wrapper.tpl').read_text() + with tempfile.TemporaryDirectory() as directory: + files = generate_library('registry', [('types.i', source)], ['types.h'], + ignore_classes=['example::Local']) + write_library(directory, files) + manifest = Path(directory) / 'registry.json' + wrapper = PybindWrapper('example', top_module_namespaces=[''], + use_boost_serialization=True, + module_template=template, + serialization_manifest=manifest) + output = wrapper.wrap_file(source, module_name='example', submodules=[]) + entry = json.loads(files['registry.json'])['entries'][0] + self.assertIn(entry['save'] + '(self, &result)', output) + self.assertIn(entry['load'] + '(&input, &value)', output) + self.assertIn('BOOST_CLASS_EXPORT(example::Local)', output) + self.assertNotIn('BOOST_CLASS_EXPORT(example::Managed)', output) + self.assertIn('#include "registry.h"', output) + with self.assertRaisesRegex(ValueError, 'requires serialization enabled'): + PybindWrapper('example', serialization_manifest=manifest) + + def test_disabled_has_no_runtime_dependency(self): + source = 'class Plain { Plain(); void serialize() const; };' + template = Path(__file__).with_name('pybind_wrapper.tpl').read_text() + output = PybindWrapper('plain', top_module_namespaces=[''], + module_template=template).wrap_file( + source, module_name='plain', submodules=[]) + self.assertNotIn('.def("serialize"', output) + self.assertNotIn('boost/serialization', output) diff --git a/tests/test_serialization_support.py b/tests/test_serialization_support.py deleted file mode 100644 index b86393f..0000000 --- a/tests/test_serialization_support.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Shared Boost export generation and cross-translation-unit registration.""" - -import os -from pathlib import Path -import shutil -import subprocess -import sys -import tempfile -import unittest - -from gtwrap.pybind_wrapper import PybindWrapper - - -ROOT = Path(__file__).resolve().parents[1] -FIXTURES = ROOT / 'tests' / 'fixtures' - - -class TestSerializationSupport(unittest.TestCase): - - def test_selected_keys_and_implementations(self): - source = (FIXTURES / 'serialization_shared.i').read_text() - wrapper = PybindWrapper('', top_module_namespaces=[''], - use_boost_serialization=True) - keys, implementations = wrapper.serialization_support( - source, 'example::', ['serialization_shared.h']) - self.assertIn('#include ', keys) - self.assertEqual(keys.count('BOOST_CLASS_EXPORT_KEY('), 2) - self.assertEqual(implementations.count('BOOST_CLASS_EXPORT_IMPLEMENT('), 2) - self.assertIn('BOOST_CLASS_EXPORT_KEY(example::First)', keys) - self.assertIn('BOOST_CLASS_EXPORT_KEY(example::Second)', keys) - self.assertNotIn('unrelated::Third', keys + implementations) - self.assertEqual(wrapper._serializing_classes, []) - - def test_comma_containing_type_keeps_existing_archive_name(self): - source = '''namespace example { -template class Pair {}; -@serializable -typedef example::Pair PairID; -}''' - wrapper = PybindWrapper('', top_module_namespaces=[''], - use_boost_serialization=True) - keys, implementations = wrapper.serialization_support( - source, 'example::') - self.assertIn('typedef example::Pair examplePairintdouble;', - keys) - self.assertIn('BOOST_CLASS_EXPORT_KEY(examplePairintdouble)', keys) - self.assertEqual(implementations, - 'BOOST_CLASS_EXPORT_IMPLEMENT(examplePairintdouble)\n') - - def test_cli_and_cross_translation_unit_round_trip(self): - compiler = shutil.which('c++') - boost_lib = next((path for path in - ('/opt/homebrew/lib', '/usr/local/lib', - '/usr/lib/x86_64-linux-gnu', '/usr/lib') - if list(Path(path).glob('libboost_serialization.*'))), - None) - if not compiler or not boost_lib: - self.skipTest('C++ compiler or Boost.Serialization library unavailable') - - with tempfile.TemporaryDirectory() as directory: - directory = Path(directory) - subprocess.run([ - sys.executable, str(ROOT / 'scripts' / 'pybind_serialization.py'), - '--src', str(FIXTURES / 'serialization_shared.i'), - '--namespace-prefix', 'example::', - '--include', 'serialization_shared.h', - '--out-header', str(directory / 'keys.h'), - '--out-implementations', str(directory / 'implementations.inc'), - ], check=True, env={**os.environ, 'PYTHONPATH': str(ROOT)}) - - # The first owner mirrors stable's existing full exports. The - # second mirrors unstable's keys plus implementation fragment. - # Each is compiled separately with a consumer translation unit. - consumers = r''' -#include "keys.h" -#include -#include -#include -#include -#include -int main() { - if (std::string(boost::serialization::guid()) != - "example::First") return 1; - std::shared_ptr value = std::make_shared(); - std::stringstream stream; - { boost::archive::text_oarchive archive(stream); archive << value; } - value.reset(); - { boost::archive::text_iarchive archive(stream); archive >> value; } - auto result = std::dynamic_pointer_cast(value); - return result && result->value == 42 ? 0 : 2; -} -''' - (directory / 'consumer.cpp').write_text(consumers) - archive_headers = '''#include -#include -''' - owners = { - 'full': archive_headers + '''#include "serialization_shared.h" -#include -BOOST_CLASS_EXPORT(example::First) -BOOST_CLASS_EXPORT(example::Second) -''', - 'split': archive_headers + '''#include "keys.h" -#include "implementations.inc" -''', - } - libraries = {} - for name, owner in owners.items(): - with self.subTest(owner=name): - (directory / 'owner.cpp').write_text(owner) - executable = directory / name - build = subprocess.run([ - compiler, '-std=c++17', '-I', str(directory), - '-I', str(Path(boost_lib).parent / 'include'), - '-I', str(FIXTURES), str(directory / 'owner.cpp'), - str(directory / 'consumer.cpp'), '-L', boost_lib, - '-lboost_serialization', '-o', str(executable), - ], capture_output=True, text=True) - self.assertEqual(build.returncode, 0, build.stderr) - subprocess.run([str(executable)], check=True, - env={**os.environ, - 'DYLD_LIBRARY_PATH': boost_lib, - 'LD_LIBRARY_PATH': boost_lib}) - - # Two extensions each own their implementations. Loading - # either first must not change the archive identifier or - # prevent the other from round-tripping the derived type. - (directory / 'consumer.cpp').write_text( - consumers.replace('int main()', - 'extern "C" int roundtrip()')) - library = directory / f'lib{name}{".dylib" if sys.platform == "darwin" else ".so"}' - shared = subprocess.run([ - compiler, '-std=c++17', '-shared', '-fPIC', - '-I', str(directory), - '-I', str(Path(boost_lib).parent / 'include'), - '-I', str(FIXTURES), str(directory / 'owner.cpp'), - str(directory / 'consumer.cpp'), '-L', boost_lib, - f'-Wl,-rpath,{boost_lib}', '-lboost_serialization', - '-o', str(library), - ], capture_output=True, text=True) - self.assertEqual(shared.returncode, 0, shared.stderr) - libraries[name] = library - (directory / 'consumer.cpp').write_text(consumers) - - load_script = '''import ctypes, sys -for path in sys.argv[1:]: - library = ctypes.CDLL(path) - assert library.roundtrip() == 0 -''' - for order in (('full',), ('full', 'split'), ('split', 'full')): - with self.subTest(import_order=order): - run = subprocess.run([ - sys.executable, '-c', load_script, - *(str(libraries[name]) for name in order), - ], capture_output=True, text=True, - env={**os.environ, 'DYLD_LIBRARY_PATH': boost_lib, - 'LD_LIBRARY_PATH': boost_lib}) - self.assertEqual(run.returncode, 0, - run.stdout + run.stderr) - - -if __name__ == '__main__': - unittest.main() From 0ee5cf36eecb3a5f5997c11fa89865fe50ecff18 Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 14:10:44 -0400 Subject: [PATCH 05/12] Validate central archives across consumers, installations, and Boost linkages --- .github/workflows/serialization.yml | 76 ++++++++++ CMakeLists.txt | 11 ++ .../serialization_library/CMakeLists.txt | 63 +++++++++ tests/fixtures/serialization_library/check.py | 51 +++++++ .../serialization_library/consumer_a.i | 9 ++ .../serialization_library/consumer_b.i | 9 ++ .../fixtures/serialization_library/legacy.cpp | 27 ++++ .../fixtures/serialization_library/module.tpl | 14 ++ tests/fixtures/serialization_library/types.h | 49 +++++++ tests/fixtures/serialization_library/types.i | 7 + .../serialization_library/unrelated.cpp | 22 +++ tests/fixtures/serialization_shared.h | 29 ---- tests/fixtures/serialization_shared.i | 16 --- tests/test_serialization_runtime.py | 130 ++++++++++++++++++ 14 files changed, 468 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/serialization.yml create mode 100644 tests/fixtures/serialization_library/CMakeLists.txt create mode 100644 tests/fixtures/serialization_library/check.py create mode 100644 tests/fixtures/serialization_library/consumer_a.i create mode 100644 tests/fixtures/serialization_library/consumer_b.i create mode 100644 tests/fixtures/serialization_library/legacy.cpp create mode 100644 tests/fixtures/serialization_library/module.tpl create mode 100644 tests/fixtures/serialization_library/types.h create mode 100644 tests/fixtures/serialization_library/types.i create mode 100644 tests/fixtures/serialization_library/unrelated.cpp delete mode 100644 tests/fixtures/serialization_shared.h delete mode 100644 tests/fixtures/serialization_shared.i create mode 100644 tests/test_serialization_runtime.py diff --git a/.github/workflows/serialization.yml b/.github/workflows/serialization.yml new file mode 100644 index 0000000..4d6c669 --- /dev/null +++ b/.github/workflows/serialization.yml @@ -0,0 +1,76 @@ +name: Central serialization runtime + +on: [pull_request] + +jobs: + runtime: + name: Serialization (${{ matrix.os }}, ${{ matrix.linkage }} Boost) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + linkage: [static, shared] + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v5 + - uses: actions/checkout@v5 + with: + repository: microsoft/vcpkg + ref: ef7dbf94b9198bc58f45951adcf1f041fcbc5ea0 + path: _vcpkg + - uses: ilammy/msvc-dev-cmd@v1 + if: runner.os == 'Windows' + - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: py312 + python-version: '3.12' + auto-activate-base: false + - name: Python and build tools + run: python -m pip install -e . pytest cmake ninja + - name: Bootstrap vcpkg (Unix) + if: runner.os != 'Windows' + run: ./_vcpkg/bootstrap-vcpkg.sh -disableMetrics + - name: Bootstrap vcpkg (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: ./_vcpkg/bootstrap-vcpkg.bat -disableMetrics + - name: Configure Boost triplet + run: | + mkdir -p _triplets + architecture=x64 + if [[ "$(uname -m)" == arm64 ]]; then architecture=arm64; fi + linkage=${{ matrix.linkage }} + if [[ "$linkage" == shared ]]; then linkage=dynamic; fi + cat > _triplets/managed.cmake <> _triplets/managed.cmake + echo 'set(VCPKG_CXX_FLAGS "-fPIC")' >> _triplets/managed.cmake + elif [[ "$RUNNER_OS" == macOS ]]; then + echo 'set(VCPKG_CMAKE_SYSTEM_NAME Darwin)' >> _triplets/managed.cmake + fi + - uses: actions/cache@v4 + with: + path: _vcpkg/installed + key: serialization-v1-${{ runner.os }}-${{ runner.arch }}-${{ matrix.linkage }}-ef7dbf94 + - name: Install Boost.Serialization + run: ./_vcpkg/vcpkg install boost-serialization:managed --overlay-triplets=_triplets + - name: Run required runtime tests + env: + GTWRAP_TEST_SERIALIZATION: '1' + GTWRAP_TEST_BOOST_LINKAGE: ${{ matrix.linkage }} + VCPKG_TARGET_TRIPLET: managed + CMAKE_GENERATOR: Ninja + run: | + # Build against the same native C++ runtime as the vcpkg libraries. + unset CC CXX CFLAGS CXXFLAGS CPPFLAGS LDFLAGS + export CMAKE_TOOLCHAIN_FILE="$PWD/_vcpkg/scripts/buildsystems/vcpkg.cmake" + export VCPKG_OVERLAY_TRIPLETS="$PWD/_triplets" + cmake -S . -B build -DWRAP_PYTHON_VERSION=3.12 + cmake --build build --target testSerialization.run --parallel 6 diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a1eeaa..97ea3b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,3 +90,14 @@ message(STATUS "${gtwrap} CMake path : ${GTWRAP_CMAKE_INSTALL_DIR}") message(STATUS "${gtwrap} library path : ${GTWRAP_LIB_INSTALL_DIR}") message(STATUS "${gtwrap} binary path : ${GTWRAP_BIN_INSTALL_DIR}") message(STATUS "${gtwrap} header path : ${GTWRAP_INCLUDE_INSTALL_DIR}") + +# Explicit test targets work with the caller's configured Python environment. +add_custom_target(testWrap.run + COMMAND "${PYTHON_EXECUTABLE}" -m pytest tests -q -o "addopts=" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" USES_TERMINAL) +add_custom_target(testSerialization.run + COMMAND "${PYTHON_EXECUTABLE}" -m pytest + tests/test_serialization_discovery.py tests/test_serialization_library.py + tests/test_serialization_bindings.py tests/test_serialization_runtime.py + -q -o "addopts=" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" USES_TERMINAL) diff --git a/tests/fixtures/serialization_library/CMakeLists.txt b/tests/fixtures/serialization_library/CMakeLists.txt new file mode 100644 index 0000000..e4e2560 --- /dev/null +++ b/tests/fixtures/serialization_library/CMakeLists.txt @@ -0,0 +1,63 @@ +cmake_minimum_required(VERSION 3.22) +project(serialization_consumers LANGUAGES CXX) +if(WRAP_INSTALLED) + find_package(gtwrap CONFIG REQUIRED) +else() + list(APPEND CMAKE_MODULE_PATH "${WRAP_ROOT}/cmake") + include(PybindWrap) + add_subdirectory("${WRAP_ROOT}/pybind11" pybind11) +endif() +find_package(Boost REQUIRED COMPONENTS serialization) +add_library(example_types INTERFACE) +target_include_directories(example_types INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}") +target_link_libraries(example_types INTERFACE Boost::headers) +if(WIN32) + target_compile_definitions(example_types INTERFACE BOOST_ALL_NO_LIB) + if(NOT Boost_USE_STATIC_LIBS) + target_compile_definitions(example_types INTERFACE BOOST_SERIALIZATION_DYN_LINK) + endif() +endif() +set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF) +pybind_add_serialization_library(registry + INTERFACES types.i consumer_a.i consumer_b.i + HEADERS types.h LINK_LIBRARIES example_types) +foreach(consumer consumer_a consumer_b) + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${consumer}/preamble" + "${CMAKE_CURRENT_BINARY_DIR}/${consumer}/specializations") + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${consumer}/preamble/${consumer}.h" "") + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${consumer}/specializations/${consumer}.h" "") + pybind_wrap(${consumer} "${CMAKE_CURRENT_SOURCE_DIR}/${consumer}.i" + "${consumer}.cpp" ${consumer} "example" "" + "${CMAKE_CURRENT_SOURCE_DIR}/module.tpl" "example_types" "" ON + SERIALIZATION_LIBRARY registry) +endforeach() +add_executable(legacy legacy.cpp) +target_link_libraries(legacy PRIVATE example_types Boost::serialization) +add_library(unrelated SHARED unrelated.cpp) +target_link_libraries(unrelated PRIVATE example_types Boost::serialization) +target_compile_features(unrelated PRIVATE cxx_std_17) +foreach(target registry consumer_a consumer_b unrelated) + set_target_properties(${target} PROPERTIES CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) + if(APPLE) + target_link_options(${target} PRIVATE LINKER:-dead_strip) + set_target_properties(${target} PROPERTIES INSTALL_RPATH "@loader_path") + elseif(UNIX) + target_link_options(${target} PRIVATE LINKER:--gc-sections LINKER:--as-needed) + set_target_properties(${target} PROPERTIES INSTALL_RPATH "$ORIGIN") + endif() +endforeach() +install(TARGETS registry consumer_a consumer_b unrelated legacy + LIBRARY DESTINATION runtime RUNTIME DESTINATION runtime) +# All independent executables/modules use the same selected Boost runtime. +# Copy a shared Boost beside them to test a relocatable package layout. +if(NOT Boost_USE_STATIC_LIBS) + install(FILES "$" DESTINATION runtime) + if(UNIX) + install(FILES "$" DESTINATION runtime) + set_target_properties(legacy PROPERTIES INSTALL_RPATH "$ORIGIN") + if(APPLE) + set_target_properties(legacy PROPERTIES INSTALL_RPATH "@loader_path") + endif() + endif() +endif() diff --git a/tests/fixtures/serialization_library/check.py b/tests/fixtures/serialization_library/check.py new file mode 100644 index 0000000..98ca550 --- /dev/null +++ b/tests/fixtures/serialization_library/check.py @@ -0,0 +1,51 @@ +"""Exercise installed modules in a fresh process with no search-path overrides.""" +import ctypes +import importlib +from pathlib import Path +import pickle +import sys + +# This script lives outside the installed prefix; emulate an installed package +# on sys.path, without adding any native library search-path override. +sys.path.insert(0, str(Path.cwd())) + +if sys.argv[1] == 'legacy': + from consumer_a import HolderA + archive = Path(sys.argv[2]) + value = HolderA() + value.deserialize(archive.read_text()) + assert value.get() == 42 + assert value.serialize() == archive.read_text() + archive.write_text(value.serialize()) +else: + unrelated = None + if sys.argv[2] == 'static': + name = ('unrelated.dll' if sys.platform == 'win32' else + 'libunrelated.dylib' if sys.platform == 'darwin' else 'libunrelated.so') + unrelated = ctypes.CDLL(str(Path.cwd() / name), mode=ctypes.RTLD_GLOBAL) + assert unrelated.unrelated_roundtrip() == 0 + classes = [] + for name in sys.argv[1].split(','): + module = importlib.import_module(name) + classes.append(getattr(module, 'HolderA' if name == 'consumer_a' else 'HolderB')) + assert classes[-1](1).get() == 42 + # Exercise both consumers only after all requested libraries have loaded. + for cls in classes: + for kind, expected in ((1, 42), (2, 7), (3, 3)): + value = cls(kind) + serialized = value.serialize() + assert ('example::' in serialized or 'examplePairintdouble' in serialized) + assert 'unrelated::' not in serialized + assert pickle.loads(pickle.dumps(value)).get() == expected + for other in classes: + restored = other() + restored.deserialize(serialized) + assert restored.get() == expected + try: + cls().deserialize('not an archive') + except RuntimeError: + pass + else: + raise AssertionError('invalid archives must raise RuntimeError') + if unrelated: + assert unrelated.unrelated_roundtrip() == 0 diff --git a/tests/fixtures/serialization_library/consumer_a.i b/tests/fixtures/serialization_library/consumer_a.i new file mode 100644 index 0000000..7642d6d --- /dev/null +++ b/tests/fixtures/serialization_library/consumer_a.i @@ -0,0 +1,9 @@ +namespace example { +#include +class HolderA { + HolderA(); + HolderA(int kind); + int get() const; + void serializable() const; +}; +} diff --git a/tests/fixtures/serialization_library/consumer_b.i b/tests/fixtures/serialization_library/consumer_b.i new file mode 100644 index 0000000..f59f54e --- /dev/null +++ b/tests/fixtures/serialization_library/consumer_b.i @@ -0,0 +1,9 @@ +namespace example { +#include +class HolderB { + HolderB(); + HolderB(int kind); + int get() const; + void serializable() const; +}; +} diff --git a/tests/fixtures/serialization_library/legacy.cpp b/tests/fixtures/serialization_library/legacy.cpp new file mode 100644 index 0000000..58a476a --- /dev/null +++ b/tests/fixtures/serialization_library/legacy.cpp @@ -0,0 +1,27 @@ +#include +#include +#include +#include "types.h" +#include +#include + +BOOST_CLASS_EXPORT(example::First) +BOOST_CLASS_EXPORT(example::Second) +typedef example::Pair examplePairintdouble; +BOOST_CLASS_EXPORT(examplePairintdouble) +BOOST_CLASS_EXPORT(example::HolderA) + +int main(int argc, char** argv) { + if (argc != 3) return 1; + example::HolderA value(1); + if (std::string(argv[1]) == "save") { + std::ofstream stream(argv[2]); + boost::archive::text_oarchive archive(stream); + archive << value; + } else { + std::ifstream stream(argv[2]); + boost::archive::text_iarchive archive(stream); + archive >> value; + if (value.get() != 42) return 2; + } +} diff --git a/tests/fixtures/serialization_library/module.tpl b/tests/fixtures/serialization_library/module.tpl new file mode 100644 index 0000000..c03e2be --- /dev/null +++ b/tests/fixtures/serialization_library/module.tpl @@ -0,0 +1,14 @@ +#include +{includes} +{boost_class_export} +namespace py = pybind11; +{submodules} +{declaration_module_def} {{ +{wrapped_declarations} +}} +{binding_module_def} {{ +{wrapped_bindings} +}} +{module_def} {{ +{module_init} +}} diff --git a/tests/fixtures/serialization_library/types.h b/tests/fixtures/serialization_library/types.h new file mode 100644 index 0000000..5e33511 --- /dev/null +++ b/tests/fixtures/serialization_library/types.h @@ -0,0 +1,49 @@ +#pragma once +#include +#include +#include + +namespace example { +struct Base { + virtual ~Base() = default; + virtual int get() const { return 0; } + template void serialize(Archive&, unsigned int) {} +}; +struct First : Base { + int value = 42; + int get() const override { return value; } + template void serialize(Archive& archive, unsigned int) { + archive & boost::serialization::base_object(*this) & value; + } +}; +struct Second : Base { + int value = 7; + int get() const override { return value; } + template void serialize(Archive& archive, unsigned int) { + archive & boost::serialization::base_object(*this) & value; + } +}; +template struct Pair : Base { + A first = 3; + B second = 1.5; + int get() const override { return static_cast(first); } + template void serialize(Archive& archive, unsigned int) { + archive & boost::serialization::base_object(*this) & first & second; + } +}; +template struct Holder { + std::shared_ptr value; + Holder() = default; + explicit Holder(int kind) { + if (kind == 1) value = std::make_shared(); + else if (kind == 3) value = std::make_shared>(); + else value = std::make_shared(); + } + int get() const { return value ? value->get() : 0; } + template void serialize(Archive& archive, unsigned int) { + archive & value; + } +}; +using HolderA = Holder<1>; +using HolderB = Holder<2>; +} diff --git a/tests/fixtures/serialization_library/types.i b/tests/fixtures/serialization_library/types.i new file mode 100644 index 0000000..caed386 --- /dev/null +++ b/tests/fixtures/serialization_library/types.i @@ -0,0 +1,7 @@ +namespace example { +virtual class Base {}; +virtual class First : example::Base { void serializable() const; }; +virtual class Second : example::Base { void serializable() const; }; +template virtual class Pair : example::Base {}; +@serializable typedef example::Pair PairID; +} diff --git a/tests/fixtures/serialization_library/unrelated.cpp b/tests/fixtures/serialization_library/unrelated.cpp new file mode 100644 index 0000000..b3e1311 --- /dev/null +++ b/tests/fixtures/serialization_library/unrelated.cpp @@ -0,0 +1,22 @@ +// A separate static Boost runtime must not steal the managed registrations. +#include +#include +#include +#include "types.h" +#include +BOOST_CLASS_EXPORT_GUID(example::First, "unrelated::First") +BOOST_CLASS_EXPORT_GUID(example::Second, "unrelated::Second") +#if defined(_WIN32) +#define TEST_EXPORT __declspec(dllexport) +#else +#define TEST_EXPORT __attribute__((visibility("default"))) +#endif +extern "C" TEST_EXPORT int unrelated_roundtrip() { + example::HolderA value(1); + std::stringstream stream; + { boost::archive::text_oarchive archive(stream); archive << value; } + if (stream.str().find("unrelated::First") == std::string::npos) return 1; + value.value.reset(); + { boost::archive::text_iarchive archive(stream); archive >> value; } + return value.get() == 42 ? 0 : 2; +} diff --git a/tests/fixtures/serialization_shared.h b/tests/fixtures/serialization_shared.h deleted file mode 100644 index fadd51d..0000000 --- a/tests/fixtures/serialization_shared.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include - -namespace example { -struct Base { - virtual ~Base() = default; - template - void serialize(Archive&, unsigned int) {} -}; - -struct First : Base { - int value = 42; - template - void serialize(Archive& archive, unsigned int) { - archive & boost::serialization::base_object(*this); - archive & value; - } -}; - -struct Second : Base { - int value = 7; - template - void serialize(Archive& archive, unsigned int) { - archive & boost::serialization::base_object(*this); - archive & value; - } -}; -} diff --git a/tests/fixtures/serialization_shared.i b/tests/fixtures/serialization_shared.i deleted file mode 100644 index e145d98..0000000 --- a/tests/fixtures/serialization_shared.i +++ /dev/null @@ -1,16 +0,0 @@ -namespace example { -#include -virtual class Base {}; -virtual class First : example::Base { - void serializable() const; -}; -virtual class Second : example::Base { - void serializable() const; -}; -} - -namespace unrelated { -class Third { - void serializable() const; -}; -} diff --git a/tests/test_serialization_runtime.py b/tests/test_serialization_runtime.py new file mode 100644 index 0000000..11e9168 --- /dev/null +++ b/tests/test_serialization_runtime.py @@ -0,0 +1,130 @@ +"""Required CI integration suite; opt in locally with GTWRAP_TEST_SERIALIZATION=1.""" +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import time +import unittest + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / 'tests/fixtures/serialization_library' + + +def run(args, **kwargs): + result = subprocess.run([str(a) for a in args], capture_output=True, + text=True, **kwargs) + if result.returncode: + raise AssertionError(f'{args}\n{result.stdout}\n{result.stderr}') + return result.stdout + + +@unittest.skipUnless(os.environ.get('GTWRAP_TEST_SERIALIZATION') == '1', + 'enable the required serialization integration suite explicitly') +class TestSerializationRuntime(unittest.TestCase): + def test_runtime_and_installation(self): + # CI selects each linkage in its own job; local runs can exercise both. + linkages = os.environ.get('GTWRAP_TEST_BOOST_LINKAGE', 'static,shared').split(',') + for linkage in linkages: + with self.subTest(linkage=linkage), tempfile.TemporaryDirectory() as temporary: + self.exercise(Path(temporary), linkage) + + def exercise(self, directory, linkage): + self.assertIn(linkage, ('static', 'shared')) + source = directory / 'source' + shutil.copytree(FIXTURE, source) + wrap = directory / 'wrap' + wrap.mkdir() + for name in ('gtwrap', 'cmake', 'scripts', 'templates', 'pybind11'): + shutil.copytree(ROOT / name, wrap / name, + ignore=shutil.ignore_patterns('__pycache__', '.git')) + for name in ('CMakeLists.txt', 'matlab.h'): + shutil.copyfile(ROOT / name, wrap / name) + env = dict(os.environ) + env['PATH'] = str(Path(sys.executable).parent) + os.pathsep + env['PATH'] + cmake_args = ['-DCMAKE_BUILD_TYPE=Release', '-DPYBIND11_FINDPYTHON=ON', + f'-DPython_EXECUTABLE={sys.executable}', + f'-DWRAP_PYTHON_VERSION={sys.version_info.major}.{sys.version_info.minor}', + f'-DBoost_USE_STATIC_LIBS={"ON" if linkage == "static" else "OFF"}'] + # Accept a caller-provided Boost/toolchain installation, never download + # dependencies or silently change linkage from inside a test. + for variable in ('CMAKE_PREFIX_PATH', 'CMAKE_TOOLCHAIN_FILE', 'VCPKG_TARGET_TRIPLET'): + if os.environ.get(variable): + cmake_args.append(f'-D{variable}={os.environ[variable]}') + + for installed in (False, True): + label = 'installed' if installed else 'source' + build = directory / (label + '-build') + prefix = directory / (label + '-prefix') + arguments = list(cmake_args) + if installed: + wrap_build = directory / 'wrap-build' + wrap_prefix = directory / 'wrap-prefix' + run(['cmake', '-S', wrap, '-B', wrap_build, + f'-DCMAKE_INSTALL_PREFIX={wrap_prefix}', *cmake_args], env=env) + run(['cmake', '--install', wrap_build, '--config', 'Release'], env=env) + arguments += ['-DWRAP_INSTALLED=ON', + f'-Dgtwrap_DIR={wrap_prefix}/lib/cmake/gtwrap'] + implementation = wrap_prefix / 'lib/gtwrap/gtwrap/serialization.py' + else: + arguments += [f'-DWRAP_ROOT={wrap}'] + implementation = wrap / 'gtwrap/serialization.py' + run(['cmake', '-S', source, '-B', build, + f'-DCMAKE_INSTALL_PREFIX={prefix}', *arguments], env=env) + run(['cmake', '--build', build, '--config', 'Release', '-j', '6'], env=env) + generated = build / 'gtwrap_serialization/registry' + manifest = json.loads((generated / 'registry.json').read_text()) + self.assertEqual(len(manifest['entries']), 5) + for consumer in ('consumer_a', 'consumer_b'): + content = (build / f'{consumer}.cpp').read_text() + self.assertNotIn('BOOST_CLASS_EXPORT(', content) + self.assertNotIn('boost::archive', content) + self.assertNotIn('gtsam::', content) + + # A no-op build is stable; edits to either source of truth regenerate + # the manifest and all consumers before they compile. + generated_source = generated / 'registry.cpp' + before = generated_source.stat().st_mtime_ns + run(['cmake', '--build', build, '--config', 'Release', '-j', '6'], env=env) + self.assertEqual(before, generated_source.stat().st_mtime_ns) + for dependency in (source / 'types.i', implementation): + time.sleep(1.05) + dependency.touch() + run(['cmake', '--build', build, '--config', 'Release', '-j', '6'], env=env) + after = generated_source.stat().st_mtime_ns + self.assertGreater(after, before) + before = after + + run(['cmake', '--install', build, '--config', 'Release'], env=env) + runtime = prefix / 'runtime' + # Ensure installed tests cannot accidentally find build products. + build.rename(directory / (label + '-hidden-build')) + clean_env = {k: v for k, v in env.items() + if k not in ('PYTHONPATH', 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH')} + for order in ('consumer_a', 'consumer_b', + 'consumer_a,consumer_b', 'consumer_b,consumer_a'): + run([sys.executable, source / 'check.py', order, linkage], + cwd=runtime, env=clean_env) + + # Compare with the historical full-export owner in a fresh process. + legacy = runtime / ('legacy.exe' if os.name == 'nt' else 'legacy') + archive = runtime / 'archive.txt' + run([legacy, 'save', archive], env=clean_env) + run([sys.executable, source / 'check.py', 'legacy', archive], + cwd=runtime, env=clean_env) + run([legacy, 'load', archive], env=clean_env) + symbols = {entry[operation] for entry in manifest['entries'] + for operation in ('save', 'load')} + if sys.platform == 'darwin': + exported = run(['nm', '-gU', runtime / 'libregistry.dylib']) + actual = {line.split()[-1].removeprefix('_') for line in exported.splitlines() if line.strip()} + elif os.name == 'nt': + exported = run(['dumpbin', '/EXPORTS', runtime / 'registry.dll']) + actual = set(re.findall(r'^\s+\d+\s+[0-9A-F]+\s+[0-9A-F]+\s+(\S+)', exported, re.M)) + else: + exported = run(['nm', '-D', '--defined-only', runtime / 'libregistry.so']) + actual = {line.split()[-1] for line in exported.splitlines() if line.strip()} + self.assertEqual(actual, symbols) From be71b4031316cfd7ab28569a67345ab6d166729a Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 14:10:44 -0400 Subject: [PATCH 06/12] Document the generic central serialization library contract --- DOCS.md | 150 ++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 120 insertions(+), 30 deletions(-) diff --git a/DOCS.md b/DOCS.md index c16beb7..6fc308b 100644 --- a/DOCS.md +++ b/DOCS.md @@ -252,41 +252,131 @@ The python wrapper supports keyword arguments for functions/methods. Hence, the - Unfortunately, this means that aliases can no longer be used. - Similarly, there can be multiple `preamble.h` and `specializations.h` files. Each of these should match the module file name. -### Shared Python Boost export keys +### Centrally managed Python serialization -When one binding translation unit serializes a polymorphic type exported by -another, both must see the same `BOOST_CLASS_EXPORT_KEY` specialization before -archive instantiation. `pybind_serialization.py` derives keys and optional -implementation macros from the same interface markers used by the ordinary -pybind generator. Select a narrow namespace to avoid including unrelated -types in every binding translation unit: +`pybind_add_serialization_library` creates an opt-in shared library that owns +Boost export registrations **and** the text-archive save/load operations used +by Python bindings. Multiple extensions link that same library. Archive +construction stays in the owner, including when Boost.Serialization itself is +static; consumers do not need their own copy of its registration tables. +Existing wrapping is unchanged unless a serialization library is selected. + +The API is independent of any particular C++ project. Supply interface files, +headers defining their types, and the domain-library targets they require: ```cmake -pybind_generate_serialization_support( - "${PROJECT_SOURCE_DIR}/gtsam/linear/linear.i" - "gtsam::noiseModel::" - "${CMAKE_CURRENT_BINARY_DIR}/noise_model_keys.h" - "${CMAKE_CURRENT_BINARY_DIR}/noise_model_implementations.inc" - "gtsam/linear/NoiseModel.h") -add_custom_target(noise_model_keys DEPENDS - "${CMAKE_CURRENT_BINARY_DIR}/noise_model_keys.h" - "${CMAKE_CURRENT_BINARY_DIR}/noise_model_implementations.inc") -add_dependencies(stable_python_module noise_model_keys) -add_dependencies(unstable_python_module noise_model_keys) +# Use find_package(gtwrap CONFIG REQUIRED) for an installed wrap package, +# or include(PybindWrap) with wrap/cmake on CMAKE_MODULE_PATH for a source tree. +find_package(Boost REQUIRED COMPONENTS serialization) + +pybind_add_serialization_library(shared_archives + INTERFACES common.i module_a.i module_b.i + HEADERS example/types.h + LINK_LIBRARIES example_types) + +pybind_wrap(module_a "${CMAKE_CURRENT_SOURCE_DIR}/module_a.i" + "module_a.cpp" "module_a" "example" "" + "${CMAKE_CURRENT_SOURCE_DIR}/module.tpl" "example_types" "" ON + SERIALIZATION_LIBRARY shared_archives) +pybind_wrap(module_b "${CMAKE_CURRENT_SOURCE_DIR}/module_b.i" + "module_b.cpp" "module_b" "example" "" + "${CMAKE_CURRENT_SOURCE_DIR}/module.tpl" "example_types" "" ON + SERIALIZATION_LIBRARY shared_archives) +``` + +`example_types` supplies the C++ headers' include paths, compile definitions, +and domain-library dependencies. The usual wrapping template and preamble +requirements still apply. The source-tree module resolves the generator script +paths itself; no extra script-variable assignment is necessary. + +The library helper accepts these keyword arguments: + +| Argument | Meaning | +| --- | --- | +| `INTERFACES` | Required list of interfaces containing the serializable types. Relative paths resolve against the current source directory. | +| `HEADERS` | Required C++ header include names defining those types. | +| `LINK_LIBRARIES` | Domain targets and their usage requirements; these must not depend on the consuming Python extensions. | +| `NAMESPACE_PREFIXES` | Optional list of namespace prefixes ending in `::`; absent means all serializable declarations. | +| `IGNORE_CLASSES` | Optional fully qualified instantiated C++ types to exclude. | +| `BOOST_SERIALIZATION_TARGET` | Existing static or shared Boost.Serialization target; defaults to `Boost::serialization`. | + +Selection uses the existing `serialize`/`serializable` markers and +`@serializable` template typedefs after instantiation. Duplicate types across +interfaces are coalesced. Empty selections and conflicting archive identifiers +are errors. Export names retain the ordinary wrapper's existing spelling, +including its comma-safe template aliases. + +Supply the complete serialization domain: both the types whose Python +save/load methods use the owner and every concrete polymorphic type reachable +through their serialized members. Wrap cannot discover arbitrary C++ runtime +subclasses or inspect the bodies of C++ serialization methods. A family-only +registration list is insufficient if the containing object's archive is still +created in an extension with its own static Boost runtime. Use one owner for +all participating boundary types and their polymorphic dependencies. + +For selected types, generated `.serialize()`, `.deserialize()`, and pickle +methods call typed functions in the library. Their normal export macros are +suppressed. Unselected types retain ordinary wrapping. Consumers include a +private generated API header through the existing `{includes}` placeholder; +no new template placeholders, manual key headers, implementation fragments, or +special owner translation units are needed. The library's registration headers +remain private to its implementation. + +This mode preserves the existing text-archive protocol and one-element pickle +state. It does not add binary/XML archives or customizable archive adapters. +Boost archive exceptions become `std::runtime_error` at the library boundary +and Python `RuntimeError` through pybind11. + +The owner is always a `SHARED` target, regardless of `BUILD_SHARED_LIBS`. Its +Boost dependency is private and can be static or shared. Static dependencies +must be suitable for linking into a shared library (including PIC where +required). Do not combine incompatible Boost versions, C++ ABIs, or C++ runtimes; +on Windows, use a consistent runtime such as `/MD` across all participants. +The typed API uses C linkage for explicit symbol exports but passes C++ objects +and strings, so it is not a cross-toolchain C ABI. Only the bridge functions are +exported; the private static Boost runtime must not interpose on another +library's registrations. + +Call the helper and pass `SERIALIZATION_LIBRARY` only when serialization is +enabled in the consuming project. Pairing the latter argument with a false +serialization flag is a configuration error. MATLAB generation is independent +of this opt-in Python facility. + +#### Installation and migration + +The caller owns the runtime package layout. Install the shared owner alongside +or in a location reachable from every consuming extension using ordinary +`install(TARGETS ...)`. Configure relative install RPATHs as appropriate (for +example `$ORIGIN` on ELF or `@loader_path` on macOS), or colocate the DLL with +Windows extensions. Package a shared Boost runtime according to the project's +existing dependency policy. Wrap does not choose installation directories or +rewrite third-party library install names. The generated manifest and headers +are build inputs; Python users do not need them at runtime. Normal linking +keeps the owner loaded while its extension consumers are loaded. + +To migrate existing bindings, identify the complete domain, create the owner, +and attach it to each consumer. Remove handwritten keys and export +implementations for the managed types, including any special-case header +exclusions. Keep existing serialization feature guards. Validate stable-only +use, both extension import orders, calls after both are loaded, legacy archive +exchange, and serialization-disabled builds before deploying the migration. + +The generic integration example in `tests/fixtures/serialization_library` +builds two independent Python consumers and a legacy archive producer. It also +checks installed source-tree/package consumption and coexistence with an +unrelated static Boost runtime. Run it using the configured Python environment: + +```sh +cmake -S . -B build +cd build +GTWRAP_TEST_SERIALIZATION=1 make -j6 testSerialization.run ``` -Add the build directory to the modules' include paths. Include the generated -key header in each serialization-enabled binding translation unit, after its -normal preamble and before `{boost_class_export}` or other serialization uses. -The translation unit whose generated `BOOST_CLASS_EXPORT` macros already own -these types must **skip** the key header: that macro declares the keys again. -In a second extension, include the key header followed by the generated -implementation fragment exactly once, after the Boost archive headers. This -retains one set of serializer implementations per extension and the existing -archive names, without an additional library or link dependency. Keep these -files private to Python bindings when C++ clients use different archive IDs. -Generate and include them only when Boost serialization is enabled. MATLAB -generation is independent of this Python helper. +Set `GTWRAP_TEST_BOOST_LINKAGE=static` or `shared` to choose one linkage; by +default both are tested. Supply `CMAKE_PREFIX_PATH`, or a toolchain and +`VCPKG_TARGET_TRIPLET`, to select the intended Boost installation. The dedicated +CI matrix provisions Boost and treats missing dependencies as errors. The +ordinary lightweight suite skips this integration test unless explicitly enabled. ## Pybind Callable Adapters From 7e4c6f732c3d9b204225a31bdd7b01fe513bdcd1 Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 14:13:11 -0400 Subject: [PATCH 07/12] Configure matching C and C++ PIC flags for Linux Boost CI --- .github/workflows/serialization.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/serialization.yml b/.github/workflows/serialization.yml index 4d6c669..7804238 100644 --- a/.github/workflows/serialization.yml +++ b/.github/workflows/serialization.yml @@ -51,6 +51,7 @@ jobs: EOF if [[ "$RUNNER_OS" == Linux ]]; then echo 'set(VCPKG_CMAKE_SYSTEM_NAME Linux)' >> _triplets/managed.cmake + echo 'set(VCPKG_C_FLAGS "-fPIC")' >> _triplets/managed.cmake echo 'set(VCPKG_CXX_FLAGS "-fPIC")' >> _triplets/managed.cmake elif [[ "$RUNNER_OS" == macOS ]]; then echo 'set(VCPKG_CMAKE_SYSTEM_NAME Darwin)' >> _triplets/managed.cmake From 1f610b5a9cd258d1fa6a1b23b5a083ba49be87c9 Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 14:18:16 -0400 Subject: [PATCH 08/12] Package transitive shared Boost components in installation tests --- .../fixtures/serialization_library/CMakeLists.txt | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/serialization_library/CMakeLists.txt b/tests/fixtures/serialization_library/CMakeLists.txt index e4e2560..75153ff 100644 --- a/tests/fixtures/serialization_library/CMakeLists.txt +++ b/tests/fixtures/serialization_library/CMakeLists.txt @@ -50,11 +50,20 @@ endforeach() install(TARGETS registry consumer_a consumer_b unrelated legacy LIBRARY DESTINATION runtime RUNTIME DESTINATION runtime) # All independent executables/modules use the same selected Boost runtime. -# Copy a shared Boost beside them to test a relocatable package layout. +# Copy shared Boost and its imported component dependencies beside them. +# Some distributions link serialization to thread/atomic, others do not. if(NOT Boost_USE_STATIC_LIBS) - install(FILES "$" DESTINATION runtime) + get_property(imported_targets DIRECTORY PROPERTY IMPORTED_TARGETS) + foreach(dependency IN LISTS imported_targets) + if(dependency MATCHES "^Boost::") + get_target_property(dependency_type ${dependency} TYPE) + if(dependency_type STREQUAL "SHARED_LIBRARY") + install(IMPORTED_RUNTIME_ARTIFACTS ${dependency} + LIBRARY DESTINATION runtime RUNTIME DESTINATION runtime) + endif() + endif() + endforeach() if(UNIX) - install(FILES "$" DESTINATION runtime) set_target_properties(legacy PROPERTIES INSTALL_RPATH "$ORIGIN") if(APPLE) set_target_properties(legacy PROPERTIES INSTALL_RPATH "@loader_path") From 1db639abeff8e5f2a69077b066f6f7e0ce8d2677 Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 14:21:29 -0400 Subject: [PATCH 09/12] Normalize Windows fixture paths and preserve dependency caches --- .github/workflows/serialization.yml | 8 +++++++- gtwrap/serialization_library.py | 2 +- tests/test_serialization_runtime.py | 14 ++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/serialization.yml b/.github/workflows/serialization.yml index 7804238..a5607a6 100644 --- a/.github/workflows/serialization.yml +++ b/.github/workflows/serialization.yml @@ -56,12 +56,18 @@ jobs: elif [[ "$RUNNER_OS" == macOS ]]; then echo 'set(VCPKG_CMAKE_SYSTEM_NAME Darwin)' >> _triplets/managed.cmake fi - - uses: actions/cache@v4 + - uses: actions/cache/restore@v4 + id: boost-cache with: path: _vcpkg/installed key: serialization-v1-${{ runner.os }}-${{ runner.arch }}-${{ matrix.linkage }}-ef7dbf94 - name: Install Boost.Serialization run: ./_vcpkg/vcpkg install boost-serialization:managed --overlay-triplets=_triplets + - uses: actions/cache/save@v4 + if: steps.boost-cache.outputs.cache-hit != 'true' + with: + path: _vcpkg/installed + key: serialization-v1-${{ runner.os }}-${{ runner.arch }}-${{ matrix.linkage }}-ef7dbf94 - name: Run required runtime tests env: GTWRAP_TEST_SERIALIZATION: '1' diff --git a/gtwrap/serialization_library.py b/gtwrap/serialization_library.py index 95fe3ac..ff24a03 100644 --- a/gtwrap/serialization_library.py +++ b/gtwrap/serialization_library.py @@ -43,7 +43,7 @@ def generate_library(name, sources, headers, namespace_prefixes=(), '#include ', f'#include "{api_name}"'] source = ['// Generated by gtwrap. Do not edit.', '// Only the typed bridge is part of this library\'s public ABI.', - '#if defined(_WIN32)', '#include ', + '#if defined(_WIN32)', '#include ', '// Keep Boost singleton optimizer annotations private.', '#undef BOOST_DLLEXPORT', '#define BOOST_DLLEXPORT', '#endif', f'#define {building}', diff --git a/tests/test_serialization_runtime.py b/tests/test_serialization_runtime.py index 11e9168..b88d627 100644 --- a/tests/test_serialization_runtime.py +++ b/tests/test_serialization_runtime.py @@ -46,14 +46,15 @@ def exercise(self, directory, linkage): env = dict(os.environ) env['PATH'] = str(Path(sys.executable).parent) + os.pathsep + env['PATH'] cmake_args = ['-DCMAKE_BUILD_TYPE=Release', '-DPYBIND11_FINDPYTHON=ON', - f'-DPython_EXECUTABLE={sys.executable}', + f'-DPython_EXECUTABLE={Path(sys.executable).as_posix()}', f'-DWRAP_PYTHON_VERSION={sys.version_info.major}.{sys.version_info.minor}', f'-DBoost_USE_STATIC_LIBS={"ON" if linkage == "static" else "OFF"}'] # Accept a caller-provided Boost/toolchain installation, never download # dependencies or silently change linkage from inside a test. for variable in ('CMAKE_PREFIX_PATH', 'CMAKE_TOOLCHAIN_FILE', 'VCPKG_TARGET_TRIPLET'): if os.environ.get(variable): - cmake_args.append(f'-D{variable}={os.environ[variable]}') + value = os.environ[variable].replace('\\', '/') + cmake_args.append(f'-D{variable}={value}') for installed in (False, True): label = 'installed' if installed else 'source' @@ -64,16 +65,17 @@ def exercise(self, directory, linkage): wrap_build = directory / 'wrap-build' wrap_prefix = directory / 'wrap-prefix' run(['cmake', '-S', wrap, '-B', wrap_build, - f'-DCMAKE_INSTALL_PREFIX={wrap_prefix}', *cmake_args], env=env) + f'-DCMAKE_INSTALL_PREFIX={wrap_prefix.as_posix()}', *cmake_args], env=env) run(['cmake', '--install', wrap_build, '--config', 'Release'], env=env) + config = next(wrap_prefix.rglob('gtwrapConfig.cmake')) arguments += ['-DWRAP_INSTALLED=ON', - f'-Dgtwrap_DIR={wrap_prefix}/lib/cmake/gtwrap'] + f'-Dgtwrap_DIR={config.parent.as_posix()}'] implementation = wrap_prefix / 'lib/gtwrap/gtwrap/serialization.py' else: - arguments += [f'-DWRAP_ROOT={wrap}'] + arguments += [f'-DWRAP_ROOT={wrap.as_posix()}'] implementation = wrap / 'gtwrap/serialization.py' run(['cmake', '-S', source, '-B', build, - f'-DCMAKE_INSTALL_PREFIX={prefix}', *arguments], env=env) + f'-DCMAKE_INSTALL_PREFIX={prefix.as_posix()}', *arguments], env=env) run(['cmake', '--build', build, '--config', 'Release', '-j', '6'], env=env) generated = build / 'gtwrap_serialization/registry' manifest = json.loads((generated / 'registry.json').read_text()) From 03077f3f05dfa39fcfc578d96902516afa87c8df Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 14:34:59 -0400 Subject: [PATCH 10/12] Keep Windows codecvt consumer definitions out of the public ABI --- gtwrap/serialization_library.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/gtwrap/serialization_library.py b/gtwrap/serialization_library.py index ff24a03..b386603 100644 --- a/gtwrap/serialization_library.py +++ b/gtwrap/serialization_library.py @@ -45,7 +45,16 @@ def generate_library(name, sources, headers, namespace_prefixes=(), '// Only the typed bridge is part of this library\'s public ABI.', '#if defined(_WIN32)', '#include ', '// Keep Boost singleton optimizer annotations private.', - '#undef BOOST_DLLEXPORT', '#define BOOST_DLLEXPORT', '#endif', + '#undef BOOST_DLLEXPORT', '#define BOOST_DLLEXPORT', + '// codecvt_null unconditionally marks even consumer definitions for export.', + '// Scope the correction to that header; preserve domain-library annotations.', + '#pragma push_macro("BOOST_SYMBOL_EXPORT")', + '#undef BOOST_SYMBOL_EXPORT', + '#if defined(BOOST_SERIALIZATION_DYN_LINK) || defined(BOOST_ALL_DYN_LINK)', + '#define BOOST_SYMBOL_EXPORT BOOST_SYMBOL_IMPORT', '#else', + '#define BOOST_SYMBOL_EXPORT', '#endif', + '#include ', + '#pragma pop_macro("BOOST_SYMBOL_EXPORT")', '#endif', f'#define {building}', '#include ', '#include ', From 3e3cc6efc3e66d2df37578a1387174a0be843e8c Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 14:35:36 -0400 Subject: [PATCH 11/12] Exercise installed consumers even when export-policy checks fail --- tests/test_serialization_runtime.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_serialization_runtime.py b/tests/test_serialization_runtime.py index b88d627..e0d7ede 100644 --- a/tests/test_serialization_runtime.py +++ b/tests/test_serialization_runtime.py @@ -129,4 +129,7 @@ def exercise(self, directory, linkage): else: exported = run(['nm', '-D', '--defined-only', runtime / 'libregistry.so']) actual = {line.split()[-1] for line in exported.splitlines() if line.strip()} - self.assertEqual(actual, symbols) + # Report export-policy failures without skipping installed-package + # runtime validation. Both consumption modes must still be exercised. + with self.subTest(consumption=label, check='public exports'): + self.assertEqual(actual, symbols) From 037762bf2b5336eba2638156c60d6f95cff8ac74 Mon Sep 17 00:00:00 2001 From: Frank Dellaert Date: Wed, 14 Oct 2026 14:45:36 -0400 Subject: [PATCH 12/12] Permit reviewed compiler-forced Windows static Boost exports --- DOCS.md | 22 +++++++++- .../windows_static_boost_exports.txt | 34 ++++++++++++++++ tests/test_serialization_runtime.py | 40 ++++++++++++++++++- 3 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/serialization_library/windows_static_boost_exports.txt diff --git a/DOCS.md b/DOCS.md index 6fc308b..40843d2 100644 --- a/DOCS.md +++ b/DOCS.md @@ -334,8 +334,26 @@ required). Do not combine incompatible Boost versions, C++ ABIs, or C++ runtimes on Windows, use a consistent runtime such as `/MD` across all participants. The typed API uses C linkage for explicit symbol exports but passes C++ objects and strings, so it is not a cross-toolchain C ABI. Only the bridge functions are -exported; the private static Boost runtime must not interpose on another -library's registrations. +supported consumer entry points. ELF and macOS export lists hide the static +Boost implementation; Windows uses explicit exports. + +Precompiled static Boost libraries on Windows can carry MSVC `dllexport` +directives that force additional implementation helpers into the owner's DLL +export table. We permit narrowly identified helpers for `codecvt_null`, +`singleton_module`, and singleton instances for text-archive serializer maps, +extended type-info registries, and void-cast registrations. They are unsupported +implementation details; consumers must not import or call them. Windows resolves +DLL imports through the named provider, and the required integration test checks +that a separate static Boost runtime retains its own registrations before and +after both managed consumers load. + +The CI exception applies only to Windows with static Boost and uses the exact +reviewed symbol names in +[`windows_static_boost_exports.txt`](tests/fixtures/serialization_library/windows_static_boost_exports.txt) +for the pinned Boost/MSVC fixture. Unknown exports and missing bridge functions +still fail. Boost or toolchain upgrades that change these names require review; +there is no blanket Boost namespace exemption. Windows shared Boost, Linux, +and macOS retain exact bridge-only export checks. Call the helper and pass `SERIALIZATION_LIBRARY` only when serialization is enabled in the consuming project. Pairing the latter argument with a false diff --git a/tests/fixtures/serialization_library/windows_static_boost_exports.txt b/tests/fixtures/serialization_library/windows_static_boost_exports.txt new file mode 100644 index 0000000..64c5380 --- /dev/null +++ b/tests/fixtures/serialization_library/windows_static_boost_exports.txt @@ -0,0 +1,34 @@ +# Exact compiler-forced exports observed with the pinned Boost 1.88 x64 MSVC +# static libraries. This exception applies ONLY to Windows + static Boost. +# These are implementation details, not supported consumer entry points. +# No namespace wildcards: Boost/toolchain changes require explicit review. +# Groups: codecvt_null; singleton_module; singleton for text archive +# serializer maps, extended type-info registries, and void-cast registrations. +??0?$codecvt_null@_W@archive@boost@@QEAA@_K@Z +??1?$codecvt_null@_W@archive@boost@@UEAA@XZ +??_F?$codecvt_null@_W@archive@boost@@QEAAXXZ +?do_always_noconv@?$codecvt_null@_W@archive@boost@@EEBA_NXZ +?do_encoding@?$codecvt_null@_W@archive@boost@@EEBAHXZ +?do_in@?$codecvt_null@_W@archive@boost@@EEBAHAEAU_Mbstatet@@PEBD1AEAPEBDPEA_W3AEAPEA_W@Z +?do_max_length@?$codecvt_null@_W@archive@boost@@EEBAHXZ +?do_out@?$codecvt_null@_W@archive@boost@@EEBAHAEAU_Mbstatet@@PEB_W1AEAPEB_WPEAD3AEAPEAD@Z +?get_const_instance@?$singleton@V?$map@Vtext_iarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SAAEBV?$map@Vtext_iarchive@archive@boost@@@extra_detail@detail@archive@3@XZ +?get_const_instance@?$singleton@V?$map@Vtext_oarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SAAEBV?$map@Vtext_oarchive@archive@boost@@@extra_detail@detail@archive@3@XZ +?get_const_instance@?$singleton@V?$multiset@PEBVextended_type_info@serialization@boost@@Ukey_compare@detail@23@V?$allocator@PEBVextended_type_info@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEBV?$multiset@PEBVextended_type_info@serialization@boost@@Ukey_compare@detail@23@V?$allocator@PEBVextended_type_info@serialization@boost@@@std@@@std@@XZ +?get_const_instance@?$singleton@V?$multiset@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@Utype_compare@234@V?$allocator@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEBV?$multiset@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@Utype_compare@234@V?$allocator@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@@std@@@std@@XZ +?get_const_instance@?$singleton@V?$set@PEBVvoid_caster@void_cast_detail@serialization@boost@@Uvoid_caster_compare@234@V?$allocator@PEBVvoid_caster@void_cast_detail@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEBV?$set@PEBVvoid_caster@void_cast_detail@serialization@boost@@Uvoid_caster_compare@234@V?$allocator@PEBVvoid_caster@void_cast_detail@serialization@boost@@@std@@@std@@XZ +?get_lock@singleton_module@serialization@boost@@AEAAAEA_NXZ +?get_mutable_instance@?$singleton@V?$map@Vtext_iarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SAAEAV?$map@Vtext_iarchive@archive@boost@@@extra_detail@detail@archive@3@XZ +?get_mutable_instance@?$singleton@V?$map@Vtext_oarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SAAEAV?$map@Vtext_oarchive@archive@boost@@@extra_detail@detail@archive@3@XZ +?get_mutable_instance@?$singleton@V?$multiset@PEBVextended_type_info@serialization@boost@@Ukey_compare@detail@23@V?$allocator@PEBVextended_type_info@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEAV?$multiset@PEBVextended_type_info@serialization@boost@@Ukey_compare@detail@23@V?$allocator@PEBVextended_type_info@serialization@boost@@@std@@@std@@XZ +?get_mutable_instance@?$singleton@V?$multiset@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@Utype_compare@234@V?$allocator@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEAV?$multiset@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@Utype_compare@234@V?$allocator@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@@std@@@std@@XZ +?get_mutable_instance@?$singleton@V?$set@PEBVvoid_caster@void_cast_detail@serialization@boost@@Uvoid_caster_compare@234@V?$allocator@PEBVvoid_caster@void_cast_detail@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEAV?$set@PEBVvoid_caster@void_cast_detail@serialization@boost@@Uvoid_caster_compare@234@V?$allocator@PEBVvoid_caster@void_cast_detail@serialization@boost@@@std@@@std@@XZ +?is_destroyed@?$singleton@V?$map@Vtext_iarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SA_NXZ +?is_destroyed@?$singleton@V?$map@Vtext_oarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SA_NXZ +?is_destroyed@?$singleton@V?$multiset@PEBVextended_type_info@serialization@boost@@Ukey_compare@detail@23@V?$allocator@PEBVextended_type_info@serialization@boost@@@std@@@std@@@serialization@boost@@SA_NXZ +?is_destroyed@?$singleton@V?$multiset@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@Utype_compare@234@V?$allocator@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@@std@@@std@@@serialization@boost@@SA_NXZ +?is_destroyed@?$singleton@V?$set@PEBVvoid_caster@void_cast_detail@serialization@boost@@Uvoid_caster_compare@234@V?$allocator@PEBVvoid_caster@void_cast_detail@serialization@boost@@@std@@@std@@@serialization@boost@@SA_NXZ +?is_locked@singleton_module@serialization@boost@@QEAA_NXZ +?lock@?1??get_lock@singleton_module@serialization@boost@@AEAAAEA_NXZ@4_NA +?lock@singleton_module@serialization@boost@@QEAAXXZ +?unlock@singleton_module@serialization@boost@@QEAAXXZ diff --git a/tests/test_serialization_runtime.py b/tests/test_serialization_runtime.py index e0d7ede..fdbc4c6 100644 --- a/tests/test_serialization_runtime.py +++ b/tests/test_serialization_runtime.py @@ -14,6 +14,44 @@ FIXTURE = ROOT / 'tests/fixtures/serialization_library' +def check_public_exports(actual, expected, platform, linkage): + """Require every bridge and reject exports outside the reviewed exception.""" + allowed = set() + if platform == 'win32' and linkage == 'static': + # Precompiled MSVC objects carry dllexport directives that our DLL's + # source-level visibility settings cannot suppress. Match exact names, + # never a Boost namespace pattern or exports discovered during this run. + allowed = {line for line in + (FIXTURE / 'windows_static_boost_exports.txt').read_text().splitlines() + if line and not line.startswith('#')} + missing = expected - actual + unexpected = actual - expected - allowed + if missing or unexpected: + raise AssertionError(f'Missing bridge exports: {sorted(missing)}\n' + f'Unexpected exports: {sorted(unexpected)}') + + +class TestExportPolicy(unittest.TestCase): + def test_exception_is_limited_to_windows_static(self): + bridge = {'gtwrap_test_save', 'gtwrap_test_load'} + actual = bridge | {'?lock@singleton_module@serialization@boost@@QEAAXXZ'} + check_public_exports(actual, bridge, 'win32', 'static') + for platform, linkage in (('win32', 'shared'), ('linux', 'static'), + ('darwin', 'static')): + with self.subTest(platform=platform, linkage=linkage): + with self.assertRaisesRegex(AssertionError, 'Unexpected exports: \\[.+'): + check_public_exports(actual, bridge, platform, linkage) + + def test_unknown_boost_helpers_are_rejected(self): + with self.assertRaisesRegex(AssertionError, 'Unexpected exports: \\[.+'): + check_public_exports({'?unknown_helper@serialization@boost@@'}, + set(), 'win32', 'static') + + def test_exception_cannot_hide_missing_bridges(self): + with self.assertRaisesRegex(AssertionError, 'Missing bridge exports: \\[.+'): + check_public_exports(set(), {'gtwrap_test_load'}, 'win32', 'static') + + def run(args, **kwargs): result = subprocess.run([str(a) for a in args], capture_output=True, text=True, **kwargs) @@ -132,4 +170,4 @@ def exercise(self, directory, linkage): # Report export-policy failures without skipping installed-package # runtime validation. Both consumption modes must still be exercised. with self.subTest(consumption=label, check='public exports'): - self.assertEqual(actual, symbols) + check_public_exports(actual, symbols, sys.platform, linkage)