diff --git a/.github/workflows/serialization.yml b/.github/workflows/serialization.yml new file mode 100644 index 0000000..a5607a6 --- /dev/null +++ b/.github/workflows/serialization.yml @@ -0,0 +1,83 @@ +name: Central serialization runtime + +on: [pull_request] + +jobs: + runtime: + name: Serialization (${{ matrix.os }}, ${{ matrix.linkage }} Boost) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + linkage: [static, shared] + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v5 + - uses: actions/checkout@v5 + with: + repository: microsoft/vcpkg + ref: ef7dbf94b9198bc58f45951adcf1f041fcbc5ea0 + path: _vcpkg + - uses: ilammy/msvc-dev-cmd@v1 + if: runner.os == 'Windows' + - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: py312 + python-version: '3.12' + auto-activate-base: false + - name: Python and build tools + run: python -m pip install -e . pytest cmake ninja + - name: Bootstrap vcpkg (Unix) + if: runner.os != 'Windows' + run: ./_vcpkg/bootstrap-vcpkg.sh -disableMetrics + - name: Bootstrap vcpkg (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: ./_vcpkg/bootstrap-vcpkg.bat -disableMetrics + - name: Configure Boost triplet + run: | + mkdir -p _triplets + architecture=x64 + if [[ "$(uname -m)" == arm64 ]]; then architecture=arm64; fi + linkage=${{ matrix.linkage }} + if [[ "$linkage" == shared ]]; then linkage=dynamic; fi + cat > _triplets/managed.cmake <> _triplets/managed.cmake + echo 'set(VCPKG_C_FLAGS "-fPIC")' >> _triplets/managed.cmake + echo 'set(VCPKG_CXX_FLAGS "-fPIC")' >> _triplets/managed.cmake + elif [[ "$RUNNER_OS" == macOS ]]; then + echo 'set(VCPKG_CMAKE_SYSTEM_NAME Darwin)' >> _triplets/managed.cmake + fi + - uses: actions/cache/restore@v4 + id: boost-cache + with: + path: _vcpkg/installed + key: serialization-v1-${{ runner.os }}-${{ runner.arch }}-${{ matrix.linkage }}-ef7dbf94 + - name: Install Boost.Serialization + run: ./_vcpkg/vcpkg install boost-serialization:managed --overlay-triplets=_triplets + - uses: actions/cache/save@v4 + if: steps.boost-cache.outputs.cache-hit != 'true' + with: + path: _vcpkg/installed + key: serialization-v1-${{ runner.os }}-${{ runner.arch }}-${{ matrix.linkage }}-ef7dbf94 + - name: Run required runtime tests + env: + GTWRAP_TEST_SERIALIZATION: '1' + GTWRAP_TEST_BOOST_LINKAGE: ${{ matrix.linkage }} + VCPKG_TARGET_TRIPLET: managed + CMAKE_GENERATOR: Ninja + run: | + # Build against the same native C++ runtime as the vcpkg libraries. + unset CC CXX CFLAGS CXXFLAGS CPPFLAGS LDFLAGS + export CMAKE_TOOLCHAIN_FILE="$PWD/_vcpkg/scripts/buildsystems/vcpkg.cmake" + export VCPKG_OVERLAY_TRIPLETS="$PWD/_triplets" + cmake -S . -B build -DWRAP_PYTHON_VERSION=3.12 + cmake --build build --target testSerialization.run --parallel 6 diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c8b33a..97ea3b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,7 +75,8 @@ endif() # Install wrapping scripts as binaries to `CMAKE_INSTALL_PREFIX/bin` so they can # be invoked for wrapping. We use DESTINATION (instead of TYPE) so we can # support older CMake versions. -install(PROGRAMS scripts/pybind_wrap.py scripts/matlab_wrap.py +install(PROGRAMS scripts/pybind_wrap.py scripts/pybind_serialization.py + scripts/matlab_wrap.py DESTINATION "${GTWRAP_BIN_INSTALL_DIR}") # Install the matlab.h file to `CMAKE_INSTALL_PREFIX/lib/gtwrap/matlab.h`. @@ -89,3 +90,14 @@ message(STATUS "${gtwrap} CMake path : ${GTWRAP_CMAKE_INSTALL_DIR}") message(STATUS "${gtwrap} library path : ${GTWRAP_LIB_INSTALL_DIR}") message(STATUS "${gtwrap} binary path : ${GTWRAP_BIN_INSTALL_DIR}") message(STATUS "${gtwrap} header path : ${GTWRAP_INCLUDE_INSTALL_DIR}") + +# Explicit test targets work with the caller's configured Python environment. +add_custom_target(testWrap.run + COMMAND "${PYTHON_EXECUTABLE}" -m pytest tests -q -o "addopts=" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" USES_TERMINAL) +add_custom_target(testSerialization.run + COMMAND "${PYTHON_EXECUTABLE}" -m pytest + tests/test_serialization_discovery.py tests/test_serialization_library.py + tests/test_serialization_bindings.py tests/test_serialization_runtime.py + -q -o "addopts=" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" USES_TERMINAL) diff --git a/DOCS.md b/DOCS.md index 8303045..40843d2 100644 --- a/DOCS.md +++ b/DOCS.md @@ -252,6 +252,150 @@ The python wrapper supports keyword arguments for functions/methods. Hence, the - Unfortunately, this means that aliases can no longer be used. - Similarly, there can be multiple `preamble.h` and `specializations.h` files. Each of these should match the module file name. +### Centrally managed Python serialization + +`pybind_add_serialization_library` creates an opt-in shared library that owns +Boost export registrations **and** the text-archive save/load operations used +by Python bindings. Multiple extensions link that same library. Archive +construction stays in the owner, including when Boost.Serialization itself is +static; consumers do not need their own copy of its registration tables. +Existing wrapping is unchanged unless a serialization library is selected. + +The API is independent of any particular C++ project. Supply interface files, +headers defining their types, and the domain-library targets they require: + +```cmake +# Use find_package(gtwrap CONFIG REQUIRED) for an installed wrap package, +# or include(PybindWrap) with wrap/cmake on CMAKE_MODULE_PATH for a source tree. +find_package(Boost REQUIRED COMPONENTS serialization) + +pybind_add_serialization_library(shared_archives + INTERFACES common.i module_a.i module_b.i + HEADERS example/types.h + LINK_LIBRARIES example_types) + +pybind_wrap(module_a "${CMAKE_CURRENT_SOURCE_DIR}/module_a.i" + "module_a.cpp" "module_a" "example" "" + "${CMAKE_CURRENT_SOURCE_DIR}/module.tpl" "example_types" "" ON + SERIALIZATION_LIBRARY shared_archives) +pybind_wrap(module_b "${CMAKE_CURRENT_SOURCE_DIR}/module_b.i" + "module_b.cpp" "module_b" "example" "" + "${CMAKE_CURRENT_SOURCE_DIR}/module.tpl" "example_types" "" ON + SERIALIZATION_LIBRARY shared_archives) +``` + +`example_types` supplies the C++ headers' include paths, compile definitions, +and domain-library dependencies. The usual wrapping template and preamble +requirements still apply. The source-tree module resolves the generator script +paths itself; no extra script-variable assignment is necessary. + +The library helper accepts these keyword arguments: + +| Argument | Meaning | +| --- | --- | +| `INTERFACES` | Required list of interfaces containing the serializable types. Relative paths resolve against the current source directory. | +| `HEADERS` | Required C++ header include names defining those types. | +| `LINK_LIBRARIES` | Domain targets and their usage requirements; these must not depend on the consuming Python extensions. | +| `NAMESPACE_PREFIXES` | Optional list of namespace prefixes ending in `::`; absent means all serializable declarations. | +| `IGNORE_CLASSES` | Optional fully qualified instantiated C++ types to exclude. | +| `BOOST_SERIALIZATION_TARGET` | Existing static or shared Boost.Serialization target; defaults to `Boost::serialization`. | + +Selection uses the existing `serialize`/`serializable` markers and +`@serializable` template typedefs after instantiation. Duplicate types across +interfaces are coalesced. Empty selections and conflicting archive identifiers +are errors. Export names retain the ordinary wrapper's existing spelling, +including its comma-safe template aliases. + +Supply the complete serialization domain: both the types whose Python +save/load methods use the owner and every concrete polymorphic type reachable +through their serialized members. Wrap cannot discover arbitrary C++ runtime +subclasses or inspect the bodies of C++ serialization methods. A family-only +registration list is insufficient if the containing object's archive is still +created in an extension with its own static Boost runtime. Use one owner for +all participating boundary types and their polymorphic dependencies. + +For selected types, generated `.serialize()`, `.deserialize()`, and pickle +methods call typed functions in the library. Their normal export macros are +suppressed. Unselected types retain ordinary wrapping. Consumers include a +private generated API header through the existing `{includes}` placeholder; +no new template placeholders, manual key headers, implementation fragments, or +special owner translation units are needed. The library's registration headers +remain private to its implementation. + +This mode preserves the existing text-archive protocol and one-element pickle +state. It does not add binary/XML archives or customizable archive adapters. +Boost archive exceptions become `std::runtime_error` at the library boundary +and Python `RuntimeError` through pybind11. + +The owner is always a `SHARED` target, regardless of `BUILD_SHARED_LIBS`. Its +Boost dependency is private and can be static or shared. Static dependencies +must be suitable for linking into a shared library (including PIC where +required). Do not combine incompatible Boost versions, C++ ABIs, or C++ runtimes; +on Windows, use a consistent runtime such as `/MD` across all participants. +The typed API uses C linkage for explicit symbol exports but passes C++ objects +and strings, so it is not a cross-toolchain C ABI. Only the bridge functions are +supported consumer entry points. ELF and macOS export lists hide the static +Boost implementation; Windows uses explicit exports. + +Precompiled static Boost libraries on Windows can carry MSVC `dllexport` +directives that force additional implementation helpers into the owner's DLL +export table. We permit narrowly identified helpers for `codecvt_null`, +`singleton_module`, and singleton instances for text-archive serializer maps, +extended type-info registries, and void-cast registrations. They are unsupported +implementation details; consumers must not import or call them. Windows resolves +DLL imports through the named provider, and the required integration test checks +that a separate static Boost runtime retains its own registrations before and +after both managed consumers load. + +The CI exception applies only to Windows with static Boost and uses the exact +reviewed symbol names in +[`windows_static_boost_exports.txt`](tests/fixtures/serialization_library/windows_static_boost_exports.txt) +for the pinned Boost/MSVC fixture. Unknown exports and missing bridge functions +still fail. Boost or toolchain upgrades that change these names require review; +there is no blanket Boost namespace exemption. Windows shared Boost, Linux, +and macOS retain exact bridge-only export checks. + +Call the helper and pass `SERIALIZATION_LIBRARY` only when serialization is +enabled in the consuming project. Pairing the latter argument with a false +serialization flag is a configuration error. MATLAB generation is independent +of this opt-in Python facility. + +#### Installation and migration + +The caller owns the runtime package layout. Install the shared owner alongside +or in a location reachable from every consuming extension using ordinary +`install(TARGETS ...)`. Configure relative install RPATHs as appropriate (for +example `$ORIGIN` on ELF or `@loader_path` on macOS), or colocate the DLL with +Windows extensions. Package a shared Boost runtime according to the project's +existing dependency policy. Wrap does not choose installation directories or +rewrite third-party library install names. The generated manifest and headers +are build inputs; Python users do not need them at runtime. Normal linking +keeps the owner loaded while its extension consumers are loaded. + +To migrate existing bindings, identify the complete domain, create the owner, +and attach it to each consumer. Remove handwritten keys and export +implementations for the managed types, including any special-case header +exclusions. Keep existing serialization feature guards. Validate stable-only +use, both extension import orders, calls after both are loaded, legacy archive +exchange, and serialization-disabled builds before deploying the migration. + +The generic integration example in `tests/fixtures/serialization_library` +builds two independent Python consumers and a legacy archive producer. It also +checks installed source-tree/package consumption and coexistence with an +unrelated static Boost runtime. Run it using the configured Python environment: + +```sh +cmake -S . -B build +cd build +GTWRAP_TEST_SERIALIZATION=1 make -j6 testSerialization.run +``` + +Set `GTWRAP_TEST_BOOST_LINKAGE=static` or `shared` to choose one linkage; by +default both are tested. Supply `CMAKE_PREFIX_PATH`, or a toolchain and +`VCPKG_TARGET_TRIPLET`, to select the intended Boost installation. The dedicated +CI matrix provisions Boost and treats missing dependencies as errors. The +ordinary lightweight suite skips this integration test unless explicitly enabled. + ## Pybind Callable Adapters The Python generator uses explicit full-signature C++ callable-pointer casts for diff --git a/cmake/PybindWrap.cmake b/cmake/PybindWrap.cmake index 2c98b69..cb8ef1d 100644 --- a/cmake/PybindWrap.cmake +++ b/cmake/PybindWrap.cmake @@ -5,6 +5,91 @@ else() set(GTWRAP_PACKAGE_DIR ${CMAKE_CURRENT_LIST_DIR}/..) endif() +# Installed packages set these before including this module. Source-tree users +# need no separate script assignments. +if(NOT PYBIND_WRAP_SCRIPT) + set(PYBIND_WRAP_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/../scripts/pybind_wrap.py") +endif() +if(NOT PYBIND_SERIALIZATION_SCRIPT) + set(PYBIND_SERIALIZATION_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/../scripts/pybind_serialization.py") +endif() +file(GLOB_RECURSE GTWRAP_GENERATOR_SOURCES CONFIGURE_DEPENDS + "${GTWRAP_PACKAGE_DIR}/gtwrap/*.py") + +# Create one shared archive owner. Callers supply the complete polymorphic +# type family and use SERIALIZATION_LIBRARY in every participating pybind_wrap. +function(pybind_add_serialization_library target) + cmake_parse_arguments(REG "" "BOOST_SERIALIZATION_TARGET" + "INTERFACES;HEADERS;LINK_LIBRARIES;NAMESPACE_PREFIXES;IGNORE_CLASSES" ${ARGN}) + if(REG_UNPARSED_ARGUMENTS OR REG_KEYWORDS_MISSING_VALUES) + message(FATAL_ERROR "Invalid pybind_add_serialization_library arguments") + endif() + if(NOT REG_INTERFACES OR NOT REG_HEADERS) + message(FATAL_ERROR "A serialization library requires INTERFACES and HEADERS") + endif() + if(NOT target MATCHES "^[A-Za-z0-9_-]+$") + message(FATAL_ERROR "Serialization target names may contain letters, digits, _ and -") + endif() + if(NOT REG_BOOST_SERIALIZATION_TARGET) + set(REG_BOOST_SERIALIZATION_TARGET Boost::serialization) + endif() + if(NOT TARGET ${REG_BOOST_SERIALIZATION_TARGET}) + message(FATAL_ERROR "Provide an existing Boost.Serialization target") + endif() + set(directory "${CMAKE_CURRENT_BINARY_DIR}/gtwrap_serialization/${target}") + set(arguments --name "${target}" --out-dir "${directory}") + set(interfaces "") + foreach(interface IN LISTS REG_INTERFACES) + get_filename_component(interface "${interface}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") + list(APPEND interfaces "${interface}") + list(APPEND arguments --src "${interface}") + endforeach() + foreach(header IN LISTS REG_HEADERS) + list(APPEND arguments --include "${header}") + endforeach() + foreach(prefix IN LISTS REG_NAMESPACE_PREFIXES) + list(APPEND arguments --namespace-prefix "${prefix}") + endforeach() + foreach(type IN LISTS REG_IGNORE_CLASSES) + list(APPEND arguments --ignore "${type}") + endforeach() + if(UNIX) + set(separator ":") + else() + set(separator ";") + endif() + set(outputs "") + foreach(suffix .h _keys.h .cpp .json .map .exports) + list(APPEND outputs "${directory}/${target}${suffix}") + endforeach() + add_custom_command(OUTPUT ${outputs} + COMMAND ${CMAKE_COMMAND} -E env + "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${separator}$ENV{PYTHONPATH}" + "${PYTHON_EXECUTABLE}" "${PYBIND_SERIALIZATION_SCRIPT}" ${arguments} + DEPENDS ${interfaces} "${PYBIND_SERIALIZATION_SCRIPT}" ${GTWRAP_GENERATOR_SOURCES} + VERBATIM) + add_custom_target(${target}_generate DEPENDS ${outputs}) + add_library(${target} SHARED "${directory}/${target}.cpp") + add_dependencies(${target} ${target}_generate) + target_compile_features(${target} PUBLIC cxx_std_17) + target_link_libraries(${target} PRIVATE ${REG_BOOST_SERIALIZATION_TARGET} + PUBLIC ${REG_LINK_LIBRARIES}) + target_include_directories(${target} PUBLIC "$") + set_target_properties(${target} PROPERTIES + CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON + WINDOWS_EXPORT_ALL_SYMBOLS OFF + GTWRAP_SERIALIZATION_MANIFEST "${directory}/${target}.json") + if(APPLE) + # Match pybind_wrap's cross-DSO RTTI behavior for header-defined types. + target_compile_options(${target} PRIVATE -fvisibility-ms-compat) + target_link_options(${target} PRIVATE "LINKER:-exported_symbols_list,${directory}/${target}.exports") + set_property(TARGET ${target} APPEND PROPERTY LINK_DEPENDS "${directory}/${target}.exports") + elseif(UNIX) + target_link_options(${target} PRIVATE "LINKER:--version-script=${directory}/${target}.map") + set_property(TARGET ${target} APPEND PROPERTY LINK_DEPENDS "${directory}/${target}.map") + endif() +endfunction() + # Get the Python version include(GtwrapUtils) message(STATUS "Checking Python Version") @@ -50,7 +135,30 @@ function( libs dependencies) set(ExtraMacroArgs ${ARGN}) - list(GET ExtraMacroArgs 0 USE_BOOST_SERIALIZATION) + set(USE_BOOST_SERIALIZATION OFF) + if(ExtraMacroArgs) + list(POP_FRONT ExtraMacroArgs USE_BOOST_SERIALIZATION) + endif() + cmake_parse_arguments(WRAP "" "SERIALIZATION_LIBRARY" "" ${ExtraMacroArgs}) + if(WRAP_UNPARSED_ARGUMENTS OR WRAP_KEYWORDS_MISSING_VALUES) + message(FATAL_ERROR "Invalid pybind_wrap optional arguments") + endif() + set(_serialization_args "") + set(_serialization_depends "") + if(WRAP_SERIALIZATION_LIBRARY) + if(NOT USE_BOOST_SERIALIZATION) + message(FATAL_ERROR "SERIALIZATION_LIBRARY requires serialization enabled") + endif() + if(NOT TARGET ${WRAP_SERIALIZATION_LIBRARY}) + message(FATAL_ERROR "Unknown serialization library ${WRAP_SERIALIZATION_LIBRARY}") + endif() + get_target_property(_manifest ${WRAP_SERIALIZATION_LIBRARY} GTWRAP_SERIALIZATION_MANIFEST) + if(NOT _manifest) + message(FATAL_ERROR "Target was not created by pybind_add_serialization_library") + endif() + set(_serialization_args --serialization-manifest "${_manifest}") + set(_serialization_depends "${_manifest}" ${WRAP_SERIALIZATION_LIBRARY}_generate) + endif() if(USE_BOOST_SERIALIZATION) set(_WRAP_BOOST_ARG "--use-boost-serialization") else(USE_BOOST_SERIALIZATION) @@ -90,8 +198,8 @@ function( --out "${cpp_file}" --module_name ${module_name} --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes} --template ${module_template} --is_submodule ${_WRAP_BOOST_ARG} - --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}" - DEPENDS "${interface_file}" ${module_template} "${module_name}/specializations/${interface}.h" "${module_name}/preamble/${interface}.h" + --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}" ${_serialization_args} + DEPENDS ${_serialization_depends} ${GTWRAP_GENERATOR_SOURCES} "${PYBIND_WRAP_SCRIPT}" "${interface_file}" ${module_template} "${module_name}/specializations/${interface}.h" "${module_name}/preamble/${interface}.h" VERBATIM) endforeach() @@ -109,8 +217,8 @@ function( --out "${generated_cpp}" --module_name ${module_name} --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes} --template ${module_template} ${_WRAP_BOOST_ARG} - --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}" - DEPENDS "${main_interface}" ${module_template} "${module_name}/specializations/${main_interface_name}.h" "${module_name}/specializations/${main_interface_name}.h" + --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}" ${_serialization_args} + DEPENDS ${_serialization_depends} ${GTWRAP_GENERATOR_SOURCES} "${PYBIND_WRAP_SCRIPT}" "${main_interface}" ${module_template} "${module_name}/specializations/${main_interface_name}.h" "${module_name}/specializations/${main_interface_name}.h" VERBATIM) add_custom_target(pybind_wrap_${module_name} DEPENDS ${cpp_files}) @@ -130,6 +238,10 @@ function( endif() add_dependencies(${target} pybind_wrap_${module_name}) + if(WRAP_SERIALIZATION_LIBRARY) + target_link_libraries(${target} PRIVATE ${WRAP_SERIALIZATION_LIBRARY}) + add_dependencies(${target} ${WRAP_SERIALIZATION_LIBRARY}_generate) + endif() if(NOT "${libs}" STREQUAL "") target_link_libraries(${target} PRIVATE "${libs}") diff --git a/cmake/gtwrapConfig.cmake.in b/cmake/gtwrapConfig.cmake.in index 182fa7e..0374ec2 100644 --- a/cmake/gtwrapConfig.cmake.in +++ b/cmake/gtwrapConfig.cmake.in @@ -9,15 +9,16 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") # Used in the *Wrap.cmake files. set_and_check(GTWRAP_PYTHON_PACKAGE_DIR @PACKAGE_INSTALL_LIB_DIR@) +# Set the variables for the wrapping scripts to be used in the build. +set_and_check(PYBIND_WRAP_SCRIPT "@PACKAGE_INSTALL_BIN_DIR@/pybind_wrap.py") +set_and_check(PYBIND_SERIALIZATION_SCRIPT "@PACKAGE_INSTALL_BIN_DIR@/pybind_serialization.py") +set_and_check(MATLAB_WRAP_SCRIPT "@PACKAGE_INSTALL_BIN_DIR@/matlab_wrap.py") + # Load all the CMake scripts from the standard location include(@PACKAGE_INSTALL_CMAKE_DIR@/PybindWrap.cmake) include(@PACKAGE_INSTALL_CMAKE_DIR@/MatlabWrap.cmake) include(@PACKAGE_INSTALL_CMAKE_DIR@/GtwrapUtils.cmake) -# Set the variables for the wrapping scripts to be used in the build. -set_and_check(PYBIND_WRAP_SCRIPT "@PACKAGE_INSTALL_BIN_DIR@/pybind_wrap.py") -set_and_check(MATLAB_WRAP_SCRIPT "@PACKAGE_INSTALL_BIN_DIR@/matlab_wrap.py") - # Load the same pybind11 provider selected when gtwrap was configured. if(@GTWRAP_USE_SYSTEM_PYBIND@) find_package(pybind11 REQUIRED) diff --git a/gtwrap/pybind_wrapper.py b/gtwrap/pybind_wrapper.py index 7b6dbf5..8c1e167 100755 --- a/gtwrap/pybind_wrapper.py +++ b/gtwrap/pybind_wrapper.py @@ -13,6 +13,8 @@ # pylint: disable=too-many-arguments, too-many-instance-attributes, no-self-use, no-else-return, too-many-arguments, unused-format-string-argument, line-too-long, consider-using-f-string import re +import json +from gtwrap.serialization import export_name from pathlib import Path from typing import List @@ -82,12 +84,24 @@ def __init__(self, use_boost_serialization=False, ignore_classes=(), module_template="", - xml_source=""): + xml_source="", + serialization_manifest=None): self.module_name = module_name self.top_module_namespaces = top_module_namespaces self.use_boost_serialization = use_boost_serialization self.ignore_classes = ignore_classes self._serializing_classes = [] + self._managed_serialization = {} + self._serialization_header = None + if serialization_manifest is not None: + if not use_boost_serialization: + raise ValueError('a serialization library requires serialization enabled') + manifest = json.loads(Path(serialization_manifest).read_text(encoding='utf-8')) + if manifest.get('version') != 1: + raise ValueError('unsupported serialization manifest version') + self._serialization_header = manifest['header'] + self._managed_serialization = { + entry['cpp_type']: entry for entry in manifest['entries']} self.module_template = module_template self.python_keywords = [ 'lambda', 'False', 'def', 'if', 'raise', 'None', 'del', 'import', @@ -184,6 +198,21 @@ def wrap_ctors(self, my_class): def _wrap_serialization(self, cpp_class): """Helper method to add serialize, deserialize and pickle methods to the wrapped class.""" + if cpp_class in self._managed_serialization: + entry = self._managed_serialization[cpp_class] + save, load = entry['save'], entry['load'] + indent = self.method_indent + return ( + f'{indent}.def("serialize", [](const {cpp_class}* self){{ ' + f'std::string result; {save}(self, &result); return result; }})' + f'{indent}.def("deserialize", []({cpp_class}* self, const std::string& serialized){{ ' + f'{load}(&serialized, self); }}, py::arg("serialized"))' + f'{indent}.def(py::pickle(' + f'[](const {cpp_class}& value){{ std::string result; ' + f'{save}(&value, &result); return py::make_tuple(result); }}, ' + f'[](py::tuple state){{ {cpp_class} value; ' + f'auto input = state[0].cast(); ' + f'{load}(&input, &value); return value; }}))') if not cpp_class in self._serializing_classes: self._serializing_classes.append(cpp_class) @@ -910,16 +939,18 @@ def wrap_file(self, wrapped_declarations, wrapped_bindings, includes = self.wrap_namespace( module) + if self._serialization_header: + includes = f'#include "{self._serialization_header}"\n' + includes + if self.use_boost_serialization: includes += "#include " # Export classes for serialization. boost_class_export = "" for cpp_class in self._serializing_classes: - new_name = cpp_class + new_name = self.serialization_export_name(cpp_class) # The boost's macro doesn't like commas, so we have to typedef. - if ',' in cpp_class: - new_name = re.sub("[,:<> ]", "", cpp_class) + if new_name != cpp_class: boost_class_export += "typedef {cpp_class} {new_name};\n".format( # noqa cpp_class=cpp_class, new_name=new_name) @@ -987,6 +1018,8 @@ def wrap_file(self, submodules_init="", ) + serialization_export_name = staticmethod(export_name) + def wrap_submodule(self, source): """ Wrap a list of submodule files, i.e. a set of interface files which are diff --git a/gtwrap/serialization.py b/gtwrap/serialization.py new file mode 100644 index 0000000..ed89c8f --- /dev/null +++ b/gtwrap/serialization.py @@ -0,0 +1,68 @@ +"""Discover and generate centrally owned Boost.Serialization support.""" + +from dataclasses import dataclass +import re + +from gtwrap import interface_parser as parser +from gtwrap import template_instantiator as instantiator + + +def export_name(cpp_type): + """Preserve the identifiers historically emitted by the Python wrapper.""" + return re.sub('[,:<> ]', '', cpp_type) if ',' in cpp_type else cpp_type + + +def is_serializable(cls): + """Use the same instantiated method markers as ordinary Python wrapping.""" + return any(method.to_cpp() in ('serialize', 'serializable') + for method in list(cls.methods) + list(cls.static_methods)) + + +@dataclass(frozen=True) +class Export: + cpp_type: str + name: str + source: str + + +def discover(sources, namespace_prefixes=(), ignore_classes=()): + """Collect exports from ``(source name, interface text)`` pairs in order. + + Selection happens after template instantiation, so aliases and explicit + instantiations have exactly the names used by the ordinary wrapper. + """ + for prefix in namespace_prefixes: + if not prefix or not prefix.endswith('::'): + raise ValueError('namespace prefixes must be nonempty and end in ::') + ignored = set(ignore_classes) + exports = {} + names = {} + + def visit(namespace, source): + for element in namespace.content: + if isinstance(element, parser.Namespace): + visit(element, source) + elif isinstance(element, instantiator.InstantiatedClass): + cpp_type = element.to_cpp() + if (cpp_type in ignored or not is_serializable(element) + or (namespace_prefixes and not any( + cpp_type.startswith(p) for p in namespace_prefixes))): + continue + name = export_name(cpp_type) + if name in names and names[name].cpp_type != cpp_type: + previous = names[name] + raise ValueError( + f'{source}: export identifier {name!r} for {cpp_type} ' + f'conflicts with {previous.cpp_type} in {previous.source}') + entry = Export(cpp_type, name, source) + exports.setdefault(cpp_type, entry) + names.setdefault(name, entry) + + source_names = [] + for source, content in sources: + source_names.append(source) + module = parser.Module.parse_string(content, source_name=source) + visit(instantiator.instantiate_namespace(module), source) + if not exports: + raise ValueError(f'{", ".join(source_names)}: no serializable types matched') + return list(exports.values()) diff --git a/gtwrap/serialization_library.py b/gtwrap/serialization_library.py new file mode 100644 index 0000000..b386603 --- /dev/null +++ b/gtwrap/serialization_library.py @@ -0,0 +1,114 @@ +"""Generate a private archive runtime and its typed consumer API.""" + +import hashlib +import json +from pathlib import Path + +from gtwrap.serialization import discover + + +def generate_library(name, sources, headers, namespace_prefixes=(), + ignore_classes=()): + """Return generated files indexed by basename; callers choose placement. + + C linkage gives the linker an exact export list; parameters still require + the same C++ ABI and runtime in consumers and the library. + """ + if not name or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-' for c in name): + raise ValueError('library name must contain only letters, digits, _ or -') + if not headers or any('\n' in h or '\r' in h or '>' in h for h in headers): + raise ValueError('supply C++ headers defining the managed types') + exports = discover(sources, namespace_prefixes, ignore_classes) + library_id = hashlib.sha256(name.encode()).hexdigest()[:16] + macro = f'GTWRAP_SERIALIZATION_{library_id.upper()}_API' + building = macro + '_BUILDING' + api_name = name + '.h' + keys_name = name + '_keys.h' + entries = [] + for export in exports: + type_id = hashlib.sha256(export.cpp_type.encode()).hexdigest()[:16] + symbol = f'gtwrap_{library_id}_{type_id}' + entries.append(dict(cpp_type=export.cpp_type, export_name=export.name, + save=symbol + '_save', load=symbol + '_load')) + + api = ['// Generated by gtwrap. Do not edit.', '#pragma once', + '#include '] + api += [f'#include <{h}>' for h in headers] + api += [f'#if defined(_WIN32) && defined({building})', + f'#define {macro} __declspec(dllexport)', + '#elif defined(_WIN32)', f'#define {macro} __declspec(dllimport)', + '#else', f'#define {macro} __attribute__((visibility("default")))', + '#endif', ''] + keys = ['// Private registration declarations.', '#pragma once', + '#include ', f'#include "{api_name}"'] + source = ['// Generated by gtwrap. Do not edit.', + '// Only the typed bridge is part of this library\'s public ABI.', + '#if defined(_WIN32)', '#include ', + '// Keep Boost singleton optimizer annotations private.', + '#undef BOOST_DLLEXPORT', '#define BOOST_DLLEXPORT', + '// codecvt_null unconditionally marks even consumer definitions for export.', + '// Scope the correction to that header; preserve domain-library annotations.', + '#pragma push_macro("BOOST_SYMBOL_EXPORT")', + '#undef BOOST_SYMBOL_EXPORT', + '#if defined(BOOST_SERIALIZATION_DYN_LINK) || defined(BOOST_ALL_DYN_LINK)', + '#define BOOST_SYMBOL_EXPORT BOOST_SYMBOL_IMPORT', '#else', + '#define BOOST_SYMBOL_EXPORT', '#endif', + '#include ', + '#pragma pop_macro("BOOST_SYMBOL_EXPORT")', '#endif', + f'#define {building}', + '#include ', + '#include ', + '#include ', + f'#include "{keys_name}"', '#include ', + '#include ', ''] + symbols = [] + for entry in entries: + cpp_type, export = entry['cpp_type'], entry['export_name'] + save, load = entry['save'], entry['load'] + symbols += [save, load] + api += [f'extern "C" {macro} void {save}(const {cpp_type}* value, std::string* output);', + f'extern "C" {macro} void {load}(const std::string* input, {cpp_type}* value);'] + if export != cpp_type: + keys.append(f'typedef {cpp_type} {export};') + keys.append(f'BOOST_CLASS_EXPORT_KEY({export})') + source.append(f'BOOST_CLASS_EXPORT_IMPLEMENT({export})') + for entry in entries: + cpp_type, save, load = entry['cpp_type'], entry['save'], entry['load'] + source.append(f''' +extern "C" {macro} void {save}(const {cpp_type}* value, std::string* output) {{ + try {{ + std::ostringstream stream; + {{ boost::archive::text_oarchive archive(stream); archive << *value; }} + *output = stream.str(); + }} catch (const boost::archive::archive_exception& error) {{ + throw std::runtime_error(error.what()); + }} +}} + +extern "C" {macro} void {load}(const std::string* input, {cpp_type}* value) {{ + try {{ + std::istringstream stream(*input); + boost::archive::text_iarchive archive(stream); + archive >> *value; + }} catch (const boost::archive::archive_exception& error) {{ + throw std::runtime_error(error.what()); + }} +}}''') + return { + api_name: '\n'.join(api) + '\n', + keys_name: '\n'.join(keys) + '\n', + name + '.cpp': '\n'.join(source) + '\n', + name + '.json': json.dumps(dict(version=1, header=api_name, + entries=entries), indent=2) + '\n', + name + '.map': '{ global:\n' + ''.join(f' {s};\n' for s in symbols) + + 'local: *;\n};\n', + name + '.exports': ''.join(f'_{s}\n' for s in symbols), + } + + +def write_library(directory, files): + """Write every output, including unchanged ones, to satisfy build timestamps.""" + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + for name, content in files.items(): + (directory / name).write_text(content, encoding='utf-8') diff --git a/scripts/pybind_serialization.py b/scripts/pybind_serialization.py new file mode 100644 index 0000000..39eea14 --- /dev/null +++ b/scripts/pybind_serialization.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Generate a central Boost.Serialization library from interface declarations.""" + +import argparse +from pathlib import Path + +from gtwrap.interface_parser import InterfaceParseError +from gtwrap.serialization_library import generate_library, write_library + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--name', required=True) + parser.add_argument('--src', required=True, action='append', type=Path) + parser.add_argument('--namespace-prefix', action='append', default=[]) + parser.add_argument('--ignore', action='append', default=[]) + parser.add_argument('--include', required=True, action='append') + parser.add_argument('--out-dir', required=True, type=Path) + args = parser.parse_args() + try: + sources = [(str(p), p.read_text(encoding='utf-8')) for p in args.src] + files = generate_library(args.name, sources, args.include, + args.namespace_prefix, args.ignore) + write_library(args.out_dir, files) + except (ValueError, OSError, InterfaceParseError) as error: + parser.error(str(error)) + + +if __name__ == '__main__': + main() diff --git a/scripts/pybind_wrap.py b/scripts/pybind_wrap.py index 4de783d..43d9b05 100644 --- a/scripts/pybind_wrap.py +++ b/scripts/pybind_wrap.py @@ -70,6 +70,8 @@ def main(): type=str, default="", help="The path to the Doxygen-generated XML documentation") + arg_parser.add_argument('--serialization-manifest', + help='Manifest generated by a central serialization library') args = arg_parser.parse_args() top_module_namespaces = args.top_module_namespaces.split("::") @@ -86,6 +88,7 @@ def main(): ignore_classes=args.ignore, module_template=template_content, xml_source=args.xml_source, + serialization_manifest=args.serialization_manifest, ) try: diff --git a/tests/fixtures/serialization_library/CMakeLists.txt b/tests/fixtures/serialization_library/CMakeLists.txt new file mode 100644 index 0000000..75153ff --- /dev/null +++ b/tests/fixtures/serialization_library/CMakeLists.txt @@ -0,0 +1,72 @@ +cmake_minimum_required(VERSION 3.22) +project(serialization_consumers LANGUAGES CXX) +if(WRAP_INSTALLED) + find_package(gtwrap CONFIG REQUIRED) +else() + list(APPEND CMAKE_MODULE_PATH "${WRAP_ROOT}/cmake") + include(PybindWrap) + add_subdirectory("${WRAP_ROOT}/pybind11" pybind11) +endif() +find_package(Boost REQUIRED COMPONENTS serialization) +add_library(example_types INTERFACE) +target_include_directories(example_types INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}") +target_link_libraries(example_types INTERFACE Boost::headers) +if(WIN32) + target_compile_definitions(example_types INTERFACE BOOST_ALL_NO_LIB) + if(NOT Boost_USE_STATIC_LIBS) + target_compile_definitions(example_types INTERFACE BOOST_SERIALIZATION_DYN_LINK) + endif() +endif() +set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF) +pybind_add_serialization_library(registry + INTERFACES types.i consumer_a.i consumer_b.i + HEADERS types.h LINK_LIBRARIES example_types) +foreach(consumer consumer_a consumer_b) + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${consumer}/preamble" + "${CMAKE_CURRENT_BINARY_DIR}/${consumer}/specializations") + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${consumer}/preamble/${consumer}.h" "") + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${consumer}/specializations/${consumer}.h" "") + pybind_wrap(${consumer} "${CMAKE_CURRENT_SOURCE_DIR}/${consumer}.i" + "${consumer}.cpp" ${consumer} "example" "" + "${CMAKE_CURRENT_SOURCE_DIR}/module.tpl" "example_types" "" ON + SERIALIZATION_LIBRARY registry) +endforeach() +add_executable(legacy legacy.cpp) +target_link_libraries(legacy PRIVATE example_types Boost::serialization) +add_library(unrelated SHARED unrelated.cpp) +target_link_libraries(unrelated PRIVATE example_types Boost::serialization) +target_compile_features(unrelated PRIVATE cxx_std_17) +foreach(target registry consumer_a consumer_b unrelated) + set_target_properties(${target} PROPERTIES CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) + if(APPLE) + target_link_options(${target} PRIVATE LINKER:-dead_strip) + set_target_properties(${target} PROPERTIES INSTALL_RPATH "@loader_path") + elseif(UNIX) + target_link_options(${target} PRIVATE LINKER:--gc-sections LINKER:--as-needed) + set_target_properties(${target} PROPERTIES INSTALL_RPATH "$ORIGIN") + endif() +endforeach() +install(TARGETS registry consumer_a consumer_b unrelated legacy + LIBRARY DESTINATION runtime RUNTIME DESTINATION runtime) +# All independent executables/modules use the same selected Boost runtime. +# Copy shared Boost and its imported component dependencies beside them. +# Some distributions link serialization to thread/atomic, others do not. +if(NOT Boost_USE_STATIC_LIBS) + get_property(imported_targets DIRECTORY PROPERTY IMPORTED_TARGETS) + foreach(dependency IN LISTS imported_targets) + if(dependency MATCHES "^Boost::") + get_target_property(dependency_type ${dependency} TYPE) + if(dependency_type STREQUAL "SHARED_LIBRARY") + install(IMPORTED_RUNTIME_ARTIFACTS ${dependency} + LIBRARY DESTINATION runtime RUNTIME DESTINATION runtime) + endif() + endif() + endforeach() + if(UNIX) + set_target_properties(legacy PROPERTIES INSTALL_RPATH "$ORIGIN") + if(APPLE) + set_target_properties(legacy PROPERTIES INSTALL_RPATH "@loader_path") + endif() + endif() +endif() diff --git a/tests/fixtures/serialization_library/check.py b/tests/fixtures/serialization_library/check.py new file mode 100644 index 0000000..98ca550 --- /dev/null +++ b/tests/fixtures/serialization_library/check.py @@ -0,0 +1,51 @@ +"""Exercise installed modules in a fresh process with no search-path overrides.""" +import ctypes +import importlib +from pathlib import Path +import pickle +import sys + +# This script lives outside the installed prefix; emulate an installed package +# on sys.path, without adding any native library search-path override. +sys.path.insert(0, str(Path.cwd())) + +if sys.argv[1] == 'legacy': + from consumer_a import HolderA + archive = Path(sys.argv[2]) + value = HolderA() + value.deserialize(archive.read_text()) + assert value.get() == 42 + assert value.serialize() == archive.read_text() + archive.write_text(value.serialize()) +else: + unrelated = None + if sys.argv[2] == 'static': + name = ('unrelated.dll' if sys.platform == 'win32' else + 'libunrelated.dylib' if sys.platform == 'darwin' else 'libunrelated.so') + unrelated = ctypes.CDLL(str(Path.cwd() / name), mode=ctypes.RTLD_GLOBAL) + assert unrelated.unrelated_roundtrip() == 0 + classes = [] + for name in sys.argv[1].split(','): + module = importlib.import_module(name) + classes.append(getattr(module, 'HolderA' if name == 'consumer_a' else 'HolderB')) + assert classes[-1](1).get() == 42 + # Exercise both consumers only after all requested libraries have loaded. + for cls in classes: + for kind, expected in ((1, 42), (2, 7), (3, 3)): + value = cls(kind) + serialized = value.serialize() + assert ('example::' in serialized or 'examplePairintdouble' in serialized) + assert 'unrelated::' not in serialized + assert pickle.loads(pickle.dumps(value)).get() == expected + for other in classes: + restored = other() + restored.deserialize(serialized) + assert restored.get() == expected + try: + cls().deserialize('not an archive') + except RuntimeError: + pass + else: + raise AssertionError('invalid archives must raise RuntimeError') + if unrelated: + assert unrelated.unrelated_roundtrip() == 0 diff --git a/tests/fixtures/serialization_library/consumer_a.i b/tests/fixtures/serialization_library/consumer_a.i new file mode 100644 index 0000000..7642d6d --- /dev/null +++ b/tests/fixtures/serialization_library/consumer_a.i @@ -0,0 +1,9 @@ +namespace example { +#include +class HolderA { + HolderA(); + HolderA(int kind); + int get() const; + void serializable() const; +}; +} diff --git a/tests/fixtures/serialization_library/consumer_b.i b/tests/fixtures/serialization_library/consumer_b.i new file mode 100644 index 0000000..f59f54e --- /dev/null +++ b/tests/fixtures/serialization_library/consumer_b.i @@ -0,0 +1,9 @@ +namespace example { +#include +class HolderB { + HolderB(); + HolderB(int kind); + int get() const; + void serializable() const; +}; +} diff --git a/tests/fixtures/serialization_library/legacy.cpp b/tests/fixtures/serialization_library/legacy.cpp new file mode 100644 index 0000000..58a476a --- /dev/null +++ b/tests/fixtures/serialization_library/legacy.cpp @@ -0,0 +1,27 @@ +#include +#include +#include +#include "types.h" +#include +#include + +BOOST_CLASS_EXPORT(example::First) +BOOST_CLASS_EXPORT(example::Second) +typedef example::Pair examplePairintdouble; +BOOST_CLASS_EXPORT(examplePairintdouble) +BOOST_CLASS_EXPORT(example::HolderA) + +int main(int argc, char** argv) { + if (argc != 3) return 1; + example::HolderA value(1); + if (std::string(argv[1]) == "save") { + std::ofstream stream(argv[2]); + boost::archive::text_oarchive archive(stream); + archive << value; + } else { + std::ifstream stream(argv[2]); + boost::archive::text_iarchive archive(stream); + archive >> value; + if (value.get() != 42) return 2; + } +} diff --git a/tests/fixtures/serialization_library/module.tpl b/tests/fixtures/serialization_library/module.tpl new file mode 100644 index 0000000..c03e2be --- /dev/null +++ b/tests/fixtures/serialization_library/module.tpl @@ -0,0 +1,14 @@ +#include +{includes} +{boost_class_export} +namespace py = pybind11; +{submodules} +{declaration_module_def} {{ +{wrapped_declarations} +}} +{binding_module_def} {{ +{wrapped_bindings} +}} +{module_def} {{ +{module_init} +}} diff --git a/tests/fixtures/serialization_library/types.h b/tests/fixtures/serialization_library/types.h new file mode 100644 index 0000000..5e33511 --- /dev/null +++ b/tests/fixtures/serialization_library/types.h @@ -0,0 +1,49 @@ +#pragma once +#include +#include +#include + +namespace example { +struct Base { + virtual ~Base() = default; + virtual int get() const { return 0; } + template void serialize(Archive&, unsigned int) {} +}; +struct First : Base { + int value = 42; + int get() const override { return value; } + template void serialize(Archive& archive, unsigned int) { + archive & boost::serialization::base_object(*this) & value; + } +}; +struct Second : Base { + int value = 7; + int get() const override { return value; } + template void serialize(Archive& archive, unsigned int) { + archive & boost::serialization::base_object(*this) & value; + } +}; +template struct Pair : Base { + A first = 3; + B second = 1.5; + int get() const override { return static_cast(first); } + template void serialize(Archive& archive, unsigned int) { + archive & boost::serialization::base_object(*this) & first & second; + } +}; +template struct Holder { + std::shared_ptr value; + Holder() = default; + explicit Holder(int kind) { + if (kind == 1) value = std::make_shared(); + else if (kind == 3) value = std::make_shared>(); + else value = std::make_shared(); + } + int get() const { return value ? value->get() : 0; } + template void serialize(Archive& archive, unsigned int) { + archive & value; + } +}; +using HolderA = Holder<1>; +using HolderB = Holder<2>; +} diff --git a/tests/fixtures/serialization_library/types.i b/tests/fixtures/serialization_library/types.i new file mode 100644 index 0000000..caed386 --- /dev/null +++ b/tests/fixtures/serialization_library/types.i @@ -0,0 +1,7 @@ +namespace example { +virtual class Base {}; +virtual class First : example::Base { void serializable() const; }; +virtual class Second : example::Base { void serializable() const; }; +template virtual class Pair : example::Base {}; +@serializable typedef example::Pair PairID; +} diff --git a/tests/fixtures/serialization_library/unrelated.cpp b/tests/fixtures/serialization_library/unrelated.cpp new file mode 100644 index 0000000..b3e1311 --- /dev/null +++ b/tests/fixtures/serialization_library/unrelated.cpp @@ -0,0 +1,22 @@ +// A separate static Boost runtime must not steal the managed registrations. +#include +#include +#include +#include "types.h" +#include +BOOST_CLASS_EXPORT_GUID(example::First, "unrelated::First") +BOOST_CLASS_EXPORT_GUID(example::Second, "unrelated::Second") +#if defined(_WIN32) +#define TEST_EXPORT __declspec(dllexport) +#else +#define TEST_EXPORT __attribute__((visibility("default"))) +#endif +extern "C" TEST_EXPORT int unrelated_roundtrip() { + example::HolderA value(1); + std::stringstream stream; + { boost::archive::text_oarchive archive(stream); archive << value; } + if (stream.str().find("unrelated::First") == std::string::npos) return 1; + value.value.reset(); + { boost::archive::text_iarchive archive(stream); archive >> value; } + return value.get() == 42 ? 0 : 2; +} diff --git a/tests/fixtures/serialization_library/windows_static_boost_exports.txt b/tests/fixtures/serialization_library/windows_static_boost_exports.txt new file mode 100644 index 0000000..64c5380 --- /dev/null +++ b/tests/fixtures/serialization_library/windows_static_boost_exports.txt @@ -0,0 +1,34 @@ +# Exact compiler-forced exports observed with the pinned Boost 1.88 x64 MSVC +# static libraries. This exception applies ONLY to Windows + static Boost. +# These are implementation details, not supported consumer entry points. +# No namespace wildcards: Boost/toolchain changes require explicit review. +# Groups: codecvt_null; singleton_module; singleton for text archive +# serializer maps, extended type-info registries, and void-cast registrations. +??0?$codecvt_null@_W@archive@boost@@QEAA@_K@Z +??1?$codecvt_null@_W@archive@boost@@UEAA@XZ +??_F?$codecvt_null@_W@archive@boost@@QEAAXXZ +?do_always_noconv@?$codecvt_null@_W@archive@boost@@EEBA_NXZ +?do_encoding@?$codecvt_null@_W@archive@boost@@EEBAHXZ +?do_in@?$codecvt_null@_W@archive@boost@@EEBAHAEAU_Mbstatet@@PEBD1AEAPEBDPEA_W3AEAPEA_W@Z +?do_max_length@?$codecvt_null@_W@archive@boost@@EEBAHXZ +?do_out@?$codecvt_null@_W@archive@boost@@EEBAHAEAU_Mbstatet@@PEB_W1AEAPEB_WPEAD3AEAPEAD@Z +?get_const_instance@?$singleton@V?$map@Vtext_iarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SAAEBV?$map@Vtext_iarchive@archive@boost@@@extra_detail@detail@archive@3@XZ +?get_const_instance@?$singleton@V?$map@Vtext_oarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SAAEBV?$map@Vtext_oarchive@archive@boost@@@extra_detail@detail@archive@3@XZ +?get_const_instance@?$singleton@V?$multiset@PEBVextended_type_info@serialization@boost@@Ukey_compare@detail@23@V?$allocator@PEBVextended_type_info@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEBV?$multiset@PEBVextended_type_info@serialization@boost@@Ukey_compare@detail@23@V?$allocator@PEBVextended_type_info@serialization@boost@@@std@@@std@@XZ +?get_const_instance@?$singleton@V?$multiset@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@Utype_compare@234@V?$allocator@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEBV?$multiset@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@Utype_compare@234@V?$allocator@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@@std@@@std@@XZ +?get_const_instance@?$singleton@V?$set@PEBVvoid_caster@void_cast_detail@serialization@boost@@Uvoid_caster_compare@234@V?$allocator@PEBVvoid_caster@void_cast_detail@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEBV?$set@PEBVvoid_caster@void_cast_detail@serialization@boost@@Uvoid_caster_compare@234@V?$allocator@PEBVvoid_caster@void_cast_detail@serialization@boost@@@std@@@std@@XZ +?get_lock@singleton_module@serialization@boost@@AEAAAEA_NXZ +?get_mutable_instance@?$singleton@V?$map@Vtext_iarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SAAEAV?$map@Vtext_iarchive@archive@boost@@@extra_detail@detail@archive@3@XZ +?get_mutable_instance@?$singleton@V?$map@Vtext_oarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SAAEAV?$map@Vtext_oarchive@archive@boost@@@extra_detail@detail@archive@3@XZ +?get_mutable_instance@?$singleton@V?$multiset@PEBVextended_type_info@serialization@boost@@Ukey_compare@detail@23@V?$allocator@PEBVextended_type_info@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEAV?$multiset@PEBVextended_type_info@serialization@boost@@Ukey_compare@detail@23@V?$allocator@PEBVextended_type_info@serialization@boost@@@std@@@std@@XZ +?get_mutable_instance@?$singleton@V?$multiset@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@Utype_compare@234@V?$allocator@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEAV?$multiset@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@Utype_compare@234@V?$allocator@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@@std@@@std@@XZ +?get_mutable_instance@?$singleton@V?$set@PEBVvoid_caster@void_cast_detail@serialization@boost@@Uvoid_caster_compare@234@V?$allocator@PEBVvoid_caster@void_cast_detail@serialization@boost@@@std@@@std@@@serialization@boost@@SAAEAV?$set@PEBVvoid_caster@void_cast_detail@serialization@boost@@Uvoid_caster_compare@234@V?$allocator@PEBVvoid_caster@void_cast_detail@serialization@boost@@@std@@@std@@XZ +?is_destroyed@?$singleton@V?$map@Vtext_iarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SA_NXZ +?is_destroyed@?$singleton@V?$map@Vtext_oarchive@archive@boost@@@extra_detail@detail@archive@boost@@@serialization@boost@@SA_NXZ +?is_destroyed@?$singleton@V?$multiset@PEBVextended_type_info@serialization@boost@@Ukey_compare@detail@23@V?$allocator@PEBVextended_type_info@serialization@boost@@@std@@@std@@@serialization@boost@@SA_NXZ +?is_destroyed@?$singleton@V?$multiset@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@Utype_compare@234@V?$allocator@PEBVextended_type_info_typeid_0@typeid_system@serialization@boost@@@std@@@std@@@serialization@boost@@SA_NXZ +?is_destroyed@?$singleton@V?$set@PEBVvoid_caster@void_cast_detail@serialization@boost@@Uvoid_caster_compare@234@V?$allocator@PEBVvoid_caster@void_cast_detail@serialization@boost@@@std@@@std@@@serialization@boost@@SA_NXZ +?is_locked@singleton_module@serialization@boost@@QEAA_NXZ +?lock@?1??get_lock@singleton_module@serialization@boost@@AEAAAEA_NXZ@4_NA +?lock@singleton_module@serialization@boost@@QEAAXXZ +?unlock@singleton_module@serialization@boost@@QEAAXXZ diff --git a/tests/test_serialization_bindings.py b/tests/test_serialization_bindings.py new file mode 100644 index 0000000..2a4aded --- /dev/null +++ b/tests/test_serialization_bindings.py @@ -0,0 +1,44 @@ +"""Managed methods call the archive owner; legacy generation remains opt-in.""" +import json +from pathlib import Path +import tempfile +import unittest + +from gtwrap.pybind_wrapper import PybindWrapper +from gtwrap.serialization_library import generate_library, write_library + + +class TestManagedBindings(unittest.TestCase): + def test_managed_and_unselected_types(self): + source = '''namespace example { +class Managed { Managed(); void serializable() const; }; +class Local { Local(); void serialize() const; }; +}''' + template = Path(__file__).with_name('pybind_wrapper.tpl').read_text() + with tempfile.TemporaryDirectory() as directory: + files = generate_library('registry', [('types.i', source)], ['types.h'], + ignore_classes=['example::Local']) + write_library(directory, files) + manifest = Path(directory) / 'registry.json' + wrapper = PybindWrapper('example', top_module_namespaces=[''], + use_boost_serialization=True, + module_template=template, + serialization_manifest=manifest) + output = wrapper.wrap_file(source, module_name='example', submodules=[]) + entry = json.loads(files['registry.json'])['entries'][0] + self.assertIn(entry['save'] + '(self, &result)', output) + self.assertIn(entry['load'] + '(&input, &value)', output) + self.assertIn('BOOST_CLASS_EXPORT(example::Local)', output) + self.assertNotIn('BOOST_CLASS_EXPORT(example::Managed)', output) + self.assertIn('#include "registry.h"', output) + with self.assertRaisesRegex(ValueError, 'requires serialization enabled'): + PybindWrapper('example', serialization_manifest=manifest) + + def test_disabled_has_no_runtime_dependency(self): + source = 'class Plain { Plain(); void serialize() const; };' + template = Path(__file__).with_name('pybind_wrapper.tpl').read_text() + output = PybindWrapper('plain', top_module_namespaces=[''], + module_template=template).wrap_file( + source, module_name='plain', submodules=[]) + self.assertNotIn('.def("serialize"', output) + self.assertNotIn('boost/serialization', output) diff --git a/tests/test_serialization_discovery.py b/tests/test_serialization_discovery.py new file mode 100644 index 0000000..395e6b5 --- /dev/null +++ b/tests/test_serialization_discovery.py @@ -0,0 +1,35 @@ +"""Selection uses instantiated declarations, without generating bindings.""" +import unittest + +from gtwrap.serialization import discover + + +class TestDiscovery(unittest.TestCase): + SOURCE = '''namespace example { +class First { void serializable() const; }; +class Second { void serialize() const; }; +class Plain {}; +}''' + + def test_sources_filters_and_exclusions(self): + sources = [('one.i', self.SOURCE), ('two.i', self.SOURCE)] + self.assertEqual([e.cpp_type for e in discover(sources)], + ['example::First', 'example::Second']) + self.assertEqual([e.cpp_type for e in discover( + sources, ['example::'], ['example::First'])], ['example::Second']) + with self.assertRaisesRegex(ValueError, 'one.i.*no serializable'): + discover(sources, ['other::']) + with self.assertRaisesRegex(ValueError, 'end in ::'): + discover(sources, ['example']) + + def test_template_alias_and_collision(self): + source = '''namespace example { +template class Pair {}; +@serializable typedef example::Pair PairID; +}''' + entry, = discover([('alias.i', source)]) + self.assertEqual(entry.cpp_type, 'example::Pair') + self.assertEqual(entry.name, 'examplePairintdouble') + collision = 'class examplePairintdouble { void serialize() const; };' + with self.assertRaisesRegex(ValueError, 'collision.i.*alias.i'): + discover([('alias.i', source), ('collision.i', collision)]) diff --git a/tests/test_serialization_library.py b/tests/test_serialization_library.py new file mode 100644 index 0000000..077c18d --- /dev/null +++ b/tests/test_serialization_library.py @@ -0,0 +1,25 @@ +"""The generated ABI keeps archive implementation details in one owner.""" +import json +import unittest + +from gtwrap.serialization_library import generate_library + + +class TestLibraryGeneration(unittest.TestCase): + def test_private_exports_and_stable_abi(self): + source = '''namespace example { +template class Pair {}; +@serializable typedef example::Pair PairID; +}''' + files = generate_library('registry', [('types.i', source)], ['types.h']) + manifest = json.loads(files['registry.json']) + entry, = manifest['entries'] + self.assertEqual(entry['export_name'], 'examplePairintdouble') + self.assertNotIn('BOOST_CLASS_EXPORT', files['registry.h']) + self.assertNotIn('boost/archive', files['registry.h']) + self.assertIn('BOOST_CLASS_EXPORT_KEY(examplePairintdouble)', files['registry_keys.h']) + self.assertIn('BOOST_CLASS_EXPORT_IMPLEMENT(examplePairintdouble)', files['registry.cpp']) + self.assertIn('throw std::runtime_error(error.what())', files['registry.cpp']) + self.assertIn(entry['save'] + ';', files['registry.map']) + self.assertIn('_' + entry['load'] + '\n', files['registry.exports']) + self.assertEqual(files, generate_library('registry', [('types.i', source)], ['types.h'])) diff --git a/tests/test_serialization_runtime.py b/tests/test_serialization_runtime.py new file mode 100644 index 0000000..fdbc4c6 --- /dev/null +++ b/tests/test_serialization_runtime.py @@ -0,0 +1,173 @@ +"""Required CI integration suite; opt in locally with GTWRAP_TEST_SERIALIZATION=1.""" +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import time +import unittest + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / 'tests/fixtures/serialization_library' + + +def check_public_exports(actual, expected, platform, linkage): + """Require every bridge and reject exports outside the reviewed exception.""" + allowed = set() + if platform == 'win32' and linkage == 'static': + # Precompiled MSVC objects carry dllexport directives that our DLL's + # source-level visibility settings cannot suppress. Match exact names, + # never a Boost namespace pattern or exports discovered during this run. + allowed = {line for line in + (FIXTURE / 'windows_static_boost_exports.txt').read_text().splitlines() + if line and not line.startswith('#')} + missing = expected - actual + unexpected = actual - expected - allowed + if missing or unexpected: + raise AssertionError(f'Missing bridge exports: {sorted(missing)}\n' + f'Unexpected exports: {sorted(unexpected)}') + + +class TestExportPolicy(unittest.TestCase): + def test_exception_is_limited_to_windows_static(self): + bridge = {'gtwrap_test_save', 'gtwrap_test_load'} + actual = bridge | {'?lock@singleton_module@serialization@boost@@QEAAXXZ'} + check_public_exports(actual, bridge, 'win32', 'static') + for platform, linkage in (('win32', 'shared'), ('linux', 'static'), + ('darwin', 'static')): + with self.subTest(platform=platform, linkage=linkage): + with self.assertRaisesRegex(AssertionError, 'Unexpected exports: \\[.+'): + check_public_exports(actual, bridge, platform, linkage) + + def test_unknown_boost_helpers_are_rejected(self): + with self.assertRaisesRegex(AssertionError, 'Unexpected exports: \\[.+'): + check_public_exports({'?unknown_helper@serialization@boost@@'}, + set(), 'win32', 'static') + + def test_exception_cannot_hide_missing_bridges(self): + with self.assertRaisesRegex(AssertionError, 'Missing bridge exports: \\[.+'): + check_public_exports(set(), {'gtwrap_test_load'}, 'win32', 'static') + + +def run(args, **kwargs): + result = subprocess.run([str(a) for a in args], capture_output=True, + text=True, **kwargs) + if result.returncode: + raise AssertionError(f'{args}\n{result.stdout}\n{result.stderr}') + return result.stdout + + +@unittest.skipUnless(os.environ.get('GTWRAP_TEST_SERIALIZATION') == '1', + 'enable the required serialization integration suite explicitly') +class TestSerializationRuntime(unittest.TestCase): + def test_runtime_and_installation(self): + # CI selects each linkage in its own job; local runs can exercise both. + linkages = os.environ.get('GTWRAP_TEST_BOOST_LINKAGE', 'static,shared').split(',') + for linkage in linkages: + with self.subTest(linkage=linkage), tempfile.TemporaryDirectory() as temporary: + self.exercise(Path(temporary), linkage) + + def exercise(self, directory, linkage): + self.assertIn(linkage, ('static', 'shared')) + source = directory / 'source' + shutil.copytree(FIXTURE, source) + wrap = directory / 'wrap' + wrap.mkdir() + for name in ('gtwrap', 'cmake', 'scripts', 'templates', 'pybind11'): + shutil.copytree(ROOT / name, wrap / name, + ignore=shutil.ignore_patterns('__pycache__', '.git')) + for name in ('CMakeLists.txt', 'matlab.h'): + shutil.copyfile(ROOT / name, wrap / name) + env = dict(os.environ) + env['PATH'] = str(Path(sys.executable).parent) + os.pathsep + env['PATH'] + cmake_args = ['-DCMAKE_BUILD_TYPE=Release', '-DPYBIND11_FINDPYTHON=ON', + f'-DPython_EXECUTABLE={Path(sys.executable).as_posix()}', + f'-DWRAP_PYTHON_VERSION={sys.version_info.major}.{sys.version_info.minor}', + f'-DBoost_USE_STATIC_LIBS={"ON" if linkage == "static" else "OFF"}'] + # Accept a caller-provided Boost/toolchain installation, never download + # dependencies or silently change linkage from inside a test. + for variable in ('CMAKE_PREFIX_PATH', 'CMAKE_TOOLCHAIN_FILE', 'VCPKG_TARGET_TRIPLET'): + if os.environ.get(variable): + value = os.environ[variable].replace('\\', '/') + cmake_args.append(f'-D{variable}={value}') + + for installed in (False, True): + label = 'installed' if installed else 'source' + build = directory / (label + '-build') + prefix = directory / (label + '-prefix') + arguments = list(cmake_args) + if installed: + wrap_build = directory / 'wrap-build' + wrap_prefix = directory / 'wrap-prefix' + run(['cmake', '-S', wrap, '-B', wrap_build, + f'-DCMAKE_INSTALL_PREFIX={wrap_prefix.as_posix()}', *cmake_args], env=env) + run(['cmake', '--install', wrap_build, '--config', 'Release'], env=env) + config = next(wrap_prefix.rglob('gtwrapConfig.cmake')) + arguments += ['-DWRAP_INSTALLED=ON', + f'-Dgtwrap_DIR={config.parent.as_posix()}'] + implementation = wrap_prefix / 'lib/gtwrap/gtwrap/serialization.py' + else: + arguments += [f'-DWRAP_ROOT={wrap.as_posix()}'] + implementation = wrap / 'gtwrap/serialization.py' + run(['cmake', '-S', source, '-B', build, + f'-DCMAKE_INSTALL_PREFIX={prefix.as_posix()}', *arguments], env=env) + run(['cmake', '--build', build, '--config', 'Release', '-j', '6'], env=env) + generated = build / 'gtwrap_serialization/registry' + manifest = json.loads((generated / 'registry.json').read_text()) + self.assertEqual(len(manifest['entries']), 5) + for consumer in ('consumer_a', 'consumer_b'): + content = (build / f'{consumer}.cpp').read_text() + self.assertNotIn('BOOST_CLASS_EXPORT(', content) + self.assertNotIn('boost::archive', content) + self.assertNotIn('gtsam::', content) + + # A no-op build is stable; edits to either source of truth regenerate + # the manifest and all consumers before they compile. + generated_source = generated / 'registry.cpp' + before = generated_source.stat().st_mtime_ns + run(['cmake', '--build', build, '--config', 'Release', '-j', '6'], env=env) + self.assertEqual(before, generated_source.stat().st_mtime_ns) + for dependency in (source / 'types.i', implementation): + time.sleep(1.05) + dependency.touch() + run(['cmake', '--build', build, '--config', 'Release', '-j', '6'], env=env) + after = generated_source.stat().st_mtime_ns + self.assertGreater(after, before) + before = after + + run(['cmake', '--install', build, '--config', 'Release'], env=env) + runtime = prefix / 'runtime' + # Ensure installed tests cannot accidentally find build products. + build.rename(directory / (label + '-hidden-build')) + clean_env = {k: v for k, v in env.items() + if k not in ('PYTHONPATH', 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH')} + for order in ('consumer_a', 'consumer_b', + 'consumer_a,consumer_b', 'consumer_b,consumer_a'): + run([sys.executable, source / 'check.py', order, linkage], + cwd=runtime, env=clean_env) + + # Compare with the historical full-export owner in a fresh process. + legacy = runtime / ('legacy.exe' if os.name == 'nt' else 'legacy') + archive = runtime / 'archive.txt' + run([legacy, 'save', archive], env=clean_env) + run([sys.executable, source / 'check.py', 'legacy', archive], + cwd=runtime, env=clean_env) + run([legacy, 'load', archive], env=clean_env) + symbols = {entry[operation] for entry in manifest['entries'] + for operation in ('save', 'load')} + if sys.platform == 'darwin': + exported = run(['nm', '-gU', runtime / 'libregistry.dylib']) + actual = {line.split()[-1].removeprefix('_') for line in exported.splitlines() if line.strip()} + elif os.name == 'nt': + exported = run(['dumpbin', '/EXPORTS', runtime / 'registry.dll']) + actual = set(re.findall(r'^\s+\d+\s+[0-9A-F]+\s+[0-9A-F]+\s+(\S+)', exported, re.M)) + else: + exported = run(['nm', '-D', '--defined-only', runtime / 'libregistry.so']) + actual = {line.split()[-1] for line in exported.splitlines() if line.strip()} + # Report export-policy failures without skipping installed-package + # runtime validation. Both consumption modes must still be exercised. + with self.subTest(consumption=label, check='public exports'): + check_public_exports(actual, symbols, sys.platform, linkage)