From 44cf245cee4e9d0c97062f058a2b026ba57fa809 Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 18 Sep 2026 21:25:35 -0400 Subject: [PATCH 01/21] Establish `BackendFromEnv` machinery --- bind/backend.go | 65 ++++++++++++++++++++++++++++++++++++++++++++ bind/backend_test.go | 46 +++++++++++++++++++++++++++++++ gen.go | 3 ++ 3 files changed, 114 insertions(+) create mode 100644 bind/backend.go create mode 100644 bind/backend_test.go diff --git a/bind/backend.go b/bind/backend.go new file mode 100644 index 00000000..cb2cc4d0 --- /dev/null +++ b/bind/backend.go @@ -0,0 +1,65 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + "fmt" + "os" + "strings" +) + +// BackendEnvVar is the environment variable that selects which tool is used +// to bind the generated cgo shim to CPython. +const BackendEnvVar = "GOPY_BACKEND" + +// Backend names a CPython binding tool. +type Backend string + +const ( + BackendPyBindGen Backend = "pybindgen" // default + BackendCFFI Backend = "cffi" + BackendPyBind11 Backend = "pybind11" + BackendNanobind Backend = "nanobind" + BackendCAPI Backend = "capi" + BackendCGO Backend = "cgo" +) + +// backends lists every known backend and whether gopy can generate it yet. +var backends = []struct { + name Backend + implemented bool +}{ + {BackendPyBindGen, true}, + {BackendCFFI, false}, + {BackendPyBind11, false}, + {BackendNanobind, false}, + {BackendCAPI, false}, + {BackendCGO, false}, +} + +// BackendFromEnv returns the backend selected by GOPY_BACKEND. +// An unset or empty variable selects pybindgen. +func BackendFromEnv() (Backend, error) { + return parseBackend(os.Getenv(BackendEnvVar)) +} + +func parseBackend(v string) (Backend, error) { + v = strings.ToLower(strings.TrimSpace(v)) + if v == "" { + return BackendPyBindGen, nil + } + names := make([]string, len(backends)) + for i, b := range backends { + names[i] = string(b.name) + if string(b.name) != v { + continue + } + if !b.implemented { + return "", fmt.Errorf("gopy: %s=%q is not implemented yet", BackendEnvVar, v) + } + return b.name, nil + } + return "", fmt.Errorf("gopy: unknown %s=%q (valid values: %s)", BackendEnvVar, v, strings.Join(names, ", ")) +} diff --git a/bind/backend_test.go b/bind/backend_test.go new file mode 100644 index 00000000..5c734356 --- /dev/null +++ b/bind/backend_test.go @@ -0,0 +1,46 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + "strings" + "testing" +) + +func TestParseBackend(t *testing.T) { + for _, tc := range []struct { + in string + want Backend + errPart string + }{ + {in: "", want: BackendPyBindGen}, + {in: "pybindgen", want: BackendPyBindGen}, + {in: " PyBindGen ", want: BackendPyBindGen}, + {in: "cffi", errPart: "not implemented yet"}, + {in: "bogus", errPart: "unknown GOPY_BACKEND"}, + } { + got, err := parseBackend(tc.in) + if tc.errPart != "" { + if err == nil || !strings.Contains(err.Error(), tc.errPart) { + t.Errorf("parseBackend(%q): got err=%v, want error containing %q", tc.in, err, tc.errPart) + } + continue + } + if err != nil || got != tc.want { + t.Errorf("parseBackend(%q) = %q, %v; want %q", tc.in, got, err, tc.want) + } + } +} + +func TestBackendFromEnv(t *testing.T) { + t.Setenv(BackendEnvVar, "") + if got, err := BackendFromEnv(); err != nil || got != BackendPyBindGen { + t.Errorf("unset: got %q, %v", got, err) + } + t.Setenv(BackendEnvVar, "bogus") + if _, err := BackendFromEnv(); err == nil { + t.Error("bogus value: want error") + } +} diff --git a/gen.go b/gen.go index 549ee2ad..c43ffd62 100644 --- a/gen.go +++ b/gen.go @@ -63,6 +63,9 @@ func genOutDir(odir string) (string, error) { // mode = gen, build, pkg, exe func genPkg(mode bind.BuildMode, cfg *BuildCfg) error { var err error + if _, err = bind.BackendFromEnv(); err != nil { + return err + } cfg.OutputDir, err = genOutDir(cfg.OutputDir) if err != nil { return err From 9cd22337b779f30f12778fec50a238bfb8c9e77b Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 18 Sep 2026 21:47:04 -0400 Subject: [PATCH 02/21] Use experimental cffi backend by GOPY_BACKEND=cffi --- bind/backend.go | 2 +- bind/backend_test.go | 3 +- bind/bind.go | 2 + bind/cffi.go | 171 +++++++++++++++++++++++++++++++++++++++++++ bind/cffi_build.py | 165 +++++++++++++++++++++++++++++++++++++++++ bind/gen.go | 28 ++++++- bind/gen_func.go | 19 +++-- bind/gen_map.go | 6 +- bind/gen_slice.go | 51 +++++++------ cmd_build.go | 31 ++++++++ gen.go | 5 +- 11 files changed, 446 insertions(+), 37 deletions(-) create mode 100644 bind/cffi.go create mode 100644 bind/cffi_build.py diff --git a/bind/backend.go b/bind/backend.go index cb2cc4d0..e19a6d9c 100644 --- a/bind/backend.go +++ b/bind/backend.go @@ -32,7 +32,7 @@ var backends = []struct { implemented bool }{ {BackendPyBindGen, true}, - {BackendCFFI, false}, + {BackendCFFI, true}, {BackendPyBind11, false}, {BackendNanobind, false}, {BackendCAPI, false}, diff --git a/bind/backend_test.go b/bind/backend_test.go index 5c734356..cbbe40ac 100644 --- a/bind/backend_test.go +++ b/bind/backend_test.go @@ -18,7 +18,8 @@ func TestParseBackend(t *testing.T) { {in: "", want: BackendPyBindGen}, {in: "pybindgen", want: BackendPyBindGen}, {in: " PyBindGen ", want: BackendPyBindGen}, - {in: "cffi", errPart: "not implemented yet"}, + {in: "cffi", want: BackendCFFI}, + {in: "pybind11", errPart: "not implemented yet"}, {in: "bogus", errPart: "unknown GOPY_BACKEND"}, } { got, err := parseBackend(tc.in) diff --git a/bind/bind.go b/bind/bind.go index 93e78af8..f60acf66 100644 --- a/bind/bind.go +++ b/bind/bind.go @@ -31,6 +31,8 @@ type BindCfg struct { // gopy version string embedded in this binary, stamped into generated // file headers so output can be traced back to the release that produced it Version string + // tool used to bind the cgo shim to CPython, see BackendFromEnv + Backend Backend } // ErrorList is a list of errors diff --git a/bind/cffi.go b/bind/cffi.go new file mode 100644 index 00000000..04dfabcd --- /dev/null +++ b/bind/cffi.go @@ -0,0 +1,171 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + _ "embed" + "strings" +) + +// The cffi backend (GOPY_BACKEND=cffi) generates the same cgo shim as the +// default backend, minus every call into the CPython C API: the shim is a +// plain C shared library that Python loads with cffi. Errors are recorded +// for the Python side to raise, instead of being set with PyErr_SetString. + +//go:embed cffi_build.py +var cffiBuildPy string + +func (g *pyGen) isCFFI() bool { + return g.cfg.Backend == BackendCFFI +} + +// cffiBuildPreamble returns the start of build.py: the cffi recorder. +func (g *pyGen) cffiBuildPreamble() string { + return strings.NewReplacer( + "@NAME@", g.cfg.Name, + "@CMD@", g.cfg.Cmd, + "@VERSION@", g.cfg.Version, + "@LIBEXT@", g.libext, + ).Replace(cffiBuildPy) +} + +// goSetError returns Go code that records an error for Python to raise. +// kind is the name of a Python builtin exception, msg a Go string expression. +func (g *pyGen) goSetError(kind, msg string) string { + return "gopySetError(\"" + kind + "\", " + msg + ")\n" +} + +// same argument positions as goPreamble: 1 = name of package, 2 = cmdstr, +// 4 = GoHandle, 5 = CGoHandle, 6 = all imports, 7 = mainstr, 10 = gopy version. +const goPreambleCFFI = `/* +cgo stubs for package %[1]s, for use with cffi. +File is generated by gopy version %[10]s. Do not edit. +%[2]s +*/ + +package main + +/* +#include +#include +#if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 202311L) +typedef uint8_t bool; +#endif +*/ +import "C" +import ( + "runtime" + "sync" + "unsafe" + "github.com/go-python/gopy/gopyh" // handler + %[6]s +) + +func main() { + %[7]s +} + +//export GoPyInit +func GoPyInit() { + %[7]s +} + +// type for the handle -- int64 for speed (can switch to string) +type GoHandle %[4]s +type CGoHandle %[5]s + +// DecRef decrements the reference count for the specified handle +// and deletes it it goes to zero. +//export DecRef +func DecRef(handle CGoHandle) { + gopyh.DecRef(gopyh.CGoHandle(handle)) +} + +// IncRef increments the reference count for the specified handle. +//export IncRef +func IncRef(handle CGoHandle) { + gopyh.IncRef(gopyh.CGoHandle(handle)) +} + +// NumHandles returns the number of handles currently in use. +//export NumHandles +func NumHandles() int { + return gopyh.NumHandles() +} + +// RequestGC runs Go's garbage collector on a dedicated goroutine, and waits. +//export RequestGC +func RequestGC() { + done := make(chan struct{}) + _gcReq <- done + <-done +} + +var _gcReq = make(chan chan struct{}) + +func init() { + go func() { + for done := range _gcReq { + runtime.GC() + close(done) + } + }() +} + +// The error of the last call, for the python side to raise. There is one +// slot for the whole process, so calls from several python threads at once +// can be given each other's errors. +var ( + gopyErrMu sync.Mutex + gopyErrMsg string + gopyHasErr bool +) + +func gopySetError(kind, msg string) { + gopyErrMu.Lock() + gopyErrMsg, gopyHasErr = kind+":"+msg, true + gopyErrMu.Unlock() +} + +// GopyTakeError returns "Kind:message" for the error recorded by the last +// call and clears it, or NULL if there is none. Free with GopyFreeString. +//export GopyTakeError +func GopyTakeError() *C.char { + gopyErrMu.Lock() + defer gopyErrMu.Unlock() + if !gopyHasErr { + return nil + } + gopyHasErr = false + return C.CString(gopyErrMsg) +} + +// GopyFreeString frees a string returned by this library. +//export GopyFreeString +func GopyFreeString(s *C.char) { + C.free(unsafe.Pointer(s)) +} + +// boolGoToPy converts a Go bool to python-compatible C.char +func boolGoToPy(b bool) C.char { + if b { + return 1 + } + return 0 +} + +// boolPyToGo converts a python-compatible C.Char to Go bool +func boolPyToGo(b C.char) bool { + return b != 0 +} + +// errorGoToPy converts a Go error to python-compatible C.CString +func errorGoToPy(e error) *C.char { + if e != nil { + return C.CString(e.Error()) + } + return C.CString("") +} +` diff --git a/bind/cffi_build.py b/bind/cffi_build.py new file mode 100644 index 00000000..d22a3950 --- /dev/null +++ b/bind/cffi_build.py @@ -0,0 +1,165 @@ +# python build stubs for package @NAME@ (cffi backend) +# File is generated by gopy version @VERSION@. Do not edit. +# @CMD@ +# +# The generated build code below is written against the pybindgen API. Here +# the same calls are only recorded, and Module.generate() then writes a cffi +# (ABI mode) module, _@NAME@.py, that loads @NAME@_go@LIBEXT@ directly. The +# exact C types come from the extern declarations in cgo's @NAME@_go.h. + +import os +import re +import sys + +import cffi + + +def retval(ctype, *a, **kw): + return ctype + + +def param(ctype, name, *a, **kw): + return (ctype, name) + + +class Module(object): + def __init__(self, name): + self.name = name + self.header = None + self.funcs = [] + + def add_include(self, inc): + self.header = inc.strip('"') + + def add_function(self, name, ret, params, *a, **kw): + self.funcs.append((name, ret, params)) + + def generate(self): + here = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(here, self.header)) as f: + typedefs, cdefs = go_decls(f.read()) + out = [MODULE_HEAD.replace("@CDEFS@", repr("\n".join(typedefs + list(cdefs.values()))))] + for name, ret, params in self.funcs: + out.append(wrapper(name, ret, params, name in cdefs)) + with open(os.path.join(here, self.name + ".py"), "w") as f: + f.write("\n".join(out)) + + +def add_checked_function(mod, name, retval, params, failure_expression="", *a, **kw): + mod.add_function(name, retval, params) + + +add_checked_string_function = add_checked_function + + +def go_decls(header): + """Returns the Go typedefs, and {function name: cdef line} for the functions cgo exports.""" + typedefs = [] + decls = {} + ffi = cffi.FFI() + for line in header.split("\n"): + if re.match(r"typedef [\w ]+ Go\w+;$", line): + try: + ffi.cdef(line) + typedefs.append(line) + except Exception: + pass # e.g. GoComplex64: not used by exports + continue + m = re.match(r"extern (.*?(\w+)\(.*\));$", line.replace("__declspec(dllexport) ", "")) + if not m or "_GoString_" in line: + continue + # cffi takes a plain char as a byte string only; the integer kinds + # (bool, int8, byte) are passed as ints, with the same C ABI. + decl = re.sub(r"\b(? Date: Fri, 18 Sep 2026 22:08:46 -0400 Subject: [PATCH 03/21] Create a separate cffi job for Github Actions --- .github/requirements-cffi.txt | 5 +++++ .github/workflows/ci.yml | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 .github/requirements-cffi.txt diff --git a/.github/requirements-cffi.txt b/.github/requirements-cffi.txt new file mode 100644 index 00000000..aff0e708 --- /dev/null +++ b/.github/requirements-cffi.txt @@ -0,0 +1,5 @@ +# Python packages for the GOPY_BACKEND=cffi job in workflows/ci.yml. +# pybindgen is deliberately absent: the cffi backend must not need it. +cffi +# used by the memory-leak checks on Windows, where the resource module is missing +psutil diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af0017d8..31cb4807 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,3 +96,45 @@ jobs: - name: Upload-Coverage if: matrix.platform == 'ubuntu-latest' uses: codecov/codecov-action@v4 + + # Builds and tests the opt-in cffi backend (GOPY_BACKEND=cffi). Runs beside + # the main matrix, on one Go version, and stops at the first failure. + cffi: + name: cffi backend (${{ matrix.platform }}, Python ${{ matrix.python-version }}) + strategy: + fail-fast: true + matrix: + platform: [ubuntu-latest, windows-latest, macos-15] + python-version: ['3.11', '3.12'] + runs-on: ${{ matrix.platform }} + env: + GOPY_BACKEND: cffi + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: .github/requirements-cffi.txt + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.25.x + cache: true + + - name: Install packages + run: | + python -m pip install -r .github/requirements-cffi.txt + go install golang.org/x/tools/cmd/goimports@v0.29.0 + + - name: Build + run: go build -v ./... + + # The skipped tests use features the cffi backend does not support yet: + # callbacks into Python, complex numbers, python bytes, and the Makefile. + - name: Test + run: go test -v -skip '^(TestBytes|TestBindFuncs|TestBindSimple|TestBuiltinSlices|TestGilString|TestGenHeaderHasVersion)$' ./... From f7da474247f94c5caaf6e75ff917bd4854635bc9 Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 18 Sep 2026 22:25:07 -0400 Subject: [PATCH 04/21] Prevent cffi from unloading Go shared library --- .github/workflows/ci.yml | 2 ++ bind/cffi_build.py | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31cb4807..2a89afa3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,6 +109,8 @@ jobs: runs-on: ${{ matrix.platform }} env: GOPY_BACKEND: cffi + # print the python stack if the process crashes, e.g. at exit + PYTHONFAULTHANDLER: 1 steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/bind/cffi_build.py b/bind/cffi_build.py index d22a3950..a1c44937 100644 --- a/bind/cffi_build.py +++ b/bind/cffi_build.py @@ -116,6 +116,7 @@ def wrapper(name, ret, params, exported): MODULE_HEAD = '''# python bindings for package @NAME@ using cffi. # File is generated by gopy version @VERSION@. Do not edit. import builtins +import ctypes import os from operator import index as _index @@ -123,7 +124,12 @@ def wrapper(name, ret, params, exported): _ffi = cffi.FFI() _ffi.cdef(@CDEFS@) -_lib = _ffi.dlopen(os.path.join(os.path.dirname(os.path.abspath(__file__)), "@NAME@_go@LIBEXT@")) +_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "@NAME@_go@LIBEXT@") +_lib = _ffi.dlopen(_path) +# Go cannot unload its runtime. cffi unloads _lib at interpreter shutdown but +# ctypes never unloads, so this second handle keeps the library loaded until +# the process exits, as for an extension module. +_pin = ctypes.CDLL(_path) def _enc(s, argn): From 2d8f4e234c583e81af983a438dc1072631888d09 Mon Sep 17 00:00:00 2001 From: b-long Date: Sat, 19 Sep 2026 08:26:21 -0400 Subject: [PATCH 05/21] Add to/from bytes & `pkg` mode, start Complex nums --- .github/workflows/ci.yml | 5 +-- bind/cffi_build.py | 28 ++++++++++++++++ bind/gen.go | 36 +++++++++++++++++++-- bind/gen_func.go | 17 ++++++++++ bind/gen_slice.go | 70 +++++++++++++++++++++++++++++++++++++--- gen.go | 7 ++-- 6 files changed, 151 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a89afa3..d849ba36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,6 +137,7 @@ jobs: run: go build -v ./... # The skipped tests use features the cffi backend does not support yet: - # callbacks into Python, complex numbers, python bytes, and the Makefile. + # Python callback arguments (TestBindFuncs), complex numbers as a plain + # function arg/return (TestBindSimple) or slice element (TestBuiltinSlices). - name: Test - run: go test -v -skip '^(TestBytes|TestBindFuncs|TestBindSimple|TestBuiltinSlices|TestGilString|TestGenHeaderHasVersion)$' ./... + run: go test -v -skip '^(TestBindFuncs|TestBindSimple|TestBuiltinSlices)$' ./... diff --git a/bind/cffi_build.py b/bind/cffi_build.py index a1c44937..cb459fda 100644 --- a/bind/cffi_build.py +++ b/bind/cffi_build.py @@ -41,6 +41,12 @@ def generate(self): out = [MODULE_HEAD.replace("@CDEFS@", repr("\n".join(typedefs + list(cdefs.values()))))] for name, ret, params in self.funcs: out.append(wrapper(name, ret, params, name in cdefs)) + # Slice_byte's converters exchange a raw pointer+length instead of a + # PyObject* (see gen_slice.go); they are recognized by name here + # rather than recorded via add_function, since their python bodies + # aren't derived from a plain C signature. + if "Slice_byte_from_bytes" in cdefs and "Slice_byte_to_bytes_ptr" in cdefs: + out.append(BYTES_FUNCS) with open(os.path.join(here, self.name + ".py"), "w") as f: f.write("\n".join(out)) @@ -161,6 +167,28 @@ def _check(): ''' +BYTES_FUNCS = ''' +def Slice_byte_from_bytes(b): + if not isinstance(b, (bytes, bytearray)): + raise TypeError("argument 1 must be bytes, not %s" % type(b).__name__) + _r = _lib.Slice_byte_from_bytes(_ffi.from_buffer(b), len(b)) + _check() + return _r + + +def Slice_byte_to_bytes(handle): + n = _lib.Slice_byte_to_bytes_len(handle) + _check() + if n == 0: + return b"" + ptr = _lib.Slice_byte_to_bytes_ptr(handle) + _check() + try: + return bytes(_ffi.buffer(ptr, n)) + finally: + _lib.Slice_byte_free_ptr(ptr) +''' + mod = Module('_@NAME@') mod.add_include('"@NAME@_go.h"') mod.add_function('GoPyInit', None, []) diff --git a/bind/gen.go b/bind/gen.go index 64687fc0..85db4741 100644 --- a/bind/gen.go +++ b/bind/gen.go @@ -469,6 +469,35 @@ build: %[9]s $(GCC) %[1]s.c %[6]s %[1]s_go$(LIBEXT) -o _%[1]s$(LIBEXT) $(CFLAGS) $(LDFLAGS) -fPIC --shared -w +` + + // same argument positions as MakefileTemplate, though only 1-5 and 10 are used: + // cffi needs none of the CPython CFLAGS/LDFLAGS that building %[1]s.c would. + MakefileTemplateCFFI = `# Makefile for python interface for package %[1]s, using cffi. +# File is generated by gopy version %[10]s. Do not edit. +# %[2]s + +GOCMD=go +GOBUILD=$(GOCMD) build -mod=mod +GOIMPORTS=goimports +PYTHON=%[4]s +LIBEXT=%[5]s + +all: gen build + +gen: + %[3]s + +build: + # goimports is needed to ensure that the imports list is valid + $(GOIMPORTS) -w %[1]s.go + # generate %[1]s_go$(LIBEXT) from %[1]s.go -- the cgo wrappers to go functions. + # unlike the default (pybindgen) backend, this is the only library gopy + # builds: cffi loads it directly, with no Python.h/libpython involved. + $(GOBUILD) -buildmode=c-shared -o %[1]s_go$(LIBEXT) %[1]s.go + # writes _%[1]s.py, the cffi module %[1]s.py imports + $(PYTHON) build.py + ` // exe version of template: 3 = gencmd, 4 = vm, 5 = libext, 8 = gopy version @@ -685,10 +714,9 @@ func (g *pyGen) genPkg(p *Package) { g.pkg = nil } -// wantMakefile reports whether to write a Makefile, which only knows how to -// build the default backend. +// wantMakefile reports whether to write a Makefile. func (g *pyGen) wantMakefile() bool { - return !NoMake && !g.isCFFI() + return !NoMake } func (g *pyGen) genGoPreamble() { @@ -838,6 +866,8 @@ func (g *pyGen) genMakefile() { if g.mode == ModeExe { g.makefile.Printf(MakefileExeTemplate, g.cfg.Name, g.cfg.Cmd, gencmd, g.cfg.VM, g.libext, pycfg.CFlags, pycfg.LdFlags, g.cfg.Version) + } else if g.isCFFI() { + g.makefile.Printf(MakefileTemplateCFFI, g.cfg.Name, g.cfg.Cmd, gencmd, g.cfg.VM, g.libext, "", "", "", "", g.cfg.Version) } else { winhack := "" if WindowsOS { diff --git a/bind/gen_func.go b/bind/gen_func.go index 57749b1b..efa24bd6 100644 --- a/bind/gen_func.go +++ b/bind/gen_func.go @@ -67,6 +67,23 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { return false } + // cffi has no way to cross a raw PyObject* (complex64/128, and Python + // callback arguments -- see isSignature() below): skip these functions + // rather than emit a signature that references the CPython C API, which + // would fail to even compile under the cffi preamble. + if g.isCFFI() { + for _, arg := range args { + if sarg := current.symtype(arg.GoType()); sarg != nil && sarg.cpyname == "PyObject*" { + return false + } + } + for _, ret := range res { + if sret := current.symtype(ret.GoType()); sret != nil && sret.cpyname == "PyObject*" { + return false + } + } + } + var ( goArgs []string pyArgs []string diff --git a/bind/gen_slice.go b/bind/gen_slice.go index 2eef8f0a..7f277e7c 100644 --- a/bind/gen_slice.go +++ b/bind/gen_slice.go @@ -70,6 +70,14 @@ func (g *pyGen) genSliceInit(slc *symbol, extTypes, pyWrapOnly bool, slob *Slice esym = current.symtype(typ.Elem()) } + // element access (elem/set/append) below would reference *C.PyObject, + // which cffi's preamble doesn't declare (see genFuncSig for the same + // restriction on plain function args/returns); skip the whole wrapper + // rather than emit code that fails to compile. + if g.isCFFI() && esym != nil && esym.cpyname == "PyObject*" { + return + } + gocl := "go." if g.pkg == goPackage { gocl = "" @@ -395,8 +403,60 @@ otherwise parameter is a python list that we copy from } if slNm == "Slice_byte" { - // these take and return python bytes objects, which the cffi backend does not support yet - if !g.isCFFI() { + if g.isCFFI() { + // PyBytes_* is off-limits for cffi (no CPython headers), so these + // exchange a raw pointer+length instead of a PyObject*; the cffi + // build script (cffi_build.py) recognizes them by name and writes + // the bytes<->buffer conversion into the generated python module. + g.gofile.Printf("//export Slice_byte_from_bytes\n") + g.gofile.Printf("func Slice_byte_from_bytes(ptr unsafe.Pointer, size C.longlong) CGoHandle {\n") + g.gofile.Indent() + g.gofile.Printf("data := make([]byte, size)\n") + g.gofile.Printf("if size > 0 {\n") + g.gofile.Indent() + g.gofile.Printf("tmp := unsafe.Slice((*byte)(ptr), size)\n") + g.gofile.Printf("copy(data, tmp)\n") + g.gofile.Outdent() + g.gofile.Printf("}\n") + g.gofile.Printf("return handleFromPtr_Slice_byte(&data)\n") + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + + g.gofile.Printf("//export Slice_byte_to_bytes_len\n") + g.gofile.Printf("func Slice_byte_to_bytes_len(handle CGoHandle) C.longlong {\n") + g.gofile.Indent() + g.gofile.Printf("s := deptrFromHandle_Slice_byte(handle)\n") + g.gofile.Printf("return C.longlong(len(s))\n") + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + + // Returning &s[0] directly would hand cgo a pointer into the Go + // heap, which cgo's pointer checks reject once it crosses back + // to the caller; copy into a C-owned buffer instead, freed by + // the python side (Slice_byte_free_ptr) once it has read it. + g.gofile.Printf("//export Slice_byte_to_bytes_ptr\n") + g.gofile.Printf("func Slice_byte_to_bytes_ptr(handle CGoHandle) unsafe.Pointer {\n") + g.gofile.Indent() + g.gofile.Printf("s := deptrFromHandle_Slice_byte(handle)\n") + g.gofile.Printf("n := len(s)\n") + g.gofile.Printf("if n == 0 {\n") + g.gofile.Indent() + g.gofile.Printf("return nil\n") + g.gofile.Outdent() + g.gofile.Printf("}\n") + g.gofile.Printf("buf := C.malloc(C.size_t(n))\n") + g.gofile.Printf("copy(unsafe.Slice((*byte)(buf), n), s)\n") + g.gofile.Printf("return buf\n") + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + + g.gofile.Printf("//export Slice_byte_free_ptr\n") + g.gofile.Printf("func Slice_byte_free_ptr(ptr unsafe.Pointer) {\n") + g.gofile.Indent() + g.gofile.Printf("C.free(ptr)\n") + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + } else { g.gofile.Printf("//export Slice_byte_from_bytes\n") g.gofile.Printf("func Slice_byte_from_bytes(o *C.PyObject) CGoHandle {\n") g.gofile.Indent() @@ -422,10 +482,10 @@ otherwise parameter is a python list that we copy from } g.gofile.Outdent() g.gofile.Printf("}\n\n") - } - g.pybuild.Printf("mod.add_function('Slice_byte_from_bytes', retval('%s'%s), [param('PyObject*', 'o', transfer_ownership=False)])\n", PyHandle, caller_owns_ret) - g.pybuild.Printf("mod.add_function('Slice_byte_to_bytes', retval('PyObject*', caller_owns_return=True), [param('%s', 'handle')])\n", PyHandle) + g.pybuild.Printf("mod.add_function('Slice_byte_from_bytes', retval('%s'%s), [param('PyObject*', 'o', transfer_ownership=False)])\n", PyHandle, caller_owns_ret) + g.pybuild.Printf("mod.add_function('Slice_byte_to_bytes', retval('PyObject*', caller_owns_return=True), [param('%s', 'handle')])\n", PyHandle) + } } } } diff --git a/gen.go b/gen.go index b7591b0e..d0cd0419 100644 --- a/gen.go +++ b/gen.go @@ -66,8 +66,11 @@ func genPkg(mode bind.BuildMode, cfg *BuildCfg) error { if cfg.Backend, err = bind.BackendFromEnv(); err != nil { return err } - if cfg.Backend == bind.BackendCFFI && (mode == bind.ModePkg || mode == bind.ModeExe) { - return fmt.Errorf("gopy: %s=%s only supports gopy gen and gopy build", bind.BackendEnvVar, cfg.Backend) + if cfg.Backend == bind.BackendCFFI && mode == bind.ModeExe { + // exe mode embeds the Python interpreter into the Go binary via the + // CPython C API (see goExePreambleC/Go in bind/gen.go), unrelated to + // how the bindings themselves are generated; cffi doesn't support it. + return fmt.Errorf("gopy: %s=%s does not support gopy exe", bind.BackendEnvVar, cfg.Backend) } cfg.OutputDir, err = genOutDir(cfg.OutputDir) if err != nil { From 93188fb1b8ce967228499263612a6ed2a1284a9d Mon Sep 17 00:00:00 2001 From: b-long Date: Sat, 19 Sep 2026 08:47:43 -0400 Subject: [PATCH 06/21] Complete complex64/128 number support --- .github/workflows/ci.yml | 7 ++-- bind/cffi_build.py | 67 +++++++++++++++++++++++++++++--- bind/gen_func.go | 83 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d849ba36..3524e3b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,7 +137,8 @@ jobs: run: go build -v ./... # The skipped tests use features the cffi backend does not support yet: - # Python callback arguments (TestBindFuncs), complex numbers as a plain - # function arg/return (TestBindSimple) or slice element (TestBuiltinSlices). + # Python callback arguments (TestBindFuncs), and complex numbers as a + # slice element rather than a plain function arg/return (TestBuiltinSlices; + # see genFuncComplexCFFI in gen_func.go for the arg/return case, which works). - name: Test - run: go test -v -skip '^(TestBindFuncs|TestBindSimple|TestBuiltinSlices)$' ./... + run: go test -v -skip '^(TestBindFuncs|TestBuiltinSlices)$' ./... diff --git a/bind/cffi_build.py b/bind/cffi_build.py index cb459fda..dc4648c5 100644 --- a/bind/cffi_build.py +++ b/bind/cffi_build.py @@ -32,15 +32,24 @@ def add_include(self, inc): self.header = inc.strip('"') def add_function(self, name, ret, params, *a, **kw): - self.funcs.append((name, ret, params)) + self.funcs.append(("plain", name, ret, params)) + + def add_complex_function(self, name, nargs): + # see genFuncComplexCFFI (gen_func.go) for the calling convention. + self.funcs.append(("complex", name, nargs)) def generate(self): here = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(here, self.header)) as f: typedefs, cdefs = go_decls(f.read()) out = [MODULE_HEAD.replace("@CDEFS@", repr("\n".join(typedefs + list(cdefs.values()))))] - for name, ret, params in self.funcs: - out.append(wrapper(name, ret, params, name in cdefs)) + for entry in self.funcs: + if entry[0] == "complex": + _, name, nargs = entry + out.append(complex_wrapper(name, nargs, name in cdefs)) + else: + _, name, ret, params = entry + out.append(wrapper(name, ret, params, name in cdefs)) # Slice_byte's converters exchange a raw pointer+length instead of a # PyObject* (see gen_slice.go); they are recognized by name here # rather than recorded via add_function, since their python bodies @@ -59,20 +68,44 @@ def add_checked_function(mod, name, retval, params, failure_expression="", *a, * def go_decls(header): - """Returns the Go typedefs, and {function name: cdef line} for the functions cgo exports.""" + """Returns the Go typedefs (plus any "struct X { ... };" body -- cgo emits + one ahead of a multi-value-returning export, see genFuncComplexCFFI in + gen_func.go), and {function name: cdef line} for the functions cgo exports. + """ typedefs = [] decls = {} ffi = cffi.FFI() - for line in header.split("\n"): + lines = header.split("\n") + i = 0 + while i < len(lines): + line = lines[i] + m = re.match(r"struct (\w+) \{$", line) + if m: + block = [line] + i += 1 + while i < len(lines) and lines[i] != "};": + block.append(lines[i]) + i += 1 + block.append("};") + i += 1 + decl = "\n".join(block) + try: + ffi.cdef(decl) + typedefs.append(decl) + except Exception as err: + print("gopy: cffi cannot declare struct %s: %s" % (m.group(1), err), file=sys.stderr) + continue if re.match(r"typedef [\w ]+ Go\w+;$", line): try: ffi.cdef(line) typedefs.append(line) except Exception: pass # e.g. GoComplex64: not used by exports + i += 1 continue m = re.match(r"extern (.*?(\w+)\(.*\));$", line.replace("__declspec(dllexport) ", "")) if not m or "_GoString_" in line: + i += 1 continue # cffi takes a plain char as a byte string only; the integer kinds # (bool, int8, byte) are passed as ints, with the same C ABI. @@ -81,8 +114,10 @@ def go_decls(header): ffi.cdef(decl) except Exception as err: print("gopy: cffi cannot declare %s: %s" % (m.group(2), err), file=sys.stderr) + i += 1 continue decls[m.group(2)] = decl + i += 1 return typedefs, decls @@ -119,6 +154,28 @@ def wrapper(name, ret, params, exported): return "\n".join(body) + "\n" +def complex_wrapper(name, nargs, exported): + """A plain function whose every argument and return value is + complex64/128 (see genFuncComplexCFFI, gen_func.go): each argument + crosses as two floats (.real, .imag), and the return value comes back + as the {r0, r1} struct cgo generates for a two-value Go return. + """ + if not exported: + return ( + "def %s(*args):\n" + " raise NotImplementedError('%s is not available with the cffi backend')\n" % (name, name) + ) + names = ["c%d" % i for i in range(nargs)] + sig = ", ".join(names) + callargs = ", ".join("%s.real, %s.imag" % (n, n) for n in names) + return ( + "def %s(%s):\n" + " _r = _lib.%s(%s)\n" + " _check()\n" + " return complex(_r.r0, _r.r1)\n" % (name, sig, name, callargs) + ) + + MODULE_HEAD = '''# python bindings for package @NAME@ using cffi. # File is generated by gopy version @VERSION@. Do not edit. import builtins diff --git a/bind/gen_func.go b/bind/gen_func.go index efa24bd6..595ac986 100644 --- a/bind/gen_func.go +++ b/bind/gen_func.go @@ -213,11 +213,94 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { } func (g *pyGen) genFunc(o *Func) { + if g.isCFFI() && g.genFuncComplexCFFI(o) { + return + } if g.genFuncSig(nil, o) { g.genFuncBody(nil, o) } } +func isComplexSym(sym *symbol) bool { + return sym != nil && (sym.goname == "complex64" || sym.goname == "complex128") +} + +// genFuncComplexCFFI generates a plain (non-method) function whose every +// argument and its one return value are complex64/128. The normal path +// (genFuncSig/genFuncBody) represents complex64/128 as a *C.PyObject, which +// the cffi preamble doesn't declare -- fine for most types, but complex +// values need actual marshaling here, not just a skip. Instead, an +// argument crosses as two plain floats (real, imag), and the return value +// uses Go's native multi-value return, which cgo exports as a small C +// struct {r0; r1;} that cffi can declare (see go_decls/complex_wrapper in +// cffi_build.py). It returns false, writing nothing, if the signature +// doesn't fit that narrow shape (methods, a mix of complex and other +// argument types, or an error return) -- genFuncSig's PyObject* check then +// skips the function instead of emitting code that fails to compile. +func (g *pyGen) genFuncComplexCFFI(fsym *Func) bool { + sig := fsym.sig + if sig == nil || fsym.isVariadic || fsym.err { + return false + } + args := sig.Params() + res := sig.Results() + if len(res) != 1 || !isComplexSym(current.symtype(res[0].GoType())) { + return false + } + for _, arg := range args { + if !isComplexSym(current.symtype(arg.GoType())) { + return false + } + } + + gname := fsym.GoName() + if g.cfg.RenameCase { + gname = toSnakeCase(gname) + } + gname, gdoc, err := extractPythonName(gname, fsym.Doc()) + if err != nil { + return false + } + + cfloatOf := func(sym *symbol) (cgo, gotyp string) { + if sym.goname == "complex64" { + return "C.float", "float32" + } + return "C.double", "float64" + } + + retFloat, _ := cfloatOf(current.symtype(res[0].GoType())) + + var goArgs, callArgs, wpArgs []string + for i, arg := range args { + anm := pySafeArg(arg.Name(), i) + cfloat, gofloat := cfloatOf(current.symtype(arg.GoType())) + reNm, imNm := anm+"_re", anm+"_im" + goArgs = append(goArgs, fmt.Sprintf("%s %s, %s %s", reNm, cfloat, imNm, cfloat)) + callArgs = append(callArgs, fmt.Sprintf("complex(%s(%s), %s(%s))", gofloat, reNm, gofloat, imNm)) + wpArgs = append(wpArgs, anm) + } + + g.gofile.Printf("\n//export %s\n", fsym.ID()) + g.gofile.Printf("func %s(%s) (%s, %s) {\n", fsym.ID(), strings.Join(goArgs, ", "), retFloat, retFloat) + g.gofile.Indent() + g.gofile.Printf("_r := %s(%s)\n", fsym.GoFmt(), strings.Join(callArgs, ", ")) + g.gofile.Printf("return %s(real(_r)), %s(imag(_r))\n", retFloat, retFloat) + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + + g.pybuild.Printf("mod.add_complex_function('%s', %d)\n", fsym.ID(), len(args)) + + g.pywrap.Printf("def %s(%s):\n", gname, strings.Join(wpArgs, ", ")) + g.pywrap.Indent() + g.pywrap.Printf(`"""%s"""`, gdoc) + g.pywrap.Printf("\n") + g.pywrap.Printf("return _%s.%s(%s)\n", g.cfg.Name, fsym.ID(), strings.Join(wpArgs, ", ")) + g.pywrap.Outdent() + + return true +} + func (g *pyGen) genMethod(s *symbol, o *Func) { if g.genFuncSig(s, o) { g.genFuncBody(s, o) From 36c319c839bde3331f64f145738bcde77fc5e8a4 Mon Sep 17 00:00:00 2001 From: b-long Date: Sat, 19 Sep 2026 20:57:54 -0400 Subject: [PATCH 07/21] Support Python callbacks in cffi backend --- .github/workflows/ci.yml | 7 +- SUPPORT_MATRIX.md | 1 + _examples/callbacks/callbacks.go | 55 ++++++++++ _examples/callbacks/test.py | 65 ++++++++++++ bind/cffi.go | 30 +++++- bind/cffi_build.py | 36 ++++++- bind/cffi_callback.go | 175 +++++++++++++++++++++++++++++++ bind/gen.go | 4 +- bind/gen_func.go | 28 +++-- main_test.go | 33 ++++++ 10 files changed, 421 insertions(+), 13 deletions(-) create mode 100644 _examples/callbacks/callbacks.go create mode 100644 _examples/callbacks/test.py create mode 100644 bind/cffi_callback.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3524e3b7..903861b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,8 +137,9 @@ jobs: run: go build -v ./... # The skipped tests use features the cffi backend does not support yet: - # Python callback arguments (TestBindFuncs), and complex numbers as a - # slice element rather than a plain function arg/return (TestBuiltinSlices; - # see genFuncComplexCFFI in gen_func.go for the arg/return case, which works). + # Python callbacks that take an interface{} or return a value + # (TestBindFuncs; see cffi_callback.go for the ones that work), and complex + # numbers as a slice element rather than a plain function arg/return + # (TestBuiltinSlices; see genFuncComplexCFFI in gen_func.go for the case that works). - name: Test run: go test -v -skip '^(TestBindFuncs|TestBuiltinSlices)$' ./... diff --git a/SUPPORT_MATRIX.md b/SUPPORT_MATRIX.md index 8f30be77..afaae308 100644 --- a/SUPPORT_MATRIX.md +++ b/SUPPORT_MATRIX.md @@ -6,6 +6,7 @@ don't modify manually. Feature |py3 --- | --- _examples/arrays | yes +_examples/callbacks | yes _examples/cgo | yes _examples/consts | yes _examples/cstrings | yes diff --git a/_examples/callbacks/callbacks.go b/_examples/callbacks/callbacks.go new file mode 100644 index 00000000..ad0add37 --- /dev/null +++ b/_examples/callbacks/callbacks.go @@ -0,0 +1,55 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package callbacks has Go functions that take Python callables, and call +// them before returning. +package callbacks + +import ( + "fmt" + "sync" +) + +// Each calls fun for i in 0..n-1, with a label made from i. +func Each(n int, fun func(i int, label string)) { + for i := 0; i < n; i++ { + fun(i, fmt.Sprintf("item-%d", i)) + } +} + +// Mixed calls fun with a bool, a float and an unsigned integer. +func Mixed(fun func(on bool, x float64, u uint8)) { + fun(true, 1.5, 200) + fun(false, -2.25, 7) +} + +// Twice calls fun, which takes no arguments, two times. +func Twice(fun func()) { + fun() + fun() +} + +// Counter counts how many times it has been visited. +type Counter struct { + N int +} + +// Visit calls fun with the counter itself, which arrives as a handle. +func (c *Counter) Visit(times int, fun func(c *Counter, n int)) { + for i := 0; i < times; i++ { + c.N++ + fun(c, c.N) + } +} + +// InGoroutine calls fun from another goroutine, and waits for it. +func InGoroutine(fun func(i int)) { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + fun(7) + }() + wg.Wait() +} diff --git a/_examples/callbacks/test.py b/_examples/callbacks/test.py new file mode 100644 index 00000000..3e652e19 --- /dev/null +++ b/_examples/callbacks/test.py @@ -0,0 +1,65 @@ +# Copyright 2026 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from __future__ import print_function + +import io +import sys + +import callbacks + +print("--- Each: int and string arguments") +callbacks.Each(3, lambda i, label: print("each:", i, label)) + +print("--- Mixed: bool, float and uint8 arguments") +# bool() because the pybindgen backend passes a bool as 1 or 0 +callbacks.Mixed(lambda on, x, u: print("mixed:", bool(on), x, u)) + +print("--- Twice: no arguments") +calls = [] +callbacks.Twice(lambda: calls.append(1)) +print("twice:", len(calls)) + +print("--- Counter.Visit: a Go struct arrives as a handle") +c = callbacks.Counter() + +def visit(handle, n): + seen = callbacks.Counter(handle=handle) + print("visit:", n, seen.N) + +c.Visit(2, visit) +print("counter:", c.N) + +print("--- a bound method") + +class Box(object): + def __init__(self): + self.items = [] + + def add(self, i, label): + self.items.append((i, label)) + +box = Box() +callbacks.Each(2, box.add) +print("box:", box.items) + +print("--- called from another goroutine") +callbacks.InGoroutine(lambda i: print("goroutine:", i)) + +print("--- an exception in a callback is reported, and Go carries on") +seen = [] + +def boom(i, label): + seen.append(i) + raise ValueError("boom %d" % i) + +stderr, sys.stderr = sys.stderr, io.StringIO() +try: + callbacks.Each(3, boom) + reported = sys.stderr.getvalue() +finally: + sys.stderr = stderr +print("calls:", len(seen), "reported:", reported.count("ValueError: boom")) + +print("OK") diff --git a/bind/cffi.go b/bind/cffi.go index 04dfabcd..8a880da2 100644 --- a/bind/cffi.go +++ b/bind/cffi.go @@ -38,7 +38,8 @@ func (g *pyGen) goSetError(kind, msg string) string { } // same argument positions as goPreamble: 1 = name of package, 2 = cmdstr, -// 4 = GoHandle, 5 = CGoHandle, 6 = all imports, 7 = mainstr, 10 = gopy version. +// 4 = GoHandle, 5 = CGoHandle, 6 = all imports, 7 = mainstr, 8 = C trampolines for +// callbacks (see cffi_callback.go), 10 = gopy version. const goPreambleCFFI = `/* cgo stubs for package %[1]s, for use with cffi. File is generated by gopy version %[10]s. Do not edit. @@ -53,6 +54,7 @@ package main #if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 202311L) typedef uint8_t bool; #endif +%[8]s */ import "C" import ( @@ -148,6 +150,32 @@ func GopyFreeString(s *C.char) { C.free(unsafe.Pointer(s)) } +// gopyCallbackScope guards a python callback passed to Go, which only exists +// while the python call it was passed to is running (see cffi_callback.go): +// close waits for callbacks that are running, and then refuses new ones. +type gopyCallbackScope struct { + mu sync.RWMutex + done bool +} + +func (s *gopyCallbackScope) enter() bool { + s.mu.RLock() + if s.done { + s.mu.RUnlock() + println("gopy: callback called after the python call it was passed to returned") + return false + } + return true +} + +func (s *gopyCallbackScope) leave() { s.mu.RUnlock() } + +func (s *gopyCallbackScope) close() { + s.mu.Lock() + s.done = true + s.mu.Unlock() +} + // boolGoToPy converts a Go bool to python-compatible C.char func boolGoToPy(b bool) C.char { if b { diff --git a/bind/cffi_build.py b/bind/cffi_build.py index dc4648c5..df76e687 100644 --- a/bind/cffi_build.py +++ b/bind/cffi_build.py @@ -130,8 +130,12 @@ def wrapper(name, ret, params, exported): " raise NotImplementedError('%s is not available with the cffi backend')\n" % (name, name) ) args = [] + setup = [] for i, (ctype, pname) in enumerate(params): - if ctype == "char*": + if ctype.startswith("callback:"): + setup.append(callback_setup(pname, ctype, i + 1)) + args.append("_ffi.cast('void*', _cb_%s)" % pname) + elif ctype == "char*": args.append("_enc(%s, %d)" % (pname, i + 1)) elif ctype == "bool": args.append("(1 if %s else 0)" % pname) @@ -141,7 +145,7 @@ def wrapper(name, ret, params, exported): args.append("_index(%s)" % pname) else: args.append(pname) - body = ["def %s(%s):" % (name, sig), " _r = _lib.%s(%s)" % (name, ", ".join(args))] + body = ["def %s(%s):" % (name, sig)] + setup + [" _r = _lib.%s(%s)" % (name, ", ".join(args))] if ret == "char*": body.append(" _r = _dec(_r)") elif ret == "bool": @@ -154,6 +158,34 @@ def wrapper(name, ret, params, exported): return "\n".join(body) + "\n" +def callback_setup(pname, ctype, argn): + """Returns the python source that wraps the callable pname in an + ffi.callback, for the C signature in ctype ("callback:void(int64_t,char*)", + see cffi_callback.go). The wrapper passes it on to Go as _cb_, + which is kept referenced by this local variable until the Go call + returns: cffi frees a callback as soon as nothing refers to it. + """ + cargs = ctype[len("callback:void("):-1] + ctypes = cargs.split(",") if cargs else [] + names = ["a%d" % i for i in range(len(ctypes))] + conv = [] + for n, t in zip(names, ctypes): + if t == "char*": + conv.append('_ffi.string(%s).decode("utf-8")' % n) + elif t == "bool": + conv.append("bool(%s)" % n) + else: + conv.append(n) + cdecl = "void(%s)" % ", ".join("unsigned char" if t == "bool" else t for t in ctypes) + return "\n".join([ + " if not callable(%s):" % pname, + " raise TypeError('argument %d must be callable, not %%s' %% type(%s).__name__)" % (argn, pname), + " def _cbfn_%s(%s):" % (pname, ", ".join(names)), + " %s(%s)" % (pname, ", ".join(conv)), + " _cb_%s = _ffi.callback(%r, _cbfn_%s)" % (pname, cdecl, pname), + ]) + + def complex_wrapper(name, nargs, exported): """A plain function whose every argument and return value is complex64/128 (see genFuncComplexCFFI, gen_func.go): each argument diff --git a/bind/cffi_callback.go b/bind/cffi_callback.go new file mode 100644 index 00000000..1c357745 --- /dev/null +++ b/bind/cffi_callback.go @@ -0,0 +1,175 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + "bytes" + "fmt" + "go/types" + "strings" +) + +// A Python callable passed to Go as a func-typed argument crosses the cffi +// boundary as a C function pointer: the python side wraps the callable in an +// ffi.callback (see callback_setup in cffi_build.py), and the Go closure +// built here calls that pointer through a small static C trampoline, since +// cgo cannot call a C function pointer directly. +// +// The callback only lives as long as the Go call it was passed to, so the +// closure goes through a gopyCallbackScope that is closed when that call +// returns, instead of jumping to freed memory if Go kept it any longer. +// +// Only what is needed so far is supported: no results, and parameters +// limited to pointer/interface handles, numbers, bool and string. Any other +// callback type makes the function be skipped, see genFuncSig. + +// cffiTrampolinesKey stands in for the trampolines in the cgo preamble, +// which is written before the callback types that need them are known. +const cffiTrampolinesKey = "@@GOPY_CFFI_TRAMPOLINES@@" + +// cffiCBParam is one parameter of a callback. +type cffiCBParam struct { + name string // name of the parameter in the func literal + gotyp string // its Go type + ctype string // how it crosses: int64_t, uint64_t, double, char* or bool + pre string // Go statements to run before the call, if any + conv string // Go expression converting it to ctype +} + +type cffiCallback struct { + params []cffiCBParam +} + +// cffiCallback returns how to pass sym, a func-typed argument, to Go +// as a callback, or nil if its type isn't supported. +func (g *pyGen) cffiCallback(sym *symbol) *cffiCallback { + sig, ok := sym.GoType().Underlying().(*types.Signature) + if !ok || sig.Results().Len() != 0 || sig.Variadic() { + return nil + } + cb := &cffiCallback{} + for i := 0; i < sig.Params().Len(); i++ { + p, ok := cffiCallbackParam(sig.Params().At(i), i) + if !ok { + return nil + } + cb.params = append(cb.params, p) + } + return cb +} + +func cffiCallbackParam(v *types.Var, i int) (cffiCBParam, bool) { + typ := v.Type() + vsym := current.symtype(typ) + if vsym == nil { + return cffiCBParam{}, false + } + nm := pySafeArg(v.Name(), i) + p := cffiCBParam{name: nm, gotyp: current.typeGoName(typ)} + + if vsym.hasHandle() && vsym.isPtrOrIface() { + p.ctype = "int64_t" + p.conv = fmt.Sprintf("C.int64_t(%s(%s)%s)", vsym.go2py, nm, vsym.go2pyParenEx) + return p, true + } + bt, ok := typ.Underlying().(*types.Basic) + if !ok { + return p, false + } + switch k := bt.Kind(); { + case types.Int <= k && k <= types.Int64: + p.ctype, p.conv = "int64_t", fmt.Sprintf("C.int64_t(%s)", nm) + case types.Uint <= k && k <= types.Uintptr: + p.ctype, p.conv = "uint64_t", fmt.Sprintf("C.uint64_t(%s)", nm) + case k == types.Float32 || k == types.Float64: + p.ctype, p.conv = "double", fmt.Sprintf("C.double(%s)", nm) + case k == types.Bool: + p.ctype, p.conv = "bool", fmt.Sprintf("C.uint8_t(boolGoToPy(bool(%s)))", nm) + case k == types.String: + // the python side copies it, so it can be freed once the call returns + p.ctype, p.conv = "char*", "_c"+nm + p.pre = fmt.Sprintf("_c%[1]s := C.CString(string(%[1]s))\ndefer C.free(unsafe.Pointer(_c%[1]s))\n", nm) + default: + return p, false + } + return p, true +} + +func (cb *cffiCallback) ctypes() []string { + ts := make([]string, len(cb.params)) + for i, p := range cb.params { + ts[i] = p.ctype + } + return ts +} + +// pyType is the type given to the callback's parameter in build.py, which +// callback_setup in cffi_build.py takes apart. +func (cb *cffiCallback) pyType() string { + return "callback:void(" + strings.Join(cb.ctypes(), ",") + ")" +} + +// cffiCallbackPrologue returns the Go statements that set up the callback +// argument named anm, ahead of cffiCallbackLit. +func cffiCallbackPrologue(anm string) string { + return fmt.Sprintf("_cbfp_%[1]s := %[1]s\n_cbs_%[1]s := new(gopyCallbackScope)\ndefer _cbs_%[1]s.close()\n", anm) +} + +// cffiCallbackLit returns a Go func literal that calls the Python callable +// passed as the argument named anm. +func (g *pyGen) cffiCallbackLit(cb *cffiCallback, anm string) string { + var decl, pre, args []string + args = append(args, "_cbfp_"+anm) + for _, p := range cb.params { + decl = append(decl, p.name+" "+p.gotyp) + if p.pre != "" { + pre = append(pre, p.pre) + } + args = append(args, p.conv) + } + return fmt.Sprintf("func(%s) {\nif !_cbs_%s.enter() {\nreturn\n}\ndefer _cbs_%[2]s.leave()\n%sC.gopy_cb_%d(%s)\n}", + strings.Join(decl, ", "), anm, strings.Join(pre, ""), g.cffiTrampoline(cb), strings.Join(args, ", ")) +} + +// cffiTrampoline returns the number of the C trampoline that calls a callback +// like cb, adding it if it is the first. +func (g *pyGen) cffiTrampoline(cb *cffiCallback) int { + key := strings.Join(cb.ctypes(), ",") + for i, k := range g.cbSigs { + if k == key { + return i + } + } + g.cbSigs = append(g.cbSigs, key) + return len(g.cbSigs) - 1 +} + +// spliceCFFITrampolines writes the trampolines into the cgo preamble. +func (g *pyGen) spliceCFFITrampolines() { + if !g.isCFFI() { + return + } + var c strings.Builder + for i, key := range g.cbSigs { + params := []string{"void* f"} + var args, ptypes []string + if key != "" { + for j, t := range strings.Split(key, ",") { + if t == "bool" { + t = "uint8_t" + } + params = append(params, fmt.Sprintf("%s a%d", t, j)) + args = append(args, fmt.Sprintf("a%d", j)) + ptypes = append(ptypes, t) + } + } else { + ptypes = append(ptypes, "void") + } + fmt.Fprintf(&c, "static inline void gopy_cb_%d(%s) { ((void (*)(%s))f)(%s); }\n", + i, strings.Join(params, ", "), strings.Join(ptypes, ", "), strings.Join(args, ", ")) + } + b := bytes.Replace(g.gofile.buf.Bytes(), []byte(cffiTrampolinesKey), []byte(c.String()), 1) + g.gofile.buf = bytes.NewBuffer(b) +} diff --git a/bind/gen.go b/bind/gen.go index 85db4741..f4552375 100644 --- a/bind/gen.go +++ b/bind/gen.go @@ -608,6 +608,7 @@ type pyGen struct { extraGccArgs string lang int // c-python api version (2,3) dynamicLink bool + cbSigs []string // cffi: the callback types that have a C trampoline, see cffi_callback.go } func (g *pyGen) gen() error { @@ -670,6 +671,7 @@ func (g *pyGen) genOut() { g.pybuild.Printf("\nmod.generate(open('%v.c', 'w'))\n\n", g.cfg.Name) } g.gofile.Printf("\n\n") + g.spliceCFFITrampolines() g.genPrintOut(g.cfg.Name+".go", g.gofile) g.genPrintOut("build.py", g.pybuild) if g.wantMakefile() { @@ -731,7 +733,7 @@ func (g *pyGen) genGoPreamble() { } if g.isCFFI() { g.gofile.Printf(goPreambleCFFI, g.cfg.Name, g.cfg.Cmd, "", GoHandle, CGoHandle, - pkgimport, g.cfg.Main, "", "", g.cfg.Version) + pkgimport, g.cfg.Main, cffiTrampolinesKey, "", g.cfg.Version) g.gofile.Printf("\n// --- generated code for package: %[1]s below: ---\n\n", g.cfg.Name) return } diff --git a/bind/gen_func.go b/bind/gen_func.go index 595ac986..0b8b7548 100644 --- a/bind/gen_func.go +++ b/bind/gen_func.go @@ -67,13 +67,20 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { return false } - // cffi has no way to cross a raw PyObject* (complex64/128, and Python - // callback arguments -- see isSignature() below): skip these functions + // cffi has no way to cross a raw PyObject* (complex64/128, and callback + // arguments of a type cffiCallback doesn't support): skip these functions // rather than emit a signature that references the CPython C API, which // would fail to even compile under the cffi preamble. if g.isCFFI() { for _, arg := range args { - if sarg := current.symtype(arg.GoType()); sarg != nil && sarg.cpyname == "PyObject*" { + sarg := current.symtype(arg.GoType()) + switch { + case sarg == nil: + case sarg.isSignature(): + if g.cffiCallback(sarg) == nil { + return false + } + case sarg.cpyname == "PyObject*": return false } } @@ -105,10 +112,14 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { } anm := pySafeArg(arg.Name(), i) - if ifchandle && arg.sym.goname == "interface{}" { + switch { + case g.isCFFI() && sarg.isSignature(): + goArgs = append(goArgs, fmt.Sprintf("%s unsafe.Pointer", anm)) + pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", g.cffiCallback(sarg).pyType(), anm)) + case ifchandle && arg.sym.goname == "interface{}": goArgs = append(goArgs, fmt.Sprintf("%s %s", anm, CGoHandle)) pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", PyHandle, anm)) - } else { + default: goArgs = append(goArgs, fmt.Sprintf("%s %s", anm, sarg.cgoname)) if sarg.cpyname == "PyObject*" { pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s', transfer_ownership=False)", sarg.cpyname, anm)) @@ -355,7 +366,10 @@ func (g *pyGen) genFuncBody(sym *symbol, fsym *Func) { g.gofile.Indent() if fsym.hasfun { for i, arg := range args { - if arg.sym.isSignature() { + switch { + case arg.sym.isSignature() && g.isCFFI(): + g.gofile.Printf("%s", cffiCallbackPrologue(pySafeArg(arg.Name(), i))) + case arg.sym.isSignature(): g.gofile.Printf("_fun_arg := %s\n", pySafeArg(arg.Name(), i)) } } @@ -405,6 +419,8 @@ if __err != nil { switch { case ifchandle && arg.sym.goname == "interface{}": na = fmt.Sprintf(`gopyh.VarFromHandle((gopyh.CGoHandle)(%s), "interface{}")`, anm) + case arg.sym.isSignature() && g.isCFFI(): + na = g.cffiCallbackLit(g.cffiCallback(arg.sym), anm) case arg.sym.isSignature(): na = fmt.Sprintf("%s", arg.sym.py2go) case arg.sym.py2go != "": diff --git a/main_test.go b/main_test.go index 0e0f7391..a1815a99 100644 --- a/main_test.go +++ b/main_test.go @@ -51,6 +51,7 @@ var ( "_examples/pkgconflict": []string{"py3"}, "_examples/variadic": []string{"py3"}, "_examples/gilstring": []string{"py3"}, + "_examples/callbacks": []string{"py3"}, } testEnvironment = os.Environ() @@ -392,6 +393,38 @@ OK }) } +func TestBindCallbacks(t *testing.T) { + // t.Parallel() + path := "_examples/callbacks" + testPkg(t, pkg{ + path: path, + lang: features[path], + cmd: "build", + extras: nil, + want: []byte(`--- Each: int and string arguments +each: 0 item-0 +each: 1 item-1 +each: 2 item-2 +--- Mixed: bool, float and uint8 arguments +mixed: True 1.5 200 +mixed: False -2.25 7 +--- Twice: no arguments +twice: 2 +--- Counter.Visit: a Go struct arrives as a handle +visit: 1 1 +visit: 2 2 +counter: 2 +--- a bound method +box: [(0, 'item-0'), (1, 'item-1')] +--- called from another goroutine +goroutine: 7 +--- an exception in a callback is reported, and Go carries on +calls: 3 reported: 3 +OK +`), + }) +} + func TestBindSimple(t *testing.T) { // t.Parallel() path := "_examples/simple" From d1206d0d06123504faf6e2382cd53099d23adcb7 Mon Sep 17 00:00:00 2001 From: b-long Date: Sat, 19 Sep 2026 21:22:22 -0400 Subject: [PATCH 08/21] Add interface{} args & results in cffi callbacks --- .github/workflows/ci.yml | 10 +- _examples/callbacks/callbacks.go | 49 +++++++++ _examples/callbacks/test.py | 18 ++++ bind/cffi_build.py | 30 ++++-- bind/cffi_callback.go | 172 ++++++++++++++++++++++--------- bind/gen.go | 2 +- main_test.go | 13 +++ 7 files changed, 228 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 903861b5..441d93e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,10 +136,8 @@ jobs: - name: Build run: go build -v ./... - # The skipped tests use features the cffi backend does not support yet: - # Python callbacks that take an interface{} or return a value - # (TestBindFuncs; see cffi_callback.go for the ones that work), and complex - # numbers as a slice element rather than a plain function arg/return - # (TestBuiltinSlices; see genFuncComplexCFFI in gen_func.go for the case that works). + # The skipped test uses a feature the cffi backend does not support yet: + # complex numbers as a slice element rather than a plain function + # arg/return (see genFuncComplexCFFI in gen_func.go for the case that works). - name: Test - run: go test -v -skip '^(TestBindFuncs|TestBuiltinSlices)$' ./... + run: go test -v -skip '^TestBuiltinSlices$' ./... diff --git a/_examples/callbacks/callbacks.go b/_examples/callbacks/callbacks.go index ad0add37..578fd5ba 100644 --- a/_examples/callbacks/callbacks.go +++ b/_examples/callbacks/callbacks.go @@ -9,6 +9,7 @@ package callbacks import ( "fmt" "sync" + "time" ) // Each calls fun for i in 0..n-1, with a label made from i. @@ -30,6 +31,49 @@ func Twice(fun func()) { fun() } +// Describe calls fun with a string and a fmt.Stringer, as interface{} values. +// They arrive as strings, made by fmt.Sprintf("%s", v). +func Describe(fun func(v interface{})) { + fun("a string") + fun(1500 * time.Millisecond) +} + +// Count returns how many of 0..n-1 keep says yes to. +func Count(n int, keep func(i int) bool) int { + total := 0 + for i := 0; i < n; i++ { + if keep(i) { + total++ + } + } + return total +} + +// Sum adds up what val returns for 0..n-1. +func Sum(n int, val func(i int) int) int { + total := 0 + for i := 0; i < n; i++ { + total += val(i) + } + return total +} + +// Widest returns the largest of what size returns for 0..n-1. +func Widest(n int, size func(i int) uint) uint { + var widest uint + for i := 0; i < n; i++ { + if w := size(i); w > widest { + widest = w + } + } + return widest +} + +// Apply returns f(x). +func Apply(x float64, f func(x float64) float64) float64 { + return f(x) +} + // Counter counts how many times it has been visited. type Counter struct { N int @@ -43,6 +87,11 @@ func (c *Counter) Visit(times int, fun func(c *Counter, n int)) { } } +// Check calls fun with the counter itself, and reports what it answered. +func (c *Counter) Check(fun func(c *Counter, n int) bool) bool { + return fun(c, c.N) +} + // InGoroutine calls fun from another goroutine, and waits for it. func InGoroutine(fun func(i int)) { var wg sync.WaitGroup diff --git a/_examples/callbacks/test.py b/_examples/callbacks/test.py index 3e652e19..d24aec27 100644 --- a/_examples/callbacks/test.py +++ b/_examples/callbacks/test.py @@ -31,6 +31,24 @@ def visit(handle, n): c.Visit(2, visit) print("counter:", c.N) +print("--- Describe: an interface{} arrives as a string") +callbacks.Describe(lambda v: print("describe:", repr(v))) + +print("--- Count: a bool result") +print("count:", callbacks.Count(10, lambda i: i % 3 == 0)) + +print("--- Sum: an int result") +print("sum:", callbacks.Sum(5, lambda i: i * i)) + +print("--- Widest: a uint result") +print("widest:", callbacks.Widest(4, lambda i: i * 10)) + +print("--- Apply: a float result") +print("apply:", callbacks.Apply(1.5, lambda x: x * 2)) + +print("--- Counter.Check: a handle argument and a bool result") +print("check:", c.Check(lambda handle, n: callbacks.Counter(handle=handle).N == n)) + print("--- a bound method") class Box(object): diff --git a/bind/cffi_build.py b/bind/cffi_build.py index df76e687..36b93ff7 100644 --- a/bind/cffi_build.py +++ b/bind/cffi_build.py @@ -160,13 +160,16 @@ def wrapper(name, ret, params, exported): def callback_setup(pname, ctype, argn): """Returns the python source that wraps the callable pname in an - ffi.callback, for the C signature in ctype ("callback:void(int64_t,char*)", - see cffi_callback.go). The wrapper passes it on to Go as _cb_, - which is kept referenced by this local variable until the Go call - returns: cffi frees a callback as soon as nothing refers to it. + ffi.callback, for the C signature in ctype + ("callback:()", see cffi_callback.go). + The wrapper passes it on to Go as _cb_, which is kept referenced by + this local variable until the Go call returns: cffi frees a callback as + soon as nothing refers to it. + + If the callable raises, cffi prints the traceback and returns 0 to Go. """ - cargs = ctype[len("callback:void("):-1] - ctypes = cargs.split(",") if cargs else [] + ret, _, rest = ctype[len("callback:"):].partition("(") + ctypes = [t for t in rest[:-1].split(",") if t] names = ["a%d" % i for i in range(len(ctypes))] conv = [] for n, t in zip(names, ctypes): @@ -176,12 +179,23 @@ def callback_setup(pname, ctype, argn): conv.append("bool(%s)" % n) else: conv.append(n) - cdecl = "void(%s)" % ", ".join("unsigned char" if t == "bool" else t for t in ctypes) + call = "%s(%s)" % (pname, ", ".join(conv)) + if ret == "void": + body = call + elif ret == "bool": + body = "return 1 if %s else 0" % call + else: + body = "return %s" % call + + def cdecl_type(t): + return "unsigned char" if t == "bool" else t + + cdecl = "%s(%s)" % (cdecl_type(ret), ", ".join(cdecl_type(t) for t in ctypes)) return "\n".join([ " if not callable(%s):" % pname, " raise TypeError('argument %d must be callable, not %%s' %% type(%s).__name__)" % (argn, pname), " def _cbfn_%s(%s):" % (pname, ", ".join(names)), - " %s(%s)" % (pname, ", ".join(conv)), + " %s" % body, " _cb_%s = _ffi.callback(%r, _cbfn_%s)" % (pname, cdecl, pname), ]) diff --git a/bind/cffi_callback.go b/bind/cffi_callback.go index 1c357745..284b6b11 100644 --- a/bind/cffi_callback.go +++ b/bind/cffi_callback.go @@ -21,9 +21,10 @@ import ( // closure goes through a gopyCallbackScope that is closed when that call // returns, instead of jumping to freed memory if Go kept it any longer. // -// Only what is needed so far is supported: no results, and parameters -// limited to pointer/interface handles, numbers, bool and string. Any other -// callback type makes the function be skipped, see genFuncSig. +// Supported: parameters that are pointer/interface handles, numbers, bool, +// string or interface{} (which arrives as a string, as with pybindgen), and a +// result that is a number or bool. Any other callback type makes the +// function be skipped, see genFuncSig. // cffiTrampolinesKey stands in for the trampolines in the cgo preamble, // which is written before the callback types that need them are known. @@ -38,15 +39,48 @@ type cffiCBParam struct { conv string // Go expression converting it to ctype } +// cffiCBResult is the result of a callback. +type cffiCBResult struct { + gotyp string // its Go type + ctype string // how it crosses: int64_t, uint64_t, double or bool +} + type cffiCallback struct { params []cffiCBParam + ret *cffiCBResult // nil if the callback has no result +} + +// cffiBasicCType returns how a value of a basic Go type crosses, or "" if it +// can't. +func cffiBasicCType(k types.BasicKind) string { + switch { + case types.Int <= k && k <= types.Int64: + return "int64_t" + case types.Uint <= k && k <= types.Uintptr: + return "uint64_t" + case k == types.Float32 || k == types.Float64: + return "double" + case k == types.Bool: + return "bool" + case k == types.String: + return "char*" + } + return "" +} + +// cffiCType returns the C type used for ctype in C code: bool is a byte. +func cffiCType(ctype string) string { + if ctype == "bool" { + return "uint8_t" + } + return ctype } // cffiCallback returns how to pass sym, a func-typed argument, to Go // as a callback, or nil if its type isn't supported. func (g *pyGen) cffiCallback(sym *symbol) *cffiCallback { sig, ok := sym.GoType().Underlying().(*types.Signature) - if !ok || sig.Results().Len() != 0 || sig.Variadic() { + if !ok || sig.Results().Len() > 1 || sig.Variadic() { return nil } cb := &cffiCallback{} @@ -57,9 +91,22 @@ func (g *pyGen) cffiCallback(sym *symbol) *cffiCallback { } cb.params = append(cb.params, p) } + if sig.Results().Len() == 1 { + r, ok := cffiCallbackResult(sig.Results().At(0).Type()) + if !ok { + return nil + } + cb.ret = r + } return cb } +// cStringPre returns the Go statements that make the C string _c from +// expr, for the duration of the call. The python side copies it. +func cStringPre(nm, expr string) string { + return fmt.Sprintf("_c%[1]s := %[2]s\ndefer C.free(unsafe.Pointer(_c%[1]s))\n", nm, expr) +} + func cffiCallbackParam(v *types.Var, i int) (cffiCBParam, bool) { typ := v.Type() vsym := current.symtype(typ) @@ -74,41 +121,55 @@ func cffiCallbackParam(v *types.Var, i int) (cffiCBParam, bool) { p.conv = fmt.Sprintf("C.int64_t(%s(%s)%s)", vsym.go2py, nm, vsym.go2pyParenEx) return p, true } + if vsym.goname == "interface{}" { + p.ctype, p.conv = "char*", "_c"+nm + p.pre = cStringPre(nm, fmt.Sprintf("%s(%s)%s", vsym.go2py, nm, vsym.go2pyParenEx)) + return p, true + } bt, ok := typ.Underlying().(*types.Basic) if !ok { return p, false } - switch k := bt.Kind(); { - case types.Int <= k && k <= types.Int64: - p.ctype, p.conv = "int64_t", fmt.Sprintf("C.int64_t(%s)", nm) - case types.Uint <= k && k <= types.Uintptr: - p.ctype, p.conv = "uint64_t", fmt.Sprintf("C.uint64_t(%s)", nm) - case k == types.Float32 || k == types.Float64: - p.ctype, p.conv = "double", fmt.Sprintf("C.double(%s)", nm) - case k == types.Bool: - p.ctype, p.conv = "bool", fmt.Sprintf("C.uint8_t(boolGoToPy(bool(%s)))", nm) - case k == types.String: - // the python side copies it, so it can be freed once the call returns - p.ctype, p.conv = "char*", "_c"+nm - p.pre = fmt.Sprintf("_c%[1]s := C.CString(string(%[1]s))\ndefer C.free(unsafe.Pointer(_c%[1]s))\n", nm) - default: + switch p.ctype = cffiBasicCType(bt.Kind()); p.ctype { + case "": return p, false + case "bool": + p.conv = fmt.Sprintf("C.uint8_t(boolGoToPy(bool(%s)))", nm) + case "char*": + p.conv = "_c" + nm + p.pre = cStringPre(nm, fmt.Sprintf("C.CString(string(%s))", nm)) + default: + p.conv = fmt.Sprintf("C.%s(%s)", p.ctype, nm) } return p, true } -func (cb *cffiCallback) ctypes() []string { - ts := make([]string, len(cb.params)) - for i, p := range cb.params { - ts[i] = p.ctype +func cffiCallbackResult(typ types.Type) (*cffiCBResult, bool) { + bt, ok := typ.Underlying().(*types.Basic) + if !ok { + return nil, false + } + switch ctype := cffiBasicCType(bt.Kind()); ctype { + case "", "char*": // the python side has no way to give Go a string it owns + return nil, false + default: + return &cffiCBResult{gotyp: current.typeGoName(typ), ctype: ctype}, true } - return ts } // pyType is the type given to the callback's parameter in build.py, which -// callback_setup in cffi_build.py takes apart. +// callback_setup in cffi_build.py takes apart: +// callback:() func (cb *cffiCallback) pyType() string { - return "callback:void(" + strings.Join(cb.ctypes(), ",") + ")" + ret := "void" + if cb.ret != nil { + ret = cb.ret.ctype + } + ts := make([]string, len(cb.params)) + for i, p := range cb.params { + ts[i] = p.ctype + } + return "callback:" + ret + "(" + strings.Join(ts, ",") + ")" } // cffiCallbackPrologue returns the Go statements that set up the callback @@ -120,30 +181,37 @@ func cffiCallbackPrologue(anm string) string { // cffiCallbackLit returns a Go func literal that calls the Python callable // passed as the argument named anm. func (g *pyGen) cffiCallbackLit(cb *cffiCallback, anm string) string { - var decl, pre, args []string - args = append(args, "_cbfp_"+anm) + var decl, pre []string + args := []string{"_cbfp_" + anm} for _, p := range cb.params { decl = append(decl, p.name+" "+p.gotyp) - if p.pre != "" { - pre = append(pre, p.pre) - } + pre = append(pre, p.pre) args = append(args, p.conv) } - return fmt.Sprintf("func(%s) {\nif !_cbs_%s.enter() {\nreturn\n}\ndefer _cbs_%[2]s.leave()\n%sC.gopy_cb_%d(%s)\n}", - strings.Join(decl, ", "), anm, strings.Join(pre, ""), g.cffiTrampoline(cb), strings.Join(args, ", ")) + call := fmt.Sprintf("C.gopy_cb_%d(%s)", g.cffiTrampoline(cb), strings.Join(args, ", ")) + result := "" + if cb.ret != nil { + // named, so that a refused call returns its zero value + result = " (_r " + cb.ret.gotyp + ")" + if cb.ret.ctype == "bool" { + call += " != 0" + } + call = "return " + cb.ret.gotyp + "(" + call + ")" + } + return fmt.Sprintf("func(%[1]s)%[2]s {\nif !_cbs_%[3]s.enter() {\nreturn\n}\ndefer _cbs_%[3]s.leave()\n%[4]s%[5]s\n}", + strings.Join(decl, ", "), result, anm, strings.Join(pre, ""), call) } // cffiTrampoline returns the number of the C trampoline that calls a callback // like cb, adding it if it is the first. func (g *pyGen) cffiTrampoline(cb *cffiCallback) int { - key := strings.Join(cb.ctypes(), ",") - for i, k := range g.cbSigs { - if k == key { + for i, c := range g.cbs { + if c.pyType() == cb.pyType() { return i } } - g.cbSigs = append(g.cbSigs, key) - return len(g.cbSigs) - 1 + g.cbs = append(g.cbs, cb) + return len(g.cbs) - 1 } // spliceCFFITrampolines writes the trampolines into the cgo preamble. @@ -152,23 +220,25 @@ func (g *pyGen) spliceCFFITrampolines() { return } var c strings.Builder - for i, key := range g.cbSigs { + for i, cb := range g.cbs { params := []string{"void* f"} - var args, ptypes []string - if key != "" { - for j, t := range strings.Split(key, ",") { - if t == "bool" { - t = "uint8_t" - } - params = append(params, fmt.Sprintf("%s a%d", t, j)) - args = append(args, fmt.Sprintf("a%d", j)) - ptypes = append(ptypes, t) - } - } else { + ptypes := []string{} + args := []string{} + for j, p := range cb.params { + t := cffiCType(p.ctype) + params = append(params, fmt.Sprintf("%s a%d", t, j)) + ptypes = append(ptypes, t) + args = append(args, fmt.Sprintf("a%d", j)) + } + if len(ptypes) == 0 { ptypes = append(ptypes, "void") } - fmt.Fprintf(&c, "static inline void gopy_cb_%d(%s) { ((void (*)(%s))f)(%s); }\n", - i, strings.Join(params, ", "), strings.Join(ptypes, ", "), strings.Join(args, ", ")) + ret, retStmt := "void", "" + if cb.ret != nil { + ret, retStmt = cffiCType(cb.ret.ctype), "return " + } + fmt.Fprintf(&c, "static inline %s gopy_cb_%d(%s) { %s((%s (*)(%s))f)(%s); }\n", + ret, i, strings.Join(params, ", "), retStmt, ret, strings.Join(ptypes, ", "), strings.Join(args, ", ")) } b := bytes.Replace(g.gofile.buf.Bytes(), []byte(cffiTrampolinesKey), []byte(c.String()), 1) g.gofile.buf = bytes.NewBuffer(b) diff --git a/bind/gen.go b/bind/gen.go index f4552375..52f694af 100644 --- a/bind/gen.go +++ b/bind/gen.go @@ -608,7 +608,7 @@ type pyGen struct { extraGccArgs string lang int // c-python api version (2,3) dynamicLink bool - cbSigs []string // cffi: the callback types that have a C trampoline, see cffi_callback.go + cbs []*cffiCallback // cffi: the callback types that have a C trampoline, see cffi_callback.go } func (g *pyGen) gen() error { diff --git a/main_test.go b/main_test.go index a1815a99..9a5a6c4d 100644 --- a/main_test.go +++ b/main_test.go @@ -414,6 +414,19 @@ twice: 2 visit: 1 1 visit: 2 2 counter: 2 +--- Describe: an interface{} arrives as a string +describe: 'a string' +describe: '1.5s' +--- Count: a bool result +count: 4 +--- Sum: an int result +sum: 30 +--- Widest: a uint result +widest: 30 +--- Apply: a float result +apply: 3.0 +--- Counter.Check: a handle argument and a bool result +check: True --- a bound method box: [(0, 'item-0'), (1, 'item-1')] --- called from another goroutine From a65a870f7055b75f547e54cb1ed09ae247749ba9 Mon Sep 17 00:00:00 2001 From: b-long Date: Sat, 19 Sep 2026 21:47:12 -0400 Subject: [PATCH 09/21] Complete cffi support for full test suite --- .github/workflows/ci.yml | 5 +- _examples/slices/slices.go | 5 ++ _examples/slices/test.py | 13 ++++++ bind/cffi.go | 96 ++++++++++++++++++++++++++++++++++++++ bind/cffi_build.py | 46 ++++-------------- bind/gen_func.go | 49 +++++++------------ bind/gen_slice.go | 42 ++++++----------- 7 files changed, 155 insertions(+), 101 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 441d93e8..f0c7b441 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,8 +136,5 @@ jobs: - name: Build run: go build -v ./... - # The skipped test uses a feature the cffi backend does not support yet: - # complex numbers as a slice element rather than a plain function - # arg/return (see genFuncComplexCFFI in gen_func.go for the case that works). - name: Test - run: go test -v -skip '^TestBuiltinSlices$' ./... + run: go test -v ./... diff --git a/_examples/slices/slices.go b/_examples/slices/slices.go index baa5d7e6..82d6777b 100644 --- a/_examples/slices/slices.go +++ b/_examples/slices/slices.go @@ -32,6 +32,7 @@ type SliceInt32 []int32 type SliceInt64 []int64 type SliceComplex []complex128 +type SliceComplex64 []complex64 type SliceIface []interface{} @@ -61,6 +62,10 @@ func CmplxSqrt(arr SliceComplex) SliceComplex { return res } +func CmplxArray() [3]complex128 { + return [3]complex128{1 + 1i, 2 + 2i, 3 + 3i} +} + func GetEmptyMatrix(xSize int, ySize int) [][]bool { result := [][]bool{} diff --git a/_examples/slices/test.py b/_examples/slices/test.py index 143f4533..b503909a 100644 --- a/_examples/slices/test.py +++ b/_examples/slices/test.py @@ -44,6 +44,19 @@ assert math.isclose(root_squared.real, orig.real) assert math.isclose(root_squared.imag, orig.imag) +# complex elements: assignment and append, in both float widths, and reading an array +cmplx[0] = 3 + 4j +assert cmplx[0] == 3 + 4j +cmplx.append(1 - 2j) +assert len(cmplx) == 17 and cmplx[16] == 1 - 2j + +cmplx64 = slices.SliceComplex64([1 + 2j, 3.5 - 4.25j]) +cmplx64[1] = -0.5 + 8j +cmplx64.append(2j) +assert list(cmplx64) == [1 + 2j, -0.5 + 8j, 2j] + +cmplx_arr = slices.CmplxArray() +assert len(cmplx_arr) == 3 and cmplx_arr[2] == 3 + 3j matrix = slices.GetEmptyMatrix(4,4) for i in range(4): diff --git a/bind/cffi.go b/bind/cffi.go index 8a880da2..2798babb 100644 --- a/bind/cffi.go +++ b/bind/cffi.go @@ -6,6 +6,7 @@ package bind import ( _ "embed" + "fmt" "strings" ) @@ -37,6 +38,84 @@ func (g *pyGen) goSetError(kind, msg string) string { return "gopySetError(\"" + kind + "\", " + msg + ")\n" } +func isComplexSym(sym *symbol) bool { + return sym != nil && (sym.goname == "complex64" || sym.goname == "complex128") +} + +// A complex64/complex128 value has no single C type that cffi can declare +// (cgo's is _Complex), and cgo won't export a struct, so under cffi it crosses +// as two floats: as two parameters (_re, _im), and as a result in +// cgo's two-value return, which it exports as a plain C struct {r0; r1;}. +// The methods below say how a value of a given symbol crosses, so that the +// generators only differ from the default backend here. + +// isCFFIComplex reports whether sym crosses as two floats. +func (g *pyGen) isCFFIComplex(sym *symbol) bool { + return g.isCFFI() && isComplexSym(sym) +} + +// cffiComplexFloat returns the cgo and the Go float type of the parts of a +// complex64 or complex128 symbol. +func cffiComplexFloat(sym *symbol) (cfloat, gofloat string) { + if sym.goname == "complex64" { + return "C.float", "float32" + } + return "C.double", "float64" +} + +// cgoParam returns the declaration of the parameter of an exported function +// that carries a value of sym. +func (g *pyGen) cgoParam(name string, sym *symbol) string { + if g.isCFFIComplex(sym) { + cf, _ := cffiComplexFloat(sym) + return fmt.Sprintf("%[1]s_re %[2]s, %[1]s_im %[2]s", name, cf) + } + return name + " " + sym.cgoname +} + +// cgoResult returns the result type of an exported function that returns a +// value of sym. +func (g *pyGen) cgoResult(sym *symbol) string { + if g.isCFFIComplex(sym) { + cf, _ := cffiComplexFloat(sym) + return "(" + cf + ", " + cf + ")" + } + return sym.cgoname +} + +// cpyName returns the type that build.py records for a value of sym. +// wrapper in cffi_build.py expands complex64 and complex128. +func (g *pyGen) cpyName(sym *symbol) string { + if g.isCFFIComplex(sym) { + return sym.goname + } + return sym.cpyname +} + +// goToCgo returns the Go expression that converts expr, a value of sym, to +// what an exported function returns. +func (g *pyGen) goToCgo(sym *symbol, expr string) string { + switch { + case g.isCFFIComplex(sym): + return sym.goname + "GoToPyCFFI(" + expr + ")" + case sym.go2py != "": + return sym.go2py + "(" + expr + ")" + sym.go2pyParenEx + } + return expr +} + +// cgoToGo returns the Go expression that converts the parameter name, as +// declared by cgoParam, to a value of sym. +func (g *pyGen) cgoToGo(sym *symbol, name string) string { + switch { + case g.isCFFIComplex(sym): + return sym.goname + "PyToGoCFFI(" + name + "_re, " + name + "_im)" + case sym.py2go != "": + return sym.py2go + "(" + name + ")" + sym.py2goParenEx + } + return name +} + // same argument positions as goPreamble: 1 = name of package, 2 = cmdstr, // 4 = GoHandle, 5 = CGoHandle, 6 = all imports, 7 = mainstr, 8 = C trampolines for // callbacks (see cffi_callback.go), 10 = gopy version. @@ -196,4 +275,21 @@ func errorGoToPy(e error) *C.char { } return C.CString("") } + +// complex values cross as two floats, see isCFFIComplex in cffi.go +func complex64GoToPyCFFI(c complex64) (C.float, C.float) { + return C.float(real(c)), C.float(imag(c)) +} + +func complex64PyToGoCFFI(re, im C.float) complex64 { + return complex(float32(re), float32(im)) +} + +func complex128GoToPyCFFI(c complex128) (C.double, C.double) { + return C.double(real(c)), C.double(imag(c)) +} + +func complex128PyToGoCFFI(re, im C.double) complex128 { + return complex(float64(re), float64(im)) +} ` diff --git a/bind/cffi_build.py b/bind/cffi_build.py index 36b93ff7..c4ff530b 100644 --- a/bind/cffi_build.py +++ b/bind/cffi_build.py @@ -32,24 +32,15 @@ def add_include(self, inc): self.header = inc.strip('"') def add_function(self, name, ret, params, *a, **kw): - self.funcs.append(("plain", name, ret, params)) - - def add_complex_function(self, name, nargs): - # see genFuncComplexCFFI (gen_func.go) for the calling convention. - self.funcs.append(("complex", name, nargs)) + self.funcs.append((name, ret, params)) def generate(self): here = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(here, self.header)) as f: typedefs, cdefs = go_decls(f.read()) out = [MODULE_HEAD.replace("@CDEFS@", repr("\n".join(typedefs + list(cdefs.values()))))] - for entry in self.funcs: - if entry[0] == "complex": - _, name, nargs = entry - out.append(complex_wrapper(name, nargs, name in cdefs)) - else: - _, name, ret, params = entry - out.append(wrapper(name, ret, params, name in cdefs)) + for name, ret, params in self.funcs: + out.append(wrapper(name, ret, params, name in cdefs)) # Slice_byte's converters exchange a raw pointer+length instead of a # PyObject* (see gen_slice.go); they are recognized by name here # rather than recorded via add_function, since their python bodies @@ -69,8 +60,9 @@ def add_checked_function(mod, name, retval, params, failure_expression="", *a, * def go_decls(header): """Returns the Go typedefs (plus any "struct X { ... };" body -- cgo emits - one ahead of a multi-value-returning export, see genFuncComplexCFFI in - gen_func.go), and {function name: cdef line} for the functions cgo exports. + one ahead of an export that returns two values, which is how a complex + number is returned, see isCFFIComplex in cffi.go), and {function name: + cdef line} for the functions cgo exports. """ typedefs = [] decls = {} @@ -137,6 +129,8 @@ def wrapper(name, ret, params, exported): args.append("_ffi.cast('void*', _cb_%s)" % pname) elif ctype == "char*": args.append("_enc(%s, %d)" % (pname, i + 1)) + elif ctype in ("complex64", "complex128"): + args.append("%s.real, %s.imag" % (pname, pname)) elif ctype == "bool": args.append("(1 if %s else 0)" % pname) elif ctype == "uint8_t": @@ -152,6 +146,8 @@ def wrapper(name, ret, params, exported): body.append(" _r = bool(_r)") elif ret == "uint8_t": body.append(" _r = _r & 0xFF") + elif ret in ("complex64", "complex128"): + body.append(" _r = complex(_r.r0, _r.r1)") body.append(" _check()") if ret is not None: body.append(" return _r") @@ -200,28 +196,6 @@ def cdecl_type(t): ]) -def complex_wrapper(name, nargs, exported): - """A plain function whose every argument and return value is - complex64/128 (see genFuncComplexCFFI, gen_func.go): each argument - crosses as two floats (.real, .imag), and the return value comes back - as the {r0, r1} struct cgo generates for a two-value Go return. - """ - if not exported: - return ( - "def %s(*args):\n" - " raise NotImplementedError('%s is not available with the cffi backend')\n" % (name, name) - ) - names = ["c%d" % i for i in range(nargs)] - sig = ", ".join(names) - callargs = ", ".join("%s.real, %s.imag" % (n, n) for n in names) - return ( - "def %s(%s):\n" - " _r = _lib.%s(%s)\n" - " _check()\n" - " return complex(_r.r0, _r.r1)\n" % (name, sig, name, callargs) - ) - - MODULE_HEAD = '''# python bindings for package @NAME@ using cffi. # File is generated by gopy version @VERSION@. Do not edit. import builtins diff --git a/bind/gen_func.go b/bind/gen_func.go index 0b8b7548..d46e3fc9 100644 --- a/bind/gen_func.go +++ b/bind/gen_func.go @@ -232,22 +232,14 @@ func (g *pyGen) genFunc(o *Func) { } } -func isComplexSym(sym *symbol) bool { - return sym != nil && (sym.goname == "complex64" || sym.goname == "complex128") -} - // genFuncComplexCFFI generates a plain (non-method) function whose every -// argument and its one return value are complex64/128. The normal path -// (genFuncSig/genFuncBody) represents complex64/128 as a *C.PyObject, which -// the cffi preamble doesn't declare -- fine for most types, but complex -// values need actual marshaling here, not just a skip. Instead, an -// argument crosses as two plain floats (real, imag), and the return value -// uses Go's native multi-value return, which cgo exports as a small C -// struct {r0; r1;} that cffi can declare (see go_decls/complex_wrapper in -// cffi_build.py). It returns false, writing nothing, if the signature -// doesn't fit that narrow shape (methods, a mix of complex and other -// argument types, or an error return) -- genFuncSig's PyObject* check then -// skips the function instead of emitting code that fails to compile. +// argument and its one return value are complex64/128, which the normal path +// (genFuncSig/genFuncBody) can't do under cffi, where a complex value crosses +// as two floats (see isCFFIComplex in cffi.go). It returns false, writing +// nothing, if the signature doesn't fit that narrow shape (methods, a mix of +// complex and other argument types, or an error return): genFuncSig's +// PyObject* check then skips the function instead of emitting code that +// fails to compile. func (g *pyGen) genFuncComplexCFFI(fsym *Func) bool { sig := fsym.sig if sig == nil || fsym.isVariadic || fsym.err { @@ -273,34 +265,25 @@ func (g *pyGen) genFuncComplexCFFI(fsym *Func) bool { return false } - cfloatOf := func(sym *symbol) (cgo, gotyp string) { - if sym.goname == "complex64" { - return "C.float", "float32" - } - return "C.double", "float64" - } - - retFloat, _ := cfloatOf(current.symtype(res[0].GoType())) - - var goArgs, callArgs, wpArgs []string + ret := current.symtype(res[0].GoType()) + var goArgs, pyArgs, callArgs, wpArgs []string for i, arg := range args { anm := pySafeArg(arg.Name(), i) - cfloat, gofloat := cfloatOf(current.symtype(arg.GoType())) - reNm, imNm := anm+"_re", anm+"_im" - goArgs = append(goArgs, fmt.Sprintf("%s %s, %s %s", reNm, cfloat, imNm, cfloat)) - callArgs = append(callArgs, fmt.Sprintf("complex(%s(%s), %s(%s))", gofloat, reNm, gofloat, imNm)) + sarg := current.symtype(arg.GoType()) + goArgs = append(goArgs, g.cgoParam(anm, sarg)) + pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", g.cpyName(sarg), anm)) + callArgs = append(callArgs, g.cgoToGo(sarg, anm)) wpArgs = append(wpArgs, anm) } g.gofile.Printf("\n//export %s\n", fsym.ID()) - g.gofile.Printf("func %s(%s) (%s, %s) {\n", fsym.ID(), strings.Join(goArgs, ", "), retFloat, retFloat) + g.gofile.Printf("func %s(%s) %s {\n", fsym.ID(), strings.Join(goArgs, ", "), g.cgoResult(ret)) g.gofile.Indent() - g.gofile.Printf("_r := %s(%s)\n", fsym.GoFmt(), strings.Join(callArgs, ", ")) - g.gofile.Printf("return %s(real(_r)), %s(imag(_r))\n", retFloat, retFloat) + g.gofile.Printf("return %s\n", g.goToCgo(ret, fmt.Sprintf("%s(%s)", fsym.GoFmt(), strings.Join(callArgs, ", ")))) g.gofile.Outdent() g.gofile.Printf("}\n\n") - g.pybuild.Printf("mod.add_complex_function('%s', %d)\n", fsym.ID(), len(args)) + g.pybuild.Printf("mod.add_function('%s', retval('%s'), [%s])\n", fsym.ID(), g.cpyName(ret), strings.Join(pyArgs, ", ")) g.pywrap.Printf("def %s(%s):\n", gname, strings.Join(wpArgs, ", ")) g.pywrap.Indent() diff --git a/bind/gen_slice.go b/bind/gen_slice.go index 7f277e7c..96f05b42 100644 --- a/bind/gen_slice.go +++ b/bind/gen_slice.go @@ -74,7 +74,7 @@ func (g *pyGen) genSliceInit(slc *symbol, extTypes, pyWrapOnly bool, slob *Slice // which cffi's preamble doesn't declare (see genFuncSig for the same // restriction on plain function args/returns); skip the whole wrapper // rather than emit code that fails to compile. - if g.isCFFI() && esym != nil && esym.cpyname == "PyObject*" { + if g.isCFFI() && esym != nil && esym.cpyname == "PyObject*" && !isComplexSym(esym) { return } @@ -329,21 +329,15 @@ otherwise parameter is a python list that we copy from g.pybuild.Printf("mod.add_function('%s_len', retval('int'), [param('%s', 'handle')])\n", slNm, PyHandle) g.gofile.Printf("//export %s_elem\n", slNm) - g.gofile.Printf("func %s_elem(handle CGoHandle, _idx int) %s {\n", slNm, esym.cgoname) + g.gofile.Printf("func %s_elem(handle CGoHandle, _idx int) %s {\n", slNm, g.cgoResult(esym)) g.gofile.Indent() g.gofile.Printf("s := deptrFromHandle_%s(handle)\n", slNm) - if esym.go2py != "" { - // If the go2py starts with handleFromPtr_, use reference &, otherwise just return the value - val_str := "" - if strings.HasPrefix(esym.go2py, "handleFromPtr_") { - val_str = "&(s[_idx])" - } else { - val_str = "s[_idx]" - } - g.gofile.Printf("return %s(%s)%s\n", esym.go2py, val_str, esym.go2pyParenEx) - } else { - g.gofile.Printf("return s[_idx]\n") + // If the go2py starts with handleFromPtr_, use reference &, otherwise just return the value + val_str := "s[_idx]" + if strings.HasPrefix(esym.go2py, "handleFromPtr_") { + val_str = "&(s[_idx])" } + g.gofile.Printf("return %s\n", g.goToCgo(esym, val_str)) g.gofile.Outdent() g.gofile.Printf("}\n\n") @@ -356,7 +350,7 @@ otherwise parameter is a python list that we copy from if esym.cpyname == "char*" { g.pybuild.Printf("add_checked_string_function(mod, '%s_elem', retval('%s'), [param('%s', 'handle'), param('int', 'idx')])\n", slNm, esym.cpyname, PyHandle) } else { - g.pybuild.Printf("mod.add_function('%s_elem', retval('%s'%s), [param('%s', 'handle'), param('int', 'idx')])\n", slNm, esym.cpyname, caller_owns_ret, PyHandle) + g.pybuild.Printf("mod.add_function('%s_elem', retval('%s'%s), [param('%s', 'handle'), param('int', 'idx')])\n", slNm, g.cpyName(esym), caller_owns_ret, PyHandle) } if slc.isSlice() { @@ -373,33 +367,25 @@ otherwise parameter is a python list that we copy from } g.gofile.Printf("//export %s_set\n", slNm) - g.gofile.Printf("func %s_set(handle CGoHandle, _idx int, _vl %s) {\n", slNm, esym.cgoname) + g.gofile.Printf("func %s_set(handle CGoHandle, _idx int, %s) {\n", slNm, g.cgoParam("_vl", esym)) g.gofile.Indent() g.gofile.Printf("s := deptrFromHandle_%s(handle)\n", slNm) - if esym.py2go != "" { - g.gofile.Printf("s[_idx] = %s(_vl)%s\n", esym.py2go, esym.py2goParenEx) - } else { - g.gofile.Printf("s[_idx] = _vl\n") - } + g.gofile.Printf("s[_idx] = %s\n", g.cgoToGo(esym, "_vl")) g.gofile.Outdent() g.gofile.Printf("}\n\n") - g.pybuild.Printf("mod.add_function('%s_set', None, [param('%s', 'handle'), param('int', 'idx'), param('%v', 'value'%s)])\n", slNm, PyHandle, esym.cpyname, transfer_ownership) + g.pybuild.Printf("mod.add_function('%s_set', None, [param('%s', 'handle'), param('int', 'idx'), param('%v', 'value'%s)])\n", slNm, PyHandle, g.cpyName(esym), transfer_ownership) if slc.isSlice() { g.gofile.Printf("//export %s_append\n", slNm) - g.gofile.Printf("func %s_append(handle CGoHandle, _vl %s) {\n", slNm, esym.cgoname) + g.gofile.Printf("func %s_append(handle CGoHandle, %s) {\n", slNm, g.cgoParam("_vl", esym)) g.gofile.Indent() g.gofile.Printf("s := ptrFromHandle_%s(handle)\n", slNm) - if esym.py2go != "" { - g.gofile.Printf("*s = append(*s, %s(_vl)%s)\n", esym.py2go, esym.py2goParenEx) - } else { - g.gofile.Printf("*s = append(*s, _vl)\n") - } + g.gofile.Printf("*s = append(*s, %s)\n", g.cgoToGo(esym, "_vl")) g.gofile.Outdent() g.gofile.Printf("}\n\n") - g.pybuild.Printf("mod.add_function('%s_append', None, [param('%s', 'handle'), param('%s', 'value'%s)])\n", slNm, PyHandle, esym.cpyname, transfer_ownership) + g.pybuild.Printf("mod.add_function('%s_append', None, [param('%s', 'handle'), param('%s', 'value'%s)])\n", slNm, PyHandle, g.cpyName(esym), transfer_ownership) } if slNm == "Slice_byte" { From 363ee92e68bf4c454b3dad202149e8b4067c184c Mon Sep 17 00:00:00 2001 From: b-long Date: Wed, 23 Sep 2026 20:17:36 -0400 Subject: [PATCH 10/21] Isolate cffi errors per OS thread --- _examples/pyerrors/test.py | 39 +++++++++++++++++++++++++++++++++++ bind/cffi.go | 42 ++++++++++++++++++++------------------ 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/_examples/pyerrors/test.py b/_examples/pyerrors/test.py index c729b118..de1a67f2 100644 --- a/_examples/pyerrors/test.py +++ b/_examples/pyerrors/test.py @@ -5,6 +5,8 @@ ## py2/py3 compat from __future__ import print_function +import threading + import pyerrors def div(a, b): @@ -30,3 +32,40 @@ def new_mystring(s): new_mystring("hello") print("OK") + + +# Concurrent calls must not swap or drop errors between Python threads: each +# thread's exception (or lack of one) must match what it, specifically, did. +def race(): + n = 200 + bad = [] + lock = threading.Lock() + + def worker(i): + try: + if i % 2 == 0: + pyerrors.Div(10, 0) + with lock: + bad.append((i, "missing exception")) + else: + r = pyerrors.Div(10, 1) + if r != 10: + with lock: + bad.append((i, "wrong result: %r" % r)) + except Exception as e: + if i % 2 == 1: + with lock: + bad.append((i, "unexpected exception: %s" % e)) + elif str(e) != "Divide by zero.": + with lock: + bad.append((i, "wrong message: %s" % e)) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(n)] + for t in threads: + t.start() + for t in threads: + t.join() + assert not bad, bad + + +race() diff --git a/bind/cffi.go b/bind/cffi.go index 2798babb..52284997 100644 --- a/bind/cffi.go +++ b/bind/cffi.go @@ -133,6 +133,21 @@ package main #if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 202311L) typedef uint8_t bool; #endif +#if defined(_MSC_VER) +#define GOPY_TLS __declspec(thread) +#else +#define GOPY_TLS _Thread_local +#endif +static GOPY_TLS char* gopy_err_msg; +static inline void gopy_set_err(char* msg) { + free(gopy_err_msg); + gopy_err_msg = msg; +} +static inline char* gopy_take_err(void) { + char* msg = gopy_err_msg; + gopy_err_msg = NULL; + return msg; +} %[8]s */ import "C" @@ -195,32 +210,19 @@ func init() { }() } -// The error of the last call, for the python side to raise. There is one -// slot for the whole process, so calls from several python threads at once -// can be given each other's errors. -var ( - gopyErrMu sync.Mutex - gopyErrMsg string - gopyHasErr bool -) - +// The error of the last call on the calling OS thread, for the python side +// to raise; gopy_err_msg is thread-local so concurrent calls on other +// threads don't see it. func gopySetError(kind, msg string) { - gopyErrMu.Lock() - gopyErrMsg, gopyHasErr = kind+":"+msg, true - gopyErrMu.Unlock() + C.gopy_set_err(C.CString(kind + ":" + msg)) } // GopyTakeError returns "Kind:message" for the error recorded by the last -// call and clears it, or NULL if there is none. Free with GopyFreeString. +// call on the calling thread and clears it, or NULL if there is none. +// Free with GopyFreeString. //export GopyTakeError func GopyTakeError() *C.char { - gopyErrMu.Lock() - defer gopyErrMu.Unlock() - if !gopyHasErr { - return nil - } - gopyHasErr = false - return C.CString(gopyErrMsg) + return C.gopy_take_err() } // GopyFreeString frees a string returned by this library. From af1fe2d58c078e28191ac9d244b15eebbc4b590a Mon Sep 17 00:00:00 2001 From: b-long Date: Wed, 23 Sep 2026 20:31:44 -0400 Subject: [PATCH 11/21] Fix cffi DLL load failure on Windows --- bind/cffi.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/bind/cffi.go b/bind/cffi.go index 52284997..b3cb09af 100644 --- a/bind/cffi.go +++ b/bind/cffi.go @@ -133,19 +133,31 @@ package main #if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 202311L) typedef uint8_t bool; #endif -#if defined(_MSC_VER) -#define GOPY_TLS __declspec(thread) +// Thread-local storage for the last error, read by the python side right +// after the call that set it, on the same OS thread. Windows gets its own +// branch: mingw's compiler-provided thread-local storage (_Thread_local) +// needs TLS support that a DLL loaded after process start (as cffi and +// ctypes do) isn't guaranteed to have, so this uses the OS API instead, +// which carries no such restriction (see TlsAlloc/TlsGetValue/TlsSetValue). +#ifdef _WIN32 +#include +static DWORD gopy_err_tls = TLS_OUT_OF_INDEXES; +static inline void gopy_err_tls_init(void) { gopy_err_tls = TlsAlloc(); } +static inline char* gopy_err_get(void) { return (char*)TlsGetValue(gopy_err_tls); } +static inline void gopy_err_put(char* msg) { TlsSetValue(gopy_err_tls, msg); } #else -#define GOPY_TLS _Thread_local +static inline void gopy_err_tls_init(void) {} +static _Thread_local char* gopy_err_msg; +static inline char* gopy_err_get(void) { return gopy_err_msg; } +static inline void gopy_err_put(char* msg) { gopy_err_msg = msg; } #endif -static GOPY_TLS char* gopy_err_msg; static inline void gopy_set_err(char* msg) { - free(gopy_err_msg); - gopy_err_msg = msg; + free(gopy_err_get()); + gopy_err_put(msg); } static inline char* gopy_take_err(void) { - char* msg = gopy_err_msg; - gopy_err_msg = NULL; + char* msg = gopy_err_get(); + gopy_err_put(NULL); return msg; } %[8]s @@ -202,6 +214,7 @@ func RequestGC() { var _gcReq = make(chan chan struct{}) func init() { + C.gopy_err_tls_init() go func() { for done := range _gcReq { runtime.GC() From 6454e067d2398bc85b2028dcdc4223db0d2565d3 Mon Sep 17 00:00:00 2001 From: b-long Date: Thu, 24 Sep 2026 21:25:25 -0400 Subject: [PATCH 12/21] Generalize cffi impl, start pybind11 GOPY_BACKEND --- bind/backend.go | 2 +- bind/backend_test.go | 2 +- bind/cffi.go | 29 ++++++++- bind/gen.go | 22 +++++-- bind/gen_func.go | 19 +++--- bind/gen_map.go | 2 +- bind/gen_slice.go | 4 +- bind/pybind11.go | 32 +++++++++ bind/pybind11_build.py | 145 +++++++++++++++++++++++++++++++++++++++++ cmd_build.go | 104 +++++++++++++++++++++++++++++ gen.go | 4 +- 11 files changed, 339 insertions(+), 26 deletions(-) create mode 100644 bind/pybind11.go create mode 100644 bind/pybind11_build.py diff --git a/bind/backend.go b/bind/backend.go index e19a6d9c..4632f301 100644 --- a/bind/backend.go +++ b/bind/backend.go @@ -33,7 +33,7 @@ var backends = []struct { }{ {BackendPyBindGen, true}, {BackendCFFI, true}, - {BackendPyBind11, false}, + {BackendPyBind11, true}, {BackendNanobind, false}, {BackendCAPI, false}, {BackendCGO, false}, diff --git a/bind/backend_test.go b/bind/backend_test.go index cbbe40ac..e17f4857 100644 --- a/bind/backend_test.go +++ b/bind/backend_test.go @@ -19,7 +19,7 @@ func TestParseBackend(t *testing.T) { {in: "pybindgen", want: BackendPyBindGen}, {in: " PyBindGen ", want: BackendPyBindGen}, {in: "cffi", want: BackendCFFI}, - {in: "pybind11", errPart: "not implemented yet"}, + {in: "pybind11", want: BackendPyBind11}, {in: "bogus", errPart: "unknown GOPY_BACKEND"}, } { got, err := parseBackend(tc.in) diff --git a/bind/cffi.go b/bind/cffi.go index b3cb09af..0f972a5b 100644 --- a/bind/cffi.go +++ b/bind/cffi.go @@ -22,6 +22,22 @@ func (g *pyGen) isCFFI() bool { return g.cfg.Backend == BackendCFFI } +func (g *pyGen) isPyBind11() bool { + return g.cfg.Backend == BackendPyBind11 +} + +// noAPIShim reports whether the exported Go functions must avoid the +// CPython C API: true for every backend whose python-facing wrapper is +// produced from outside the Go build (cffi loads a plain shared library; +// pybind11 compiles a C++ file against the same one). Both share the shim +// this decides the shape of -- see cffi_callback.go and the complex64/128 +// and byte-slice handling below and in gen_slice.go -- and each then has its +// own way of consuming it (cffiBuildPreamble here, pybind11BuildPreamble in +// pybind11.go). +func (g *pyGen) noAPIShim() bool { + return g.isCFFI() || g.isPyBind11() +} + // cffiBuildPreamble returns the start of build.py: the cffi recorder. func (g *pyGen) cffiBuildPreamble() string { return strings.NewReplacer( @@ -120,7 +136,7 @@ func (g *pyGen) cgoToGo(sym *symbol, name string) string { // 4 = GoHandle, 5 = CGoHandle, 6 = all imports, 7 = mainstr, 8 = C trampolines for // callbacks (see cffi_callback.go), 10 = gopy version. const goPreambleCFFI = `/* -cgo stubs for package %[1]s, for use with cffi. +cgo stubs for package %[1]s, for use with cffi or pybind11. File is generated by gopy version %[10]s. Do not edit. %[2]s */ @@ -130,9 +146,16 @@ package main /* #include #include -#if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 202311L) +// pybind11's consumer #includes this same preamble (via the generated +// header) as C++, where bool/thread_local are keywords, not the C ones below. +#if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || (__STDC_VERSION__ < 202311L)) typedef uint8_t bool; #endif +#if defined(__cplusplus) +#define GOPY_THREAD_LOCAL thread_local +#else +#define GOPY_THREAD_LOCAL _Thread_local +#endif // Thread-local storage for the last error, read by the python side right // after the call that set it, on the same OS thread. Windows gets its own // branch: mingw's compiler-provided thread-local storage (_Thread_local) @@ -147,7 +170,7 @@ static inline char* gopy_err_get(void) { return (char*)TlsGetValue(gopy_err_tls) static inline void gopy_err_put(char* msg) { TlsSetValue(gopy_err_tls, msg); } #else static inline void gopy_err_tls_init(void) {} -static _Thread_local char* gopy_err_msg; +static GOPY_THREAD_LOCAL char* gopy_err_msg; static inline char* gopy_err_get(void) { return gopy_err_msg; } static inline void gopy_err_put(char* msg) { gopy_err_msg = msg; } #endif diff --git a/bind/gen.go b/bind/gen.go index 52f694af..1264e2af 100644 --- a/bind/gen.go +++ b/bind/gen.go @@ -665,9 +665,10 @@ func (g *pyGen) genPrintOut(outfn string, pr *printer) { } func (g *pyGen) genOut() { - if g.isCFFI() { + switch { + case g.noAPIShim(): g.pybuild.Printf("\nmod.generate()\n\n") - } else { + default: g.pybuild.Printf("\nmod.generate(open('%v.c', 'w'))\n\n", g.cfg.Name) } g.gofile.Printf("\n\n") @@ -731,9 +732,13 @@ func (g *pyGen) genGoPreamble() { pkgimport += fmt.Sprintf("\n\t%q", pp) } } - if g.isCFFI() { + if g.noAPIShim() { + trampolines := "" + if g.isCFFI() { + trampolines = cffiTrampolinesKey + } g.gofile.Printf(goPreambleCFFI, g.cfg.Name, g.cfg.Cmd, "", GoHandle, CGoHandle, - pkgimport, g.cfg.Main, cffiTrampolinesKey, "", g.cfg.Version) + pkgimport, g.cfg.Main, trampolines, "", g.cfg.Version) g.gofile.Printf("\n// --- generated code for package: %[1]s below: ---\n\n", g.cfg.Name) return } @@ -773,11 +778,14 @@ func (g *pyGen) genGoPreamble() { } func (g *pyGen) genPyBuildPreamble() { - if g.isCFFI() { + switch { + case g.isCFFI(): g.pybuild.Printf("%s", g.cffiBuildPreamble()) - return + case g.isPyBind11(): + g.pybuild.Printf("%s", g.pybind11BuildPreamble()) + default: + g.pybuild.Printf(PyBuildPreamble, g.cfg.Name, g.cfg.Cmd, g.cfg.Version) } - g.pybuild.Printf(PyBuildPreamble, g.cfg.Name, g.cfg.Cmd, g.cfg.Version) } func (g *pyGen) genPyWrapPreamble() { diff --git a/bind/gen_func.go b/bind/gen_func.go index d46e3fc9..083874bb 100644 --- a/bind/gen_func.go +++ b/bind/gen_func.go @@ -67,11 +67,12 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { return false } - // cffi has no way to cross a raw PyObject* (complex64/128, and callback - // arguments of a type cffiCallback doesn't support): skip these functions - // rather than emit a signature that references the CPython C API, which - // would fail to even compile under the cffi preamble. - if g.isCFFI() { + // Neither no-API backend (cffi, pybind11) can cross a raw PyObject* + // (complex64/128, and -- for cffi, which is all that supports callbacks + // so far -- a callback argument of a type cffiCallback doesn't handle): + // skip these functions rather than emit a signature referencing the + // CPython C API, which would fail to even compile under their preamble. + if g.noAPIShim() { for _, arg := range args { sarg := current.symtype(arg.GoType()) switch { @@ -358,8 +359,8 @@ func (g *pyGen) genFuncBody(sym *symbol, fsym *Func) { } } - // cffi releases the GIL itself around every call - if !g.isCFFI() { + // cffi and pybind11 each release the GIL themselves around every call + if !g.noAPIShim() { g.gofile.Printf("_saved_thread := C.PyEval_SaveThread()\n") if !rvIsErr && nres != 2 { g.gofile.Printf("defer C.PyEval_RestoreThread(_saved_thread)\n") @@ -526,14 +527,14 @@ if __err != nil { if rvIsErr || nres == 2 { g.gofile.Printf("\n") - if !g.isCFFI() { + if !g.noAPIShim() { g.gofile.Printf("C.PyEval_RestoreThread(_saved_thread)\n") } g.gofile.Printf("if __err != nil {\n") g.gofile.Indent() g.gofile.Printf("estr := C.CString(__err.Error())\n") - if g.isCFFI() { + if g.noAPIShim() { g.gofile.Printf("%s", g.goSetError("RuntimeError", "__err.Error()")) } else { g.gofile.Printf("C.PyErr_SetString(C.PyExc_RuntimeError, estr)\n") diff --git a/bind/gen_map.go b/bind/gen_map.go index d0d19c20..5da36a71 100644 --- a/bind/gen_map.go +++ b/bind/gen_map.go @@ -299,7 +299,7 @@ otherwise parameter is a python list that we copy from } g.gofile.Printf("if !ok {\n") g.gofile.Indent() - if g.isCFFI() { + if g.noAPIShim() { g.gofile.Printf("%s", g.goSetError("KeyError", `"key not in map"`)) } else { g.gofile.Printf("C.PyErr_SetString(C.PyExc_KeyError, C.CString(\"key not in map\"))\n") diff --git a/bind/gen_slice.go b/bind/gen_slice.go index 96f05b42..74309a2a 100644 --- a/bind/gen_slice.go +++ b/bind/gen_slice.go @@ -74,7 +74,7 @@ func (g *pyGen) genSliceInit(slc *symbol, extTypes, pyWrapOnly bool, slob *Slice // which cffi's preamble doesn't declare (see genFuncSig for the same // restriction on plain function args/returns); skip the whole wrapper // rather than emit code that fails to compile. - if g.isCFFI() && esym != nil && esym.cpyname == "PyObject*" && !isComplexSym(esym) { + if g.noAPIShim() && esym != nil && esym.cpyname == "PyObject*" && !g.isCFFIComplex(esym) { return } @@ -389,7 +389,7 @@ otherwise parameter is a python list that we copy from } if slNm == "Slice_byte" { - if g.isCFFI() { + if g.noAPIShim() { // PyBytes_* is off-limits for cffi (no CPython headers), so these // exchange a raw pointer+length instead of a PyObject*; the cffi // build script (cffi_build.py) recognizes them by name and writes diff --git a/bind/pybind11.go b/bind/pybind11.go new file mode 100644 index 00000000..67256687 --- /dev/null +++ b/bind/pybind11.go @@ -0,0 +1,32 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + _ "embed" + "strings" +) + +// The pybind11 backend (GOPY_BACKEND=pybind11) shares its cgo shim with cffi +// (see noAPIShim in cffi.go): the same plain-C exported functions, the same +// gopySetError/GopyTakeError error channel, the same byte-slice and callback +// conventions. What differs is the consumer: instead of a pure-Python module +// that loads the shim at runtime with ctypes, pybind11Build.py writes a C++ +// file that #includes the shim's own header and compiles against it directly, +// so (unlike cffi_build.py) it needs no runtime description of the C types -- +// the C++ compiler gets them from the header, the same way pybindgen's +// generated .c file does. + +//go:embed pybind11_build.py +var pybind11BuildPy string + +// pybind11BuildPreamble returns the start of build.py: the pybind11 recorder. +func (g *pyGen) pybind11BuildPreamble() string { + return strings.NewReplacer( + "@NAME@", g.cfg.Name, + "@CMD@", g.cfg.Cmd, + "@VERSION@", g.cfg.Version, + ).Replace(pybind11BuildPy) +} diff --git a/bind/pybind11_build.py b/bind/pybind11_build.py new file mode 100644 index 00000000..bc067320 --- /dev/null +++ b/bind/pybind11_build.py @@ -0,0 +1,145 @@ +# python build stub for package @NAME@ (pybind11 backend) +# File is generated by gopy version @VERSION@. Do not edit. +# @CMD@ +# +# The generated build code below is written against the pybindgen API. Here +# the same calls are only recorded, and Module.generate() then writes +# @NAME@.cpp: a pybind11 module that #includes @NAME@_go.h (cgo's own header) +# and calls the shim functions directly, so -- unlike the cffi backend -- +# their C types come from the C++ compiler, not from parsing the header here. + + +def retval(ctype, *a, **kw): + return ctype + + +def param(ctype, name, *a, **kw): + return (ctype, name) + + +class Module(object): + def __init__(self, name, cpp_name): + self.name = name # the compiled extension's import name, e.g. "_hi" + self.cpp_name = cpp_name # source file to write, e.g. "hi.cpp" + self.header = None + self.funcs = [] + + def add_include(self, inc): + self.header = inc.strip('"') + + def add_function(self, name, ret, params, *a, **kw): + self.funcs.append((name, ret, params)) + + def generate(self): + import os + + here = os.path.dirname(os.path.abspath(__file__)) + defs = [d for d in (wrapper(name, ret, params) for name, ret, params in self.funcs) if d] + cpp = MODULE_TEMPLATE.replace("@HEADER@", self.header).replace( + "@DEFS@", "\n".join(defs) + ) + with open(os.path.join(here, self.cpp_name), "w") as f: + f.write(cpp) + + +def add_checked_function(mod, name, retval, params, failure_expression="", *a, **kw): + mod.add_function(name, retval, params) + + +add_checked_string_function = add_checked_function + + +def wrapper(name, ret, params): + """Returns the m.def(...) call binding name, or "" if its signature + isn't supported yet (a PyObject*, i.e. complex64/128 or a callback + argument): the .cpp simply never binds it, so calling it from python + raises AttributeError instead of NotImplementedError -- close enough for + a function nothing in gopy's own generated wrapper calls unconditionally. + """ + if ret == "PyObject*" or any(p[0] == "PyObject*" for p in params): + return "" + args = [] + call_args = [] + for ctype, pname in params: + if ctype == "char*": + args.append("const std::string& " + pname) + call_args.append("const_cast(%s.c_str())" % pname) + elif ctype == "bool": + args.append("bool " + pname) + call_args.append("(char)(%s ? 1 : 0)" % pname) + else: + args.append(ctype + " " + pname) + call_args.append(pname) + call = "%s(%s)" % (name, ", ".join(call_args)) + if ret is None: + body, cpptype = "%s;\n _check();" % call, "void" + elif ret == "char*": + body = ( + "char* _r = %s;\n" + " std::string _s(_r ? _r : \"\");\n" + " free(_r);\n" + " _check();\n" + " return _s;" % call + ) + cpptype = "std::string" + elif ret == "bool": + body = "auto _r = %s;\n _check();\n return _r != 0;" % call + cpptype = "bool" + else: + body = "auto _r = %s;\n _check();\n return _r;" % call + cpptype = ret + return ' m.def("%s", [](%s) -> %s {\n %s\n });' % ( + name, + ", ".join(args), + cpptype, + body, + ) + + +MODULE_TEMPLATE = '''// python bindings for package @NAME@ using pybind11. +// File is generated by gopy version @VERSION@. Do not edit. +#include +#include + +#include + +namespace py = pybind11; + +extern "C" { +#include "@HEADER@" +} + +// Raises the exception, if any, that the last Go call on this thread +// recorded (see gopySetError/GopyTakeError, shared with the cffi backend). +static inline void _check() { + char* e = GopyTakeError(); + if (!e) { + return; + } + std::string s(e); + free(e); + auto sep = s.find(':'); + std::string kind = sep == std::string::npos ? s : s.substr(0, sep); + std::string msg = sep == std::string::npos ? "" : s.substr(sep + 1); + PyObject* exc = PyExc_RuntimeError; + if (kind == "ValueError") exc = PyExc_ValueError; + else if (kind == "TypeError") exc = PyExc_TypeError; + else if (kind == "KeyError") exc = PyExc_KeyError; + else if (kind == "IndexError") exc = PyExc_IndexError; + else if (kind == "AttributeError") exc = PyExc_AttributeError; + PyErr_SetString(exc, msg.c_str()); + throw py::error_already_set(); +} + +PYBIND11_MODULE(_@NAME@, m) { +@DEFS@ +} +''' + +mod = Module('_@NAME@', '@NAME@.cpp') +mod.add_include('"@NAME@_go.h"') +mod.add_function('GoPyInit', None, []) +mod.add_function('DecRef', None, [param('int64_t', 'handle')]) +mod.add_function('IncRef', None, [param('int64_t', 'handle')]) +mod.add_function('NumHandles', retval('int'), []) +mod.add_function('RequestGC', None, []) diff --git a/cmd_build.go b/cmd_build.go index 4a541134..291992fd 100644 --- a/cmd_build.go +++ b/cmd_build.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "runtime" "strings" @@ -128,6 +129,13 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { } pycfg, err := bind.GetPythonConfig(cfg.VM) + if err != nil { + return err + } + + if cfg.Backend == bind.BackendPyBind11 { + return buildPyBind11(cfg, buildname+libExt, pycfg) + } if mode == bind.ModeExe { of, err := os.Create(buildname + ".h") // overwrite existing @@ -364,3 +372,99 @@ func buildCFFI(cfg *BuildCfg, buildLib string) error { } return err } + +// buildPyBind11 builds the cgo shim as a plain shared library (same shape as +// buildCFFI's), runs build.py to write a pybind11 C++ module wrapping it, and +// compiles+links that with a C++ compiler. The current directory is the +// output directory. +func buildPyBind11(cfg *BuildCfg, buildLib string, pycfg bind.PyConfig) error { + args := []string{"build", "-mod=mod", "-buildmode=c-shared"} + if cfg.BuildTags != "" { + args = append(args, "-tags", cfg.BuildTags) + } + if !cfg.Symbols { + args = append(args, "-ldflags=-s -w") + } + args = append(args, "-o", buildLib, ".") + fmt.Printf("go %v\n", strings.Join(args, " ")) + cmdout, err := exec.Command("go", args...).CombinedOutput() + if err != nil { + fmt.Printf("cmd had error: %v output:\n%v\n", err, string(cmdout)) + return err + } + + fmt.Printf("%v build.py\n", cfg.VM) + cmdout, err = exec.Command(cfg.VM, "build.py").CombinedOutput() + if err != nil { + fmt.Printf("cmd had error: %v output:\n%v\n", err, string(cmdout)) + return err + } + + cmdout, err = exec.Command(cfg.VM, "-m", "pybind11", "--includes").CombinedOutput() + if err != nil { + fmt.Printf("cmd had error: %v output:\n%v\n(is pybind11 installed? pip install pybind11)\n", err, string(cmdout)) + return err + } + pyinc := strings.Fields(strings.TrimSpace(string(cmdout))) + + extext := libExt + if runtime.GOOS == "windows" { + extext = ".pyd" + } + if pycfg.ExtSuffix != "" { + extext = pycfg.ExtSuffix + } + modlib := "_" + cfg.Name + extext + + cxx := os.Getenv("CXX") + if cxx == "" { + cxx = "c++" + } + // pycfg.CFlags/LdFlags quote each path (for the shell that CGO_CFLAGS/ + // CGO_LDFLAGS normally go through); exec.Command runs the compiler + // directly, with no shell to strip those, so unquote each field here. + unquote := func(fields []string) []string { + o := make([]string, len(fields)) + for i, f := range fields { + o[i] = strings.Trim(f, `"`) + } + return o + } + // modlib depends on buildLib (alongside it) and libpython (wherever this + // VM's own one lives, e.g. not on the loader's default search path for a + // uv- or pyenv-managed Python); without an rpath for each, the loader + // only finds them if they happen to already be on its search path. + var libdir string + if m := regexp.MustCompile(`-L(\S+)`).FindStringSubmatch(pycfg.LdFlags); m != nil { + libdir = strings.Trim(m[1], `"`) + } + cxxArgs := []string{"-std=c++17", "-fPIC", "-shared", "-O2"} + switch runtime.GOOS { + case "darwin": + cxxArgs = append(cxxArgs, "-Wl,-rpath,@loader_path") + if libdir != "" { + cxxArgs = append(cxxArgs, "-Wl,-rpath,"+libdir) + } + case "windows": + // TODO: no rpath equivalent, and unlike buildCFFI (which loads + // buildLib explicitly, with a known path) modlib depends on it + // implicitly; Windows may not find it unless the output directory + // is already on PATH. Unverified -- no Windows environment to test. + default: + cxxArgs = append(cxxArgs, "-Wl,-rpath,$ORIGIN") + if libdir != "" { + cxxArgs = append(cxxArgs, "-Wl,-rpath,"+libdir) + } + } + cxxArgs = append(cxxArgs, pyinc...) + cxxArgs = append(cxxArgs, unquote(strings.Fields(pycfg.CFlags))...) + cxxArgs = append(cxxArgs, cfg.Name+".cpp", buildLib) + cxxArgs = append(cxxArgs, unquote(strings.Fields(pycfg.LdFlags))...) + cxxArgs = append(cxxArgs, "-o", modlib) + fmt.Printf("%v %v\n", cxx, strings.Join(cxxArgs, " ")) + cmdout, err = exec.Command(cxx, cxxArgs...).CombinedOutput() + if err != nil { + fmt.Printf("cmd had error: %v output:\n%v\n", err, string(cmdout)) + } + return err +} diff --git a/gen.go b/gen.go index d0cd0419..4b41eabe 100644 --- a/gen.go +++ b/gen.go @@ -66,10 +66,10 @@ func genPkg(mode bind.BuildMode, cfg *BuildCfg) error { if cfg.Backend, err = bind.BackendFromEnv(); err != nil { return err } - if cfg.Backend == bind.BackendCFFI && mode == bind.ModeExe { + if (cfg.Backend == bind.BackendCFFI || cfg.Backend == bind.BackendPyBind11) && mode == bind.ModeExe { // exe mode embeds the Python interpreter into the Go binary via the // CPython C API (see goExePreambleC/Go in bind/gen.go), unrelated to - // how the bindings themselves are generated; cffi doesn't support it. + // how the bindings themselves are generated; neither backend supports it. return fmt.Errorf("gopy: %s=%s does not support gopy exe", bind.BackendEnvVar, cfg.Backend) } cfg.OutputDir, err = genOutDir(cfg.OutputDir) From d385fff9f652cfbb85aa3a7cb2573e7488ecde85 Mon Sep 17 00:00:00 2001 From: b-long Date: Thu, 24 Sep 2026 21:45:13 -0400 Subject: [PATCH 13/21] Add pybind11 backend to CI matrix --- .github/requirements-pybind11.txt | 5 +++ .github/workflows/ci.yml | 55 +++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 .github/requirements-pybind11.txt diff --git a/.github/requirements-pybind11.txt b/.github/requirements-pybind11.txt new file mode 100644 index 00000000..4656bd43 --- /dev/null +++ b/.github/requirements-pybind11.txt @@ -0,0 +1,5 @@ +# Python packages for the GOPY_BACKEND=pybind11 job in workflows/ci.yml. +# pybindgen is deliberately absent: the pybind11 backend must not need it. +pybind11 +# used by the memory-leak checks on Windows, where the resource module is missing +psutil diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0c7b441..bd3403d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,3 +138,58 @@ jobs: - name: Test run: go test -v ./... + + # Builds and tests the opt-in pybind11 backend (GOPY_BACKEND=pybind11). + # Runs beside the main matrix, on one Go version, and stops at the first + # failure. Shares its cgo shim with cffi (see noAPIShim in bind/cffi.go); + # what's new here is compiling a C++ file against that shim with a real + # C++ compiler, so unlike the cffi job this needs no extra system packages + # (a C++ compiler is already required for CGO_ENABLED=1 on every platform). + pybind11: + name: pybind11 backend (${{ matrix.platform }}, Python ${{ matrix.python-version }}) + strategy: + fail-fast: true + matrix: + platform: [ubuntu-latest, windows-latest, macos-15] + python-version: ['3.11', '3.12'] + runs-on: ${{ matrix.platform }} + env: + GOPY_BACKEND: pybind11 + # CXX defaults to "c++" in buildPyBind11 (cmd_build.go), which isn't a + # recognized command on windows-latest's MinGW toolchain (the same one + # CGO_ENABLED=1 already needs there, so no extra install is needed). + CXX: g++ + PYTHONFAULTHANDLER: 1 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: .github/requirements-pybind11.txt + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.25.x + cache: true + + - name: Install packages + run: | + python -m pip install -r .github/requirements-pybind11.txt + go install golang.org/x/tools/cmd/goimports@v0.29.0 + + - name: Build + run: go build -v ./... + + # The skipped tests use features the pybind11 backend does not support + # yet: Python callbacks (TestBindFuncs, TestBindCallbacks), complex + # numbers (TestBindSimple, TestBuiltinSlices), []byte<->bytes (TestBytes), + # and pybind11's own auto-generated TypeError wording, which differs from + # pybindgen's for one case in TestHi (Person's setters given the wrong + # argument type) -- a message-text difference, not a functional one. + - name: Test + run: go test -v -skip '^(TestHi|TestBytes|TestBindFuncs|TestBindCallbacks|TestBindSimple|TestBuiltinSlices)$' ./... From b7474fd2a67add48afaa8b5ccc16a048cbb274e3 Mon Sep 17 00:00:00 2001 From: b-long Date: Thu, 24 Sep 2026 22:06:10 -0400 Subject: [PATCH 14/21] Fix pybind11 DLL load failure on Windows --- bind/gen.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bind/gen.go b/bind/gen.go index 1264e2af..b4a4afa6 100644 --- a/bind/gen.go +++ b/bind/gen.go @@ -335,6 +335,11 @@ except ImportError: cwd = os.getcwd() currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) os.chdir(currentdir) +# Windows only searches a dependent DLL's own directory for its further +# dependencies (e.g. the pybind11 backend's _%[1]s.pyd needs %[1]s_go.pyd) +# if that directory was added explicitly; harmless, and a no-op elsewhere. +if hasattr(os, 'add_dll_directory'): + os.add_dll_directory(currentdir) # When multiple gopy extensions coexist in one Python process each carries its own # independent Go runtime. Loading each extension without RTLD_GLOBAL below keeps its # Go runtime symbols (including the per-runtime goroutine-pointer TLS slot) local to From 346013804e021b70661339f5f4437d21a0e38106 Mon Sep 17 00:00:00 2001 From: b-long Date: Thu, 24 Sep 2026 22:24:15 -0400 Subject: [PATCH 15/21] fix: static-link libstdc++/libgcc on Windows --- cmd_build.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd_build.go b/cmd_build.go index 291992fd..8bb73b63 100644 --- a/cmd_build.go +++ b/cmd_build.go @@ -446,10 +446,16 @@ func buildPyBind11(cfg *BuildCfg, buildLib string, pycfg bind.PyConfig) error { cxxArgs = append(cxxArgs, "-Wl,-rpath,"+libdir) } case "windows": - // TODO: no rpath equivalent, and unlike buildCFFI (which loads - // buildLib explicitly, with a known path) modlib depends on it - // implicitly; Windows may not find it unless the output directory - // is already on PATH. Unverified -- no Windows environment to test. + // No rpath equivalent, but modlib finding buildLib (in the same + // directory) is handled at import time instead, by the generated + // wrapper's os.add_dll_directory() call (see PyWrapPreamble). + // MinGW's own runtime (libstdc++/libgcc/libwinpthread), which g++ + // links dynamically by default, has no such fix available -- it + // isn't found by name alone unless its directory happens to be on + // PATH -- so link it in statically instead. The C runtime (ucrt) + // stays dynamic, shared with Python's own. + cxxArgs = append(cxxArgs, "-static-libgcc", "-static-libstdc++", + "-Wl,-Bstatic,--whole-archive", "-lwinpthread", "-Wl,--no-whole-archive", "-Wl,-Bdynamic") default: cxxArgs = append(cxxArgs, "-Wl,-rpath,$ORIGIN") if libdir != "" { From 9325cee85abddbe8d22d421d7f8ca56933158ba2 Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 25 Sep 2026 20:00:41 -0400 Subject: [PATCH 16/21] Add []byte support, shrink pybind11 CI skips --- .github/workflows/ci.yml | 10 +++++----- bind/pybind11_build.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd3403d8..72a86204 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -187,9 +187,9 @@ jobs: # The skipped tests use features the pybind11 backend does not support # yet: Python callbacks (TestBindFuncs, TestBindCallbacks), complex - # numbers (TestBindSimple, TestBuiltinSlices), []byte<->bytes (TestBytes), - # and pybind11's own auto-generated TypeError wording, which differs from - # pybindgen's for one case in TestHi (Person's setters given the wrong - # argument type) -- a message-text difference, not a functional one. + # numbers (TestBindSimple, TestBuiltinSlices), and pybind11's own + # auto-generated TypeError wording, which differs from pybindgen's for + # one case in TestHi (Person's setters given the wrong argument type) -- + # a message-text difference, not a functional one. - name: Test - run: go test -v -skip '^(TestHi|TestBytes|TestBindFuncs|TestBindCallbacks|TestBindSimple|TestBuiltinSlices)$' ./... + run: go test -v -skip '^(TestHi|TestBindFuncs|TestBindCallbacks|TestBindSimple|TestBuiltinSlices)$' ./... diff --git a/bind/pybind11_build.py b/bind/pybind11_build.py index bc067320..3daaef37 100644 --- a/bind/pybind11_build.py +++ b/bind/pybind11_build.py @@ -132,6 +132,25 @@ def wrapper(name, ret, params): } PYBIND11_MODULE(_@NAME@, m) { + // gen_slice.go always exports these 4 (under noAPIShim()) for the + // built-in byte slice, regardless of whether the package uses []byte; + // they exchange a raw pointer+length rather than a PyObject*, same as + // the cffi backend, but bound directly here rather than by name-sniffing + // the header (cffi_build.py's BYTES_FUNCS) since nothing here needs to. + m.def("Slice_byte_from_bytes", [](py::bytes b) -> int64_t { + std::string s = b; + return Slice_byte_from_bytes(const_cast(s.data()), (long long)s.size()); + }); + m.def("Slice_byte_to_bytes", [](int64_t handle) -> py::bytes { + long long n = Slice_byte_to_bytes_len(handle); + if (n == 0) { + return py::bytes("", 0); + } + void* ptr = Slice_byte_to_bytes_ptr(handle); + py::bytes result(static_cast(ptr), (size_t)n); + Slice_byte_free_ptr(ptr); + return result; + }); @DEFS@ } ''' From faed63eb553b2cd7221a46f9dfdf2af9012f39cb Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 25 Sep 2026 20:21:17 -0400 Subject: [PATCH 17/21] Raise pybind11 CI test timeout to 30m --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72a86204..c2a04e7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,5 +191,10 @@ jobs: # auto-generated TypeError wording, which differs from pybindgen's for # one case in TestHi (Person's setters given the wrong argument type) -- # a message-text difference, not a functional one. + # + # -timeout raises go test's own default (10m): each test here compiles + # twice (cgo, then a separate C++ step), against cffi's one, and once + # measured at ~18.5s/test average on windows-latest, 32 tests alone used + # 590s of the default budget. - name: Test - run: go test -v -skip '^(TestHi|TestBindFuncs|TestBindCallbacks|TestBindSimple|TestBuiltinSlices)$' ./... + run: go test -v -timeout=30m -skip '^(TestHi|TestBindFuncs|TestBindCallbacks|TestBindSimple|TestBuiltinSlices)$' ./... From 9fb32764ab0e740456ce65d23a97014b71a51e99 Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 25 Sep 2026 20:47:06 -0400 Subject: [PATCH 18/21] Support complex numbers in pybind11 backend --- .github/workflows/ci.yml | 11 ++-- bind/cffi.go | 115 ++------------------------------------ bind/gen_func.go | 4 +- bind/gen_slice.go | 2 +- bind/noapi.go | 116 +++++++++++++++++++++++++++++++++++++++ bind/pybind11_build.py | 22 ++++++-- 6 files changed, 147 insertions(+), 123 deletions(-) create mode 100644 bind/noapi.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2a04e7f..31be9187 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,15 +186,14 @@ jobs: run: go build -v ./... # The skipped tests use features the pybind11 backend does not support - # yet: Python callbacks (TestBindFuncs, TestBindCallbacks), complex - # numbers (TestBindSimple, TestBuiltinSlices), and pybind11's own - # auto-generated TypeError wording, which differs from pybindgen's for - # one case in TestHi (Person's setters given the wrong argument type) -- - # a message-text difference, not a functional one. + # yet: Python callbacks (TestBindFuncs, TestBindCallbacks), and + # pybind11's own auto-generated TypeError wording, which differs from + # pybindgen's for one case in TestHi (Person's setters given the wrong + # argument type) -- a message-text difference, not a functional one. # # -timeout raises go test's own default (10m): each test here compiles # twice (cgo, then a separate C++ step), against cffi's one, and once # measured at ~18.5s/test average on windows-latest, 32 tests alone used # 590s of the default budget. - name: Test - run: go test -v -timeout=30m -skip '^(TestHi|TestBindFuncs|TestBindCallbacks|TestBindSimple|TestBuiltinSlices)$' ./... + run: go test -v -timeout=30m -skip '^(TestHi|TestBindFuncs|TestBindCallbacks)$' ./... diff --git a/bind/cffi.go b/bind/cffi.go index 0f972a5b..14f0b220 100644 --- a/bind/cffi.go +++ b/bind/cffi.go @@ -6,38 +6,17 @@ package bind import ( _ "embed" - "fmt" "strings" ) -// The cffi backend (GOPY_BACKEND=cffi) generates the same cgo shim as the -// default backend, minus every call into the CPython C API: the shim is a -// plain C shared library that Python loads with cffi. Errors are recorded -// for the Python side to raise, instead of being set with PyErr_SetString. +// The cffi backend (GOPY_BACKEND=cffi) uses the shared no-API cgo shim (see +// noapi.go): a plain C shared library that Python loads with cffi. Errors +// are recorded for the Python side to raise, instead of being set with +// PyErr_SetString. //go:embed cffi_build.py var cffiBuildPy string -func (g *pyGen) isCFFI() bool { - return g.cfg.Backend == BackendCFFI -} - -func (g *pyGen) isPyBind11() bool { - return g.cfg.Backend == BackendPyBind11 -} - -// noAPIShim reports whether the exported Go functions must avoid the -// CPython C API: true for every backend whose python-facing wrapper is -// produced from outside the Go build (cffi loads a plain shared library; -// pybind11 compiles a C++ file against the same one). Both share the shim -// this decides the shape of -- see cffi_callback.go and the complex64/128 -// and byte-slice handling below and in gen_slice.go -- and each then has its -// own way of consuming it (cffiBuildPreamble here, pybind11BuildPreamble in -// pybind11.go). -func (g *pyGen) noAPIShim() bool { - return g.isCFFI() || g.isPyBind11() -} - // cffiBuildPreamble returns the start of build.py: the cffi recorder. func (g *pyGen) cffiBuildPreamble() string { return strings.NewReplacer( @@ -48,90 +27,6 @@ func (g *pyGen) cffiBuildPreamble() string { ).Replace(cffiBuildPy) } -// goSetError returns Go code that records an error for Python to raise. -// kind is the name of a Python builtin exception, msg a Go string expression. -func (g *pyGen) goSetError(kind, msg string) string { - return "gopySetError(\"" + kind + "\", " + msg + ")\n" -} - -func isComplexSym(sym *symbol) bool { - return sym != nil && (sym.goname == "complex64" || sym.goname == "complex128") -} - -// A complex64/complex128 value has no single C type that cffi can declare -// (cgo's is _Complex), and cgo won't export a struct, so under cffi it crosses -// as two floats: as two parameters (_re, _im), and as a result in -// cgo's two-value return, which it exports as a plain C struct {r0; r1;}. -// The methods below say how a value of a given symbol crosses, so that the -// generators only differ from the default backend here. - -// isCFFIComplex reports whether sym crosses as two floats. -func (g *pyGen) isCFFIComplex(sym *symbol) bool { - return g.isCFFI() && isComplexSym(sym) -} - -// cffiComplexFloat returns the cgo and the Go float type of the parts of a -// complex64 or complex128 symbol. -func cffiComplexFloat(sym *symbol) (cfloat, gofloat string) { - if sym.goname == "complex64" { - return "C.float", "float32" - } - return "C.double", "float64" -} - -// cgoParam returns the declaration of the parameter of an exported function -// that carries a value of sym. -func (g *pyGen) cgoParam(name string, sym *symbol) string { - if g.isCFFIComplex(sym) { - cf, _ := cffiComplexFloat(sym) - return fmt.Sprintf("%[1]s_re %[2]s, %[1]s_im %[2]s", name, cf) - } - return name + " " + sym.cgoname -} - -// cgoResult returns the result type of an exported function that returns a -// value of sym. -func (g *pyGen) cgoResult(sym *symbol) string { - if g.isCFFIComplex(sym) { - cf, _ := cffiComplexFloat(sym) - return "(" + cf + ", " + cf + ")" - } - return sym.cgoname -} - -// cpyName returns the type that build.py records for a value of sym. -// wrapper in cffi_build.py expands complex64 and complex128. -func (g *pyGen) cpyName(sym *symbol) string { - if g.isCFFIComplex(sym) { - return sym.goname - } - return sym.cpyname -} - -// goToCgo returns the Go expression that converts expr, a value of sym, to -// what an exported function returns. -func (g *pyGen) goToCgo(sym *symbol, expr string) string { - switch { - case g.isCFFIComplex(sym): - return sym.goname + "GoToPyCFFI(" + expr + ")" - case sym.go2py != "": - return sym.go2py + "(" + expr + ")" + sym.go2pyParenEx - } - return expr -} - -// cgoToGo returns the Go expression that converts the parameter name, as -// declared by cgoParam, to a value of sym. -func (g *pyGen) cgoToGo(sym *symbol, name string) string { - switch { - case g.isCFFIComplex(sym): - return sym.goname + "PyToGoCFFI(" + name + "_re, " + name + "_im)" - case sym.py2go != "": - return sym.py2go + "(" + name + ")" + sym.py2goParenEx - } - return name -} - // same argument positions as goPreamble: 1 = name of package, 2 = cmdstr, // 4 = GoHandle, 5 = CGoHandle, 6 = all imports, 7 = mainstr, 8 = C trampolines for // callbacks (see cffi_callback.go), 10 = gopy version. @@ -314,7 +209,7 @@ func errorGoToPy(e error) *C.char { return C.CString("") } -// complex values cross as two floats, see isCFFIComplex in cffi.go +// complex values cross as two floats, see isComplexShim in cffi.go func complex64GoToPyCFFI(c complex64) (C.float, C.float) { return C.float(real(c)), C.float(imag(c)) } diff --git a/bind/gen_func.go b/bind/gen_func.go index 083874bb..d83785aa 100644 --- a/bind/gen_func.go +++ b/bind/gen_func.go @@ -225,7 +225,7 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { } func (g *pyGen) genFunc(o *Func) { - if g.isCFFI() && g.genFuncComplexCFFI(o) { + if g.noAPIShim() && g.genFuncComplexCFFI(o) { return } if g.genFuncSig(nil, o) { @@ -236,7 +236,7 @@ func (g *pyGen) genFunc(o *Func) { // genFuncComplexCFFI generates a plain (non-method) function whose every // argument and its one return value are complex64/128, which the normal path // (genFuncSig/genFuncBody) can't do under cffi, where a complex value crosses -// as two floats (see isCFFIComplex in cffi.go). It returns false, writing +// as two floats (see isComplexShim in cffi.go). It returns false, writing // nothing, if the signature doesn't fit that narrow shape (methods, a mix of // complex and other argument types, or an error return): genFuncSig's // PyObject* check then skips the function instead of emitting code that diff --git a/bind/gen_slice.go b/bind/gen_slice.go index 74309a2a..ee1c9127 100644 --- a/bind/gen_slice.go +++ b/bind/gen_slice.go @@ -74,7 +74,7 @@ func (g *pyGen) genSliceInit(slc *symbol, extTypes, pyWrapOnly bool, slob *Slice // which cffi's preamble doesn't declare (see genFuncSig for the same // restriction on plain function args/returns); skip the whole wrapper // rather than emit code that fails to compile. - if g.noAPIShim() && esym != nil && esym.cpyname == "PyObject*" && !g.isCFFIComplex(esym) { + if g.noAPIShim() && esym != nil && esym.cpyname == "PyObject*" && !g.isComplexShim(esym) { return } diff --git a/bind/noapi.go b/bind/noapi.go new file mode 100644 index 00000000..1115fe77 --- /dev/null +++ b/bind/noapi.go @@ -0,0 +1,116 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import "fmt" + +// A "no-API" backend generates exported Go functions that never touch the +// CPython C API, because their python-facing wrapper is produced outside the +// Go build: cffi loads a plain shared library at runtime; pybind11 compiles +// a C++ file against that same library. This file holds what the two share +// about the shape of that library -- backend detection, error reporting +// (gopySetError, defined in cffi.go's preamble and used by both), and how a +// complex64/128 value crosses the boundary -- leaving each backend's own file +// for what only it needs (cffiBuildPreamble in cffi.go, pybind11BuildPreamble +// in pybind11.go). + +func (g *pyGen) isCFFI() bool { + return g.cfg.Backend == BackendCFFI +} + +func (g *pyGen) isPyBind11() bool { + return g.cfg.Backend == BackendPyBind11 +} + +// noAPIShim reports whether the exported Go functions must avoid the +// CPython C API (see the file comment above). +func (g *pyGen) noAPIShim() bool { + return g.isCFFI() || g.isPyBind11() +} + +// goSetError returns Go code that records an error for Python to raise. +// kind is the name of a Python builtin exception, msg a Go string expression. +func (g *pyGen) goSetError(kind, msg string) string { + return "gopySetError(\"" + kind + "\", " + msg + ")\n" +} + +func isComplexSym(sym *symbol) bool { + return sym != nil && (sym.goname == "complex64" || sym.goname == "complex128") +} + +// A complex64/complex128 value has no single C type that cffi or pybind11 +// can declare (cgo's is _Complex), and cgo won't export a struct, so under +// either it crosses as two floats: as two parameters (_re, _im), +// and as a result in cgo's two-value return, which it exports as a plain C +// struct {r0; r1;}. The methods below say how a value of a given symbol +// crosses, so that the generators only differ from the default backend here. + +// isComplexShim reports whether sym crosses as two floats, under either +// no-API backend (cffi or pybind11). +func (g *pyGen) isComplexShim(sym *symbol) bool { + return g.noAPIShim() && isComplexSym(sym) +} + +// cffiComplexFloat returns the cgo and the Go float type of the parts of a +// complex64 or complex128 symbol. +func cffiComplexFloat(sym *symbol) (cfloat, gofloat string) { + if sym.goname == "complex64" { + return "C.float", "float32" + } + return "C.double", "float64" +} + +// cgoParam returns the declaration of the parameter of an exported function +// that carries a value of sym. +func (g *pyGen) cgoParam(name string, sym *symbol) string { + if g.isComplexShim(sym) { + cf, _ := cffiComplexFloat(sym) + return fmt.Sprintf("%[1]s_re %[2]s, %[1]s_im %[2]s", name, cf) + } + return name + " " + sym.cgoname +} + +// cgoResult returns the result type of an exported function that returns a +// value of sym. +func (g *pyGen) cgoResult(sym *symbol) string { + if g.isComplexShim(sym) { + cf, _ := cffiComplexFloat(sym) + return "(" + cf + ", " + cf + ")" + } + return sym.cgoname +} + +// cpyName returns the type that build.py records for a value of sym. +// wrapper in cffi_build.py and pybind11_build.py expands complex64/128. +func (g *pyGen) cpyName(sym *symbol) string { + if g.isComplexShim(sym) { + return sym.goname + } + return sym.cpyname +} + +// goToCgo returns the Go expression that converts expr, a value of sym, to +// what an exported function returns. +func (g *pyGen) goToCgo(sym *symbol, expr string) string { + switch { + case g.isComplexShim(sym): + return sym.goname + "GoToPyCFFI(" + expr + ")" + case sym.go2py != "": + return sym.go2py + "(" + expr + ")" + sym.go2pyParenEx + } + return expr +} + +// cgoToGo returns the Go expression that converts the parameter name, as +// declared by cgoParam, to a value of sym. +func (g *pyGen) cgoToGo(sym *symbol, name string) string { + switch { + case g.isComplexShim(sym): + return sym.goname + "PyToGoCFFI(" + name + "_re, " + name + "_im)" + case sym.py2go != "": + return sym.py2go + "(" + name + ")" + sym.py2goParenEx + } + return name +} diff --git a/bind/pybind11_build.py b/bind/pybind11_build.py index 3daaef37..df1b241a 100644 --- a/bind/pybind11_build.py +++ b/bind/pybind11_build.py @@ -51,10 +51,10 @@ def add_checked_function(mod, name, retval, params, failure_expression="", *a, * def wrapper(name, ret, params): """Returns the m.def(...) call binding name, or "" if its signature - isn't supported yet (a PyObject*, i.e. complex64/128 or a callback - argument): the .cpp simply never binds it, so calling it from python - raises AttributeError instead of NotImplementedError -- close enough for - a function nothing in gopy's own generated wrapper calls unconditionally. + isn't supported yet (a raw PyObject*, i.e. a callback argument): the .cpp + simply never binds it, so calling it from python raises AttributeError + instead of NotImplementedError -- close enough for a function nothing in + gopy's own generated wrapper calls unconditionally. """ if ret == "PyObject*" or any(p[0] == "PyObject*" for p in params): return "" @@ -67,6 +67,10 @@ def wrapper(name, ret, params): elif ctype == "bool": args.append("bool " + pname) call_args.append("(char)(%s ? 1 : 0)" % pname) + elif ctype in ("complex64", "complex128"): + cxxfloat = "float" if ctype == "complex64" else "double" + args.append("std::complex<%s> %s" % (cxxfloat, pname)) + call_args.append("%s.real(), %s.imag()" % (pname, pname)) else: args.append(ctype + " " + pname) call_args.append(pname) @@ -85,6 +89,14 @@ def wrapper(name, ret, params): elif ret == "bool": body = "auto _r = %s;\n _check();\n return _r != 0;" % call cpptype = "bool" + elif ret in ("complex64", "complex128"): + cxxfloat = "float" if ret == "complex64" else "double" + cpptype = "std::complex<%s>" % cxxfloat + body = ( + "auto _r = %s;\n" + " _check();\n" + " return %s(_r.r0, _r.r1);" % (call, cpptype) + ) else: body = "auto _r = %s;\n _check();\n return _r;" % call cpptype = ret @@ -98,9 +110,11 @@ def wrapper(name, ret, params): MODULE_TEMPLATE = '''// python bindings for package @NAME@ using pybind11. // File is generated by gopy version @VERSION@. Do not edit. +#include #include #include +#include #include namespace py = pybind11; From 4af56433d1fef695b49658399f7756fb36e29f4f Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 25 Sep 2026 21:43:39 -0400 Subject: [PATCH 19/21] Add pybind11 callbacks; fix GIL deadlock and crash --- .github/workflows/ci.yml | 11 ++- bind/gen.go | 6 +- bind/gen_func.go | 9 +++ bind/pybind11_build.py | 159 ++++++++++++++++++++++++++++++++++++-- bind/pybind11_callback.go | 87 +++++++++++++++++++++ cmd_build.go | 70 ++++++++++++----- 6 files changed, 306 insertions(+), 36 deletions(-) create mode 100644 bind/pybind11_callback.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31be9187..49d4edf8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,15 +185,14 @@ jobs: - name: Build run: go build -v ./... - # The skipped tests use features the pybind11 backend does not support - # yet: Python callbacks (TestBindFuncs, TestBindCallbacks), and - # pybind11's own auto-generated TypeError wording, which differs from - # pybindgen's for one case in TestHi (Person's setters given the wrong - # argument type) -- a message-text difference, not a functional one. + # The skipped test is pybind11's own auto-generated TypeError wording, + # which differs from pybindgen's for one case in TestHi (Person's + # setters given the wrong argument type) -- a message-text difference, + # not a functional one. # # -timeout raises go test's own default (10m): each test here compiles # twice (cgo, then a separate C++ step), against cffi's one, and once # measured at ~18.5s/test average on windows-latest, 32 tests alone used # 590s of the default budget. - name: Test - run: go test -v -timeout=30m -skip '^(TestHi|TestBindFuncs|TestBindCallbacks)$' ./... + run: go test -v -timeout=30m -skip '^TestHi$' ./... diff --git a/bind/gen.go b/bind/gen.go index b4a4afa6..2e904657 100644 --- a/bind/gen.go +++ b/bind/gen.go @@ -678,6 +678,7 @@ func (g *pyGen) genOut() { } g.gofile.Printf("\n\n") g.spliceCFFITrampolines() + g.splicePyBind11Trampolines() g.genPrintOut(g.cfg.Name+".go", g.gofile) g.genPrintOut("build.py", g.pybuild) if g.wantMakefile() { @@ -739,8 +740,11 @@ func (g *pyGen) genGoPreamble() { } if g.noAPIShim() { trampolines := "" - if g.isCFFI() { + switch { + case g.isCFFI(): trampolines = cffiTrampolinesKey + case g.isPyBind11(): + trampolines = pybind11TrampolinesKey } g.gofile.Printf(goPreambleCFFI, g.cfg.Name, g.cfg.Cmd, "", GoHandle, CGoHandle, pkgimport, g.cfg.Main, trampolines, "", g.cfg.Version) diff --git a/bind/gen_func.go b/bind/gen_func.go index d83785aa..860625c6 100644 --- a/bind/gen_func.go +++ b/bind/gen_func.go @@ -117,6 +117,9 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { case g.isCFFI() && sarg.isSignature(): goArgs = append(goArgs, fmt.Sprintf("%s unsafe.Pointer", anm)) pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", g.cffiCallback(sarg).pyType(), anm)) + case g.isPyBind11() && sarg.isSignature(): + goArgs = append(goArgs, fmt.Sprintf("%s CGoHandle", anm)) + pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", g.cffiCallback(sarg).pyType(), anm)) case ifchandle && arg.sym.goname == "interface{}": goArgs = append(goArgs, fmt.Sprintf("%s %s", anm, CGoHandle)) pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", PyHandle, anm)) @@ -353,6 +356,10 @@ func (g *pyGen) genFuncBody(sym *symbol, fsym *Func) { switch { case arg.sym.isSignature() && g.isCFFI(): g.gofile.Printf("%s", cffiCallbackPrologue(pySafeArg(arg.Name(), i))) + case arg.sym.isSignature() && g.isPyBind11(): + // no Go-side setup: the C++ registry itself refuses a call + // once the wrapping python call unregisters it (see + // pybind11_callback.go) case arg.sym.isSignature(): g.gofile.Printf("_fun_arg := %s\n", pySafeArg(arg.Name(), i)) } @@ -405,6 +412,8 @@ if __err != nil { na = fmt.Sprintf(`gopyh.VarFromHandle((gopyh.CGoHandle)(%s), "interface{}")`, anm) case arg.sym.isSignature() && g.isCFFI(): na = g.cffiCallbackLit(g.cffiCallback(arg.sym), anm) + case arg.sym.isSignature() && g.isPyBind11(): + na = g.pybind11CallbackLit(g.cffiCallback(arg.sym), anm) case arg.sym.isSignature(): na = fmt.Sprintf("%s", arg.sym.py2go) case arg.sym.py2go != "": diff --git a/bind/pybind11_build.py b/bind/pybind11_build.py index df1b241a..591a9062 100644 --- a/bind/pybind11_build.py +++ b/bind/pybind11_build.py @@ -34,9 +34,17 @@ def generate(self): import os here = os.path.dirname(os.path.abspath(__file__)) - defs = [d for d in (wrapper(name, ret, params) for name, ret, params in self.funcs) if d] - cpp = MODULE_TEMPLATE.replace("@HEADER@", self.header).replace( - "@DEFS@", "\n".join(defs) + # a callback's C++ trampoline is shared by every callable of the same + # shape (see pybind11_callback.go), keyed and numbered here in the + # same first-seen order Go numbered them in, so "gopy_cb_" means + # the same thing on both sides without the two ever exchanging it. + callback_kinds = {} + defs = [d for d in (wrapper(name, ret, params, callback_kinds) for name, ret, params in self.funcs) if d] + trampolines = "\n".join(callback_trampoline(ctype, i) for ctype, i in callback_kinds.items()) + cpp = ( + MODULE_TEMPLATE.replace("@HEADER@", self.header) + .replace("@CALLBACK_TRAMPOLINES@", trampolines) + .replace("@DEFS@", "\n".join(defs)) ) with open(os.path.join(here, self.cpp_name), "w") as f: f.write(cpp) @@ -49,16 +57,19 @@ def add_checked_function(mod, name, retval, params, failure_expression="", *a, * add_checked_string_function = add_checked_function -def wrapper(name, ret, params): +def wrapper(name, ret, params, callback_kinds): """Returns the m.def(...) call binding name, or "" if its signature - isn't supported yet (a raw PyObject*, i.e. a callback argument): the .cpp - simply never binds it, so calling it from python raises AttributeError - instead of NotImplementedError -- close enough for a function nothing in - gopy's own generated wrapper calls unconditionally. + isn't supported yet (a raw PyObject*): the .cpp simply never binds it, so + calling it from python raises AttributeError instead of + NotImplementedError -- close enough for a function nothing in gopy's own + generated wrapper calls unconditionally. callback_kinds is shared across + every call from Module.generate, one entry per distinct callback shape + seen so far (see there and callback_trampoline). """ if ret == "PyObject*" or any(p[0] == "PyObject*" for p in params): return "" args = [] + setup = [] call_args = [] for ctype, pname in params: if ctype == "char*": @@ -71,10 +82,29 @@ def wrapper(name, ret, params): cxxfloat = "float" if ctype == "complex64" else "double" args.append("std::complex<%s> %s" % (cxxfloat, pname)) call_args.append("%s.real(), %s.imag()" % (pname, pname)) + elif ctype.startswith("callback:"): + i = callback_kinds.setdefault(ctype, len(callback_kinds)) + args.append("py::function " + pname) + setup.append( + "int64_t _h_%s = gopy_cb_register(%s);\n" + " GopyCBGuard _g_%s{_h_%s};" % (pname, pname, pname, pname) + ) + call_args.append("_h_%s" % pname) else: args.append(ctype + " " + pname) call_args.append(pname) call = "%s(%s)" % (name, ", ".join(call_args)) + # Releasing the GIL only around the call itself (not the setup/result + # handling around it, which need it) matches what cffi gets for free + # from ctypes/cffi's own default behavior, and is what makes a callback + # arrive correctly rather than deadlock: Go may run it from a goroutine + # (see InGoroutine in _examples/callbacks) while this call's own thread + # blocks waiting for that goroutine, so it must not be left holding the + # only GIL there is. + if ret is None: + call = "[&]{ py::gil_scoped_release _rel; %s; }()" % call + else: + call = "[&]{ py::gil_scoped_release _rel; return %s; }()" % call if ret is None: body, cpptype = "%s;\n _check();" % call, "void" elif ret == "char*": @@ -100,6 +130,8 @@ def wrapper(name, ret, params): else: body = "auto _r = %s;\n _check();\n return _r;" % call cpptype = ret + if setup: + body = "\n ".join(setup) + "\n " + body return ' m.def("%s", [](%s) -> %s {\n %s\n });' % ( name, ", ".join(args), @@ -108,13 +140,78 @@ def wrapper(name, ret, params): ) +def callback_trampoline(ctype, idx): + """Returns the static gopy_cb_ trampoline for the callback shape in + ctype ("callback:()", see cffiCallback + in cffi_callback.go): the Go closure for every callable of this shape + calls gopy_cb_, passing its own registry handle as the first + argument (see pybind11CallbackLit in pybind11_callback.go). + """ + ret, _, rest = ctype[len("callback:") :].partition("(") + ctypes_ = [t for t in rest[:-1].split(",") if t] + names = ["a%d" % i for i in range(len(ctypes_))] + + def cxxparam(t): + return "unsigned char" if t == "bool" else t + + params = "".join(", %s %s" % (cxxparam(t), n) for t, n in zip(ctypes_, names)) + call_args = [] + for t, n in zip(ctypes_, names): + if t == "char*": + call_args.append("%s ? py::str(%s) : py::str()" % (n, n)) + elif t == "bool": + call_args.append("py::bool_(%s != 0)" % n) + else: + call_args.append(n) + call = "fn(%s)" % ", ".join(call_args) + cxxret = "void" if ret == "void" else cxxparam(ret) + zero = "" if ret == "void" else " 0" + if ret == "void": + body = "%s;" % call + elif ret == "bool": + body = "return %s.cast() ? 1 : 0;" % call + else: + body = "return %s.cast<%s>();" % (call, ret) + # A raised exception must not reach the extern "C" boundary as a C++ + # exception: unwinding through Go's compiled call frames is undefined + # behavior (a hard crash in practice). Printing it and returning the + # zero value instead matches what cffi's ffi.callback does by default. + body = ( + "try {\n" + " %s\n" + " } catch (py::error_already_set& e) {\n" + " e.restore();\n" + " PyErr_Print();\n" + " return%s;\n" + " }" % (body, zero) + ) + return ( + # gil must be declared (and so acquired) before fn: C++ destroys + # locals in reverse declaration order, and fn (a py::function) needs + # the GIL held for its own destructor -- declared the other way + # around, gil would release it first, and fn would decref without it. + 'extern "C" %s gopy_cb_%d(int64_t h%s) {\n' + " py::gil_scoped_acquire gil;\n" + " py::function fn;\n" + " if (!gopy_cb_lookup(h, fn)) {\n" + " return%s;\n" + " }\n" + " %s\n" + "}" % (cxxret, idx, params, zero, body) + ) + + MODULE_TEMPLATE = '''// python bindings for package @NAME@ using pybind11. // File is generated by gopy version @VERSION@. Do not edit. #include +#include #include +#include #include +#include #include +#include #include namespace py = pybind11; @@ -145,6 +242,52 @@ def wrapper(name, ret, params): throw py::error_already_set(); } +// A python callable passed as a func-typed argument is registered here for +// the duration of the call it was passed to (see pybind11_callback.go for +// why a registry rather than one C function pointer per callable), and the +// trampolines below (one per callback shape, see callback_trampoline in +// pybind11_build.py) look it up by handle each time Go calls back in. +static std::mutex gopy_cb_mutex; +static std::unordered_map gopy_cb_registry; +static int64_t gopy_cb_next = 1; + +static int64_t gopy_cb_register(py::function fn) { + std::lock_guard lock(gopy_cb_mutex); + int64_t h = gopy_cb_next++; + gopy_cb_registry[h] = std::move(fn); + return h; +} + +static void gopy_cb_unregister(int64_t h) { + std::lock_guard lock(gopy_cb_mutex); + gopy_cb_registry.erase(h); +} + +// Unregisters a callback's handle once the call it was passed to returns, +// even if that call raised: playing the same role gopyCallbackScope plays +// for cffi. +struct GopyCBGuard { + int64_t h; + ~GopyCBGuard() { gopy_cb_unregister(h); } +}; + +// Looks up the callable registered under h, or returns false if the call it +// was passed to has already returned (h was never valid, or was already +// unregistered). The caller must already hold the GIL (see +// callback_trampoline in pybind11_build.py for why it acquires that itself, +// rather than here). +static bool gopy_cb_lookup(int64_t h, py::function& out) { + std::lock_guard lock(gopy_cb_mutex); + auto it = gopy_cb_registry.find(h); + if (it == gopy_cb_registry.end()) { + return false; + } + out = it->second; + return true; +} + +@CALLBACK_TRAMPOLINES@ + PYBIND11_MODULE(_@NAME@, m) { // gen_slice.go always exports these 4 (under noAPIShim()) for the // built-in byte slice, regardless of whether the package uses []byte; diff --git a/bind/pybind11_callback.go b/bind/pybind11_callback.go new file mode 100644 index 00000000..e613c029 --- /dev/null +++ b/bind/pybind11_callback.go @@ -0,0 +1,87 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + "bytes" + "fmt" + "strings" +) + +// A Python callable passed to Go as a func-typed argument crosses the +// pybind11 boundary as an int64 handle, not a raw C function pointer: unlike +// cffi (whose ffi.callback creates one real function pointer per Python +// callable, via a libffi closure allocated at runtime), pybind11 has no way +// to synthesize new C-ABI function pointers at runtime, so instead one +// static C++ trampoline exists per callback signature, shared by every +// callable of that shape, and the actual py::function is looked up from a +// registry by handle each time Go calls back in (see MODULE_TEMPLATE and +// callback_trampoline in pybind11_build.py). Registering/unregistering the +// handle around the call (there, not here) is what makes a callback stop +// working once the python call it was passed to returns, playing the same +// role gopyCallbackScope plays for cffi. +// +// The parameter/result type rules are shared with cffi (cffiCallback, +// cffiCallbackParam, cffiCallbackResult in cffi_callback.go): the same Go +// types are supported, crossing as the same int64_t/uint64_t/double/bool/ +// char* vocabulary either way. + +// pybind11TrampolinesKey stands in for the extern declarations in the cgo +// preamble, which are written before the callback types that need them are +// known. Unlike cffiTrampolinesKey, these are declarations only: the +// trampolines themselves are defined in the .cpp pybind11_build.py writes +// (callback_trampoline), not here -- see the buildPyBind11 doc comment +// (cmd_build.go) for why Go and that .cpp can't be two separate libraries +// with a dependency in each direction. +const pybind11TrampolinesKey = "@@GOPY_PYBIND11_TRAMPOLINES@@" + +// splicePyBind11Trampolines writes the extern declarations into the cgo +// preamble, so the C compiler accepts calls to a function it never sees +// defined; the actual gopy_cb_N functions are resolved at the final link +// step in buildPyBind11, against the object code pybind11_build.py's +// generated .cpp compiles to. +func (g *pyGen) splicePyBind11Trampolines() { + if !g.isPyBind11() { + return + } + var c strings.Builder + for i, cb := range g.cbs { + params := []string{"int64_t h"} + for j, p := range cb.params { + params = append(params, fmt.Sprintf("%s a%d", cffiCType(p.ctype), j)) + } + ret := "void" + if cb.ret != nil { + ret = cffiCType(cb.ret.ctype) + } + fmt.Fprintf(&c, "extern %s gopy_cb_%d(%s);\n", ret, i, strings.Join(params, ", ")) + } + b := bytes.Replace(g.gofile.buf.Bytes(), []byte(pybind11TrampolinesKey), []byte(c.String()), 1) + g.gofile.buf = bytes.NewBuffer(b) +} + +// pybind11CallbackLit returns a Go func literal that calls the Python +// callable registered under the handle named anm. +func (g *pyGen) pybind11CallbackLit(cb *cffiCallback, anm string) string { + var decl, pre []string + args := []string{"C.int64_t(" + anm + ")"} + for _, p := range cb.params { + decl = append(decl, p.name+" "+p.gotyp) + if p.pre != "" { + pre = append(pre, p.pre) + } + args = append(args, p.conv) + } + call := fmt.Sprintf("C.gopy_cb_%d(%s)", g.cffiTrampoline(cb), strings.Join(args, ", ")) + result := "" + if cb.ret != nil { + result = " " + cb.ret.gotyp + if cb.ret.ctype == "bool" { + call += " != 0" + } + call = "return " + cb.ret.gotyp + "(" + call + ")" + } + return fmt.Sprintf("func(%s)%s {\n%s%s\n}", strings.Join(decl, ", "), result, strings.Join(pre, ""), call) +} diff --git a/cmd_build.go b/cmd_build.go index 8bb73b63..d9f732b9 100644 --- a/cmd_build.go +++ b/cmd_build.go @@ -134,7 +134,7 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { } if cfg.Backend == bind.BackendPyBind11 { - return buildPyBind11(cfg, buildname+libExt, pycfg) + return buildPyBind11(cfg, buildname, pycfg) } if mode == bind.ModeExe { @@ -373,19 +373,29 @@ func buildCFFI(cfg *BuildCfg, buildLib string) error { return err } -// buildPyBind11 builds the cgo shim as a plain shared library (same shape as -// buildCFFI's), runs build.py to write a pybind11 C++ module wrapping it, and -// compiles+links that with a C++ compiler. The current directory is the -// output directory. -func buildPyBind11(cfg *BuildCfg, buildLib string, pycfg bind.PyConfig) error { - args := []string{"build", "-mod=mod", "-buildmode=c-shared"} +// buildPyBind11 builds the cgo shim as a static archive, runs build.py to +// write a pybind11 C++ module wrapping it, and compiles+links that with a +// C++ compiler. The current directory is the output directory. +// +// Unlike cffi (buildCFFI), the wrapper Go generates (pybind11_callback.go) +// has Go call INTO the wrapper's own C++ code (the per-callback-shape +// trampolines) as well as the other way around. Two separately-built +// shared libraries can't have a dependency cycle like that -- neither can +// exist as a complete, loadable file before the other -- so instead of a +// shared library (buildCFFI's buildLib), the Go side here builds as a +// static archive (-buildmode=c-archive), with its symbols left unresolved +// until the single final link below, alongside the C++ object code that +// defines them. +func buildPyBind11(cfg *BuildCfg, buildname string, pycfg bind.PyConfig) error { + archive := buildname + ".a" + args := []string{"build", "-mod=mod", "-buildmode=c-archive"} if cfg.BuildTags != "" { args = append(args, "-tags", cfg.BuildTags) } if !cfg.Symbols { args = append(args, "-ldflags=-s -w") } - args = append(args, "-o", buildLib, ".") + args = append(args, "-o", archive, ".") fmt.Printf("go %v\n", strings.Join(args, " ")) cmdout, err := exec.Command("go", args...).CombinedOutput() if err != nil { @@ -430,14 +440,25 @@ func buildPyBind11(cfg *BuildCfg, buildLib string, pycfg bind.PyConfig) error { } return o } - // modlib depends on buildLib (alongside it) and libpython (wherever this - // VM's own one lives, e.g. not on the loader's default search path for a - // uv- or pyenv-managed Python); without an rpath for each, the loader - // only finds them if they happen to already be on its search path. + // modlib depends on libpython (wherever this VM's own one lives, e.g. + // not on the loader's default search path for a uv- or pyenv-managed + // Python); without an rpath, the loader only finds it if it happens to + // already be on its search path. var libdir string if m := regexp.MustCompile(`-L(\S+)`).FindStringSubmatch(pycfg.LdFlags); m != nil { libdir = strings.Trim(m[1], `"`) } + // The archive's Go runtime code calls into gopy_cb_N (defined below, in + // the .cpp), so the linker must be told to keep every object in it -- + // left to its own judgement, it would see nothing in the .cpp calling + // into the archive first and drop it as unused. GNU ld (Linux, and + // Windows' MinGW) and ld64 (macOS) spell that differently. + var archiveArgs []string + if runtime.GOOS == "darwin" { + archiveArgs = []string{"-Wl,-force_load," + archive} + } else { + archiveArgs = []string{"-Wl,--whole-archive", archive, "-Wl,--no-whole-archive"} + } cxxArgs := []string{"-std=c++17", "-fPIC", "-shared", "-O2"} switch runtime.GOOS { case "darwin": @@ -446,14 +467,14 @@ func buildPyBind11(cfg *BuildCfg, buildLib string, pycfg bind.PyConfig) error { cxxArgs = append(cxxArgs, "-Wl,-rpath,"+libdir) } case "windows": - // No rpath equivalent, but modlib finding buildLib (in the same - // directory) is handled at import time instead, by the generated - // wrapper's os.add_dll_directory() call (see PyWrapPreamble). - // MinGW's own runtime (libstdc++/libgcc/libwinpthread), which g++ - // links dynamically by default, has no such fix available -- it - // isn't found by name alone unless its directory happens to be on - // PATH -- so link it in statically instead. The C runtime (ucrt) - // stays dynamic, shared with Python's own. + // No rpath equivalent; modlib depends on nothing but libpython now + // that the Go side is a static archive, not a separate DLL of its + // own (see the buildPyBind11 doc comment). MinGW's own runtime + // (libstdc++/libgcc/libwinpthread), which g++ links dynamically by + // default, has no such fix available -- it isn't found by name + // alone unless its directory happens to be on PATH -- so link it in + // statically instead. The C runtime (ucrt) stays dynamic, shared + // with Python's own. cxxArgs = append(cxxArgs, "-static-libgcc", "-static-libstdc++", "-Wl,-Bstatic,--whole-archive", "-lwinpthread", "-Wl,--no-whole-archive", "-Wl,-Bdynamic") default: @@ -464,8 +485,15 @@ func buildPyBind11(cfg *BuildCfg, buildLib string, pycfg bind.PyConfig) error { } cxxArgs = append(cxxArgs, pyinc...) cxxArgs = append(cxxArgs, unquote(strings.Fields(pycfg.CFlags))...) - cxxArgs = append(cxxArgs, cfg.Name+".cpp", buildLib) + cxxArgs = append(cxxArgs, cfg.Name+".cpp") + cxxArgs = append(cxxArgs, archiveArgs...) cxxArgs = append(cxxArgs, unquote(strings.Fields(pycfg.LdFlags))...) + // c-archive mode (unlike c-shared) doesn't resolve the Go runtime's own + // dependencies on these itself; TODO: verified only on Linux -- unclear + // yet whether Windows/macOS need anything of their own added here too. + if runtime.GOOS != "windows" { + cxxArgs = append(cxxArgs, "-lpthread", "-ldl", "-lm") + } cxxArgs = append(cxxArgs, "-o", modlib) fmt.Printf("%v %v\n", cxx, strings.Join(cxxArgs, " ")) cmdout, err = exec.Command(cxx, cxxArgs...).CombinedOutput() From 325a6a5407fb06f67171a19ef9c00e2ee4d848b0 Mon Sep 17 00:00:00 2001 From: b-long Date: Fri, 25 Sep 2026 22:09:56 -0400 Subject: [PATCH 20/21] Add benchmark job for cffi, pybind11, & pybindgen --- .github/workflows/ci.yml | 35 +++++++++++++++++++++++++++ _examples/bench/bench.go | 23 ++++++++++++++++++ _examples/bench/run.sh | 40 ++++++++++++++++++++++++++++++ _examples/bench/run_bench.py | 47 ++++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+) create mode 100644 _examples/bench/bench.go create mode 100755 _examples/bench/run.sh create mode 100644 _examples/bench/run_bench.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49d4edf8..f331dbb7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -196,3 +196,38 @@ jobs: # 590s of the default budget. - name: Test run: go test -v -timeout=30m -skip '^TestHi$' ./... + + # Compares per-call overhead across backends (see _examples/bench/run.sh): + # not a pass/fail check, just a table uploaded as a build artifact. Runs + # on ubuntu-latest only -- the comparison is between backends, not OSes. + benchmark: + name: benchmark backends + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.25.x + cache: true + + - name: Install packages + run: | + python -m pip install pybindgen cffi pybind11 + go install golang.org/x/tools/cmd/goimports@v0.29.0 + + - name: Run benchmark + run: bash _examples/bench/run.sh | tee benchmark.txt + + - name: Upload benchmark table + uses: actions/upload-artifact@v4 + with: + name: benchmark + path: benchmark.txt diff --git a/_examples/bench/bench.go b/_examples/bench/bench.go new file mode 100644 index 00000000..1599d75f --- /dev/null +++ b/_examples/bench/bench.go @@ -0,0 +1,23 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bench has a few trivial functions used to compare the per-call +// overhead of gopy's backends (GOPY_BACKEND); see bench.py and the +// "benchmark" job in .github/workflows/ci.yml. Not part of the test suite +// itself: no _examples/bench entry in main_test.go's features map. +package bench + +// Add returns the sum of its arguments: the cheapest possible call, to +// isolate per-call FFI overhead from any argument-marshaling cost. +func Add(i, j int) int { + return i + j +} + +// Concat concatenates two strings: a second data point, since string +// arguments/returns cross the boundary very differently from an int +// (a managed buffer + length or a null-terminated copy, depending on the +// backend) and might not share the int case's relative cost. +func Concat(a, b string) string { + return a + b +} diff --git a/_examples/bench/run.sh b/_examples/bench/run.sh new file mode 100755 index 00000000..de0685b3 --- /dev/null +++ b/_examples/bench/run.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2026 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Builds _examples/bench under each of gopy's backends and prints a table +# comparing run_bench.py's timing/memory numbers across them. Run from the repo +# root; needs pybindgen, cffi and pybind11 all installed for the python +# interpreter named by $PYTHON (defaults to python3), and a C++ compiler. +set -eu + +PYTHON="${PYTHON:-python3}" +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +echo "building gopy..." +GOPY="$WORK/gopy" +(cd "$REPO" && go build -o "$GOPY" .) + +printf '%-10s %8s %14s %14s %10s\n' backend calls "add (s)" "concat (s)" "peak (KB)" + +for backend in pybindgen cffi pybind11; do + out="$WORK/$backend" + mkdir -p "$out" + # gopy build cds into -output and runs go build there; give it a module + # that resolves back to this checkout, same as go.mod already does for + # anyone building _examples/* in place. + printf 'module dummy\n\nrequire github.com/go-python/gopy v0.0.0\nreplace github.com/go-python/gopy => %s\n' "$REPO" >"$out/go.mod" + GOPY_BACKEND="$backend" "$GOPY" build -vm="$PYTHON" -output="$out" -no-make -package-prefix= \ + "$REPO/_examples/bench" >"$out/build.log" 2>&1 || { + echo "$backend: build failed, see $out/build.log" >&2 + tail -n 20 "$out/build.log" >&2 + continue + } + cp "$REPO/_examples/bench/run_bench.py" "$out/" + row="$(cd "$out" && "$PYTHON" run_bench.py)" + IFS=, read -r calls add_s concat_s peak_kb <<<"$row" + printf '%-10s %8s %14s %14s %10s\n' "$backend" "$calls" "$add_s" "$concat_s" "$peak_kb" +done diff --git a/_examples/bench/run_bench.py b/_examples/bench/run_bench.py new file mode 100644 index 00000000..a86f96b9 --- /dev/null +++ b/_examples/bench/run_bench.py @@ -0,0 +1,47 @@ +# Copyright 2026 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Times N calls each of bench.Add and bench.Concat, and the peak memory +# after them, printing one CSV line: iterations,add_seconds,concat_seconds,peak_kb +# (see run.sh, which builds this example under each backend and prints the +# resulting rows as a table -- this script itself doesn't know which +# backend it was built with). + +from __future__ import print_function + +import sys +import time + +import bench + +if sys.platform == "win32": + import psutil + + def peak_kb(): + return psutil.Process().memory_info().rss // 1024 +else: + import resource + + def peak_kb(): + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + + +N = 1000 +WARMUP = 100 + +for i in range(WARMUP): + bench.Add(i, i) + bench.Concat("a", "b") + +start = time.perf_counter() +for i in range(N): + bench.Add(i, i) +add_seconds = time.perf_counter() - start + +start = time.perf_counter() +for i in range(N): + bench.Concat("a", "b") +concat_seconds = time.perf_counter() - start + +print("%d,%f,%f,%d" % (N, add_seconds, concat_seconds, peak_kb())) From cfffb8862f0089400078402ccd5380988219d5af Mon Sep 17 00:00:00 2001 From: b-long Date: Sun, 27 Sep 2026 19:57:18 -0400 Subject: [PATCH 21/21] Add nanobind backend (GOPY_BACKEND=nanobind) --- .github/requirements-nanobind.txt | 5 + .github/workflows/ci.yml | 53 ++++- _examples/bench/run.sh | 4 +- bind/backend.go | 2 +- bind/backend_test.go | 2 + bind/gen.go | 4 +- bind/gen_func.go | 15 +- bind/nanobind.go | 29 +++ bind/nanobind_build.py | 321 ++++++++++++++++++++++++++++++ bind/noapi.go | 39 ++-- bind/pybind11_callback.go | 9 +- cmd_build.go | 65 ++++-- gen.go | 5 +- 13 files changed, 508 insertions(+), 45 deletions(-) create mode 100644 .github/requirements-nanobind.txt create mode 100644 bind/nanobind.go create mode 100644 bind/nanobind_build.py diff --git a/.github/requirements-nanobind.txt b/.github/requirements-nanobind.txt new file mode 100644 index 00000000..130ee127 --- /dev/null +++ b/.github/requirements-nanobind.txt @@ -0,0 +1,5 @@ +# Python packages for the GOPY_BACKEND=nanobind job in workflows/ci.yml. +# pybindgen is deliberately absent: the nanobind backend must not need it. +nanobind +# used by the memory-leak checks on Windows, where the resource module is missing +psutil diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f331dbb7..4b5334b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,7 +155,7 @@ jobs: runs-on: ${{ matrix.platform }} env: GOPY_BACKEND: pybind11 - # CXX defaults to "c++" in buildPyBind11 (cmd_build.go), which isn't a + # CXX defaults to "c++" in buildCXXModule (cmd_build.go), which isn't a # recognized command on windows-latest's MinGW toolchain (the same one # CGO_ENABLED=1 already needs there, so no extra install is needed). CXX: g++ @@ -197,6 +197,55 @@ jobs: - name: Test run: go test -v -timeout=30m -skip '^TestHi$' ./... + # Builds and tests the opt-in nanobind backend (GOPY_BACKEND=nanobind). + # Same shape as the pybind11 job above (the two share a cgo shim and a + # build step, see buildCXXModule in cmd_build.go); nanobind additionally + # compiles its own runtime (nb_combined.cpp) into every module, which + # needs no extra system packages either. + nanobind: + name: nanobind backend (${{ matrix.platform }}, Python ${{ matrix.python-version }}) + strategy: + fail-fast: true + matrix: + platform: [ubuntu-latest, windows-latest, macos-15] + python-version: ['3.11', '3.12'] + runs-on: ${{ matrix.platform }} + env: + GOPY_BACKEND: nanobind + # see the pybind11 job + CXX: g++ + PYTHONFAULTHANDLER: 1 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: .github/requirements-nanobind.txt + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.25.x + cache: true + + - name: Install packages + run: | + python -m pip install -r .github/requirements-nanobind.txt + go install golang.org/x/tools/cmd/goimports@v0.29.0 + + - name: Build + run: go build -v ./... + + # Skipped for the same reason as in the pybind11 job: nanobind words its + # own auto-generated TypeError the same way pybind11 does, not the way + # pybindgen does. + - name: Test + run: go test -v -timeout=30m -skip '^TestHi$' ./... + # Compares per-call overhead across backends (see _examples/bench/run.sh): # not a pass/fail check, just a table uploaded as a build artifact. Runs # on ubuntu-latest only -- the comparison is between backends, not OSes. @@ -220,7 +269,7 @@ jobs: - name: Install packages run: | - python -m pip install pybindgen cffi pybind11 + python -m pip install pybindgen cffi pybind11 nanobind go install golang.org/x/tools/cmd/goimports@v0.29.0 - name: Run benchmark diff --git a/_examples/bench/run.sh b/_examples/bench/run.sh index de0685b3..e8c85b0d 100755 --- a/_examples/bench/run.sh +++ b/_examples/bench/run.sh @@ -5,7 +5,7 @@ # Builds _examples/bench under each of gopy's backends and prints a table # comparing run_bench.py's timing/memory numbers across them. Run from the repo -# root; needs pybindgen, cffi and pybind11 all installed for the python +# root; needs pybindgen, cffi, pybind11 and nanobind all installed for the python # interpreter named by $PYTHON (defaults to python3), and a C++ compiler. set -eu @@ -20,7 +20,7 @@ GOPY="$WORK/gopy" printf '%-10s %8s %14s %14s %10s\n' backend calls "add (s)" "concat (s)" "peak (KB)" -for backend in pybindgen cffi pybind11; do +for backend in pybindgen cffi pybind11 nanobind; do out="$WORK/$backend" mkdir -p "$out" # gopy build cds into -output and runs go build there; give it a module diff --git a/bind/backend.go b/bind/backend.go index 4632f301..ff22b53b 100644 --- a/bind/backend.go +++ b/bind/backend.go @@ -34,7 +34,7 @@ var backends = []struct { {BackendPyBindGen, true}, {BackendCFFI, true}, {BackendPyBind11, true}, - {BackendNanobind, false}, + {BackendNanobind, true}, {BackendCAPI, false}, {BackendCGO, false}, } diff --git a/bind/backend_test.go b/bind/backend_test.go index e17f4857..08bb9ae6 100644 --- a/bind/backend_test.go +++ b/bind/backend_test.go @@ -20,6 +20,8 @@ func TestParseBackend(t *testing.T) { {in: " PyBindGen ", want: BackendPyBindGen}, {in: "cffi", want: BackendCFFI}, {in: "pybind11", want: BackendPyBind11}, + {in: "nanobind", want: BackendNanobind}, + {in: "capi", errPart: "not implemented yet"}, {in: "bogus", errPart: "unknown GOPY_BACKEND"}, } { got, err := parseBackend(tc.in) diff --git a/bind/gen.go b/bind/gen.go index 2e904657..b3ee8288 100644 --- a/bind/gen.go +++ b/bind/gen.go @@ -743,7 +743,7 @@ func (g *pyGen) genGoPreamble() { switch { case g.isCFFI(): trampolines = cffiTrampolinesKey - case g.isPyBind11(): + case g.isCXXShim(): trampolines = pybind11TrampolinesKey } g.gofile.Printf(goPreambleCFFI, g.cfg.Name, g.cfg.Cmd, "", GoHandle, CGoHandle, @@ -792,6 +792,8 @@ func (g *pyGen) genPyBuildPreamble() { g.pybuild.Printf("%s", g.cffiBuildPreamble()) case g.isPyBind11(): g.pybuild.Printf("%s", g.pybind11BuildPreamble()) + case g.isNanobind(): + g.pybuild.Printf("%s", g.nanobindBuildPreamble()) default: g.pybuild.Printf(PyBuildPreamble, g.cfg.Name, g.cfg.Cmd, g.cfg.Version) } diff --git a/bind/gen_func.go b/bind/gen_func.go index 860625c6..33d14082 100644 --- a/bind/gen_func.go +++ b/bind/gen_func.go @@ -67,9 +67,9 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { return false } - // Neither no-API backend (cffi, pybind11) can cross a raw PyObject* - // (complex64/128, and -- for cffi, which is all that supports callbacks - // so far -- a callback argument of a type cffiCallback doesn't handle): + // None of the no-API backends (cffi, pybind11, nanobind) can cross a raw + // PyObject* (complex64/128, and a callback argument of a type + // cffiCallback doesn't handle): // skip these functions rather than emit a signature referencing the // CPython C API, which would fail to even compile under their preamble. if g.noAPIShim() { @@ -117,7 +117,7 @@ func (g *pyGen) genFuncSig(sym *symbol, fsym *Func) bool { case g.isCFFI() && sarg.isSignature(): goArgs = append(goArgs, fmt.Sprintf("%s unsafe.Pointer", anm)) pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", g.cffiCallback(sarg).pyType(), anm)) - case g.isPyBind11() && sarg.isSignature(): + case g.isCXXShim() && sarg.isSignature(): goArgs = append(goArgs, fmt.Sprintf("%s CGoHandle", anm)) pyArgs = append(pyArgs, fmt.Sprintf("param('%s', '%s')", g.cffiCallback(sarg).pyType(), anm)) case ifchandle && arg.sym.goname == "interface{}": @@ -356,7 +356,7 @@ func (g *pyGen) genFuncBody(sym *symbol, fsym *Func) { switch { case arg.sym.isSignature() && g.isCFFI(): g.gofile.Printf("%s", cffiCallbackPrologue(pySafeArg(arg.Name(), i))) - case arg.sym.isSignature() && g.isPyBind11(): + case arg.sym.isSignature() && g.isCXXShim(): // no Go-side setup: the C++ registry itself refuses a call // once the wrapping python call unregisters it (see // pybind11_callback.go) @@ -366,7 +366,8 @@ func (g *pyGen) genFuncBody(sym *symbol, fsym *Func) { } } - // cffi and pybind11 each release the GIL themselves around every call + // cffi, pybind11 and nanobind each release the GIL themselves around + // every call if !g.noAPIShim() { g.gofile.Printf("_saved_thread := C.PyEval_SaveThread()\n") if !rvIsErr && nres != 2 { @@ -412,7 +413,7 @@ if __err != nil { na = fmt.Sprintf(`gopyh.VarFromHandle((gopyh.CGoHandle)(%s), "interface{}")`, anm) case arg.sym.isSignature() && g.isCFFI(): na = g.cffiCallbackLit(g.cffiCallback(arg.sym), anm) - case arg.sym.isSignature() && g.isPyBind11(): + case arg.sym.isSignature() && g.isCXXShim(): na = g.pybind11CallbackLit(g.cffiCallback(arg.sym), anm) case arg.sym.isSignature(): na = fmt.Sprintf("%s", arg.sym.py2go) diff --git a/bind/nanobind.go b/bind/nanobind.go new file mode 100644 index 00000000..dea64be7 --- /dev/null +++ b/bind/nanobind.go @@ -0,0 +1,29 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bind + +import ( + _ "embed" + "strings" +) + +// The nanobind backend (GOPY_BACKEND=nanobind) is the pybind11 backend with +// a different C++ library on the consumer side: the same cgo shim (see +// noAPIShim), the same handle-registry callbacks (see isCXXShim and +// pybind11_callback.go), the same single final link against a c-archive (see +// buildCXXModule in cmd_build.go). Only nanobind_build.py, a fork of +// pybind11_build.py written against nanobind's API, is its own. + +//go:embed nanobind_build.py +var nanobindBuildPy string + +// nanobindBuildPreamble returns the start of build.py: the nanobind recorder. +func (g *pyGen) nanobindBuildPreamble() string { + return strings.NewReplacer( + "@NAME@", g.cfg.Name, + "@CMD@", g.cfg.Cmd, + "@VERSION@", g.cfg.Version, + ).Replace(nanobindBuildPy) +} diff --git a/bind/nanobind_build.py b/bind/nanobind_build.py new file mode 100644 index 00000000..5492c43c --- /dev/null +++ b/bind/nanobind_build.py @@ -0,0 +1,321 @@ +# python build stub for package @NAME@ (nanobind backend) +# File is generated by gopy version @VERSION@. Do not edit. +# @CMD@ +# +# The generated build code below is written against the pybindgen API. Here +# the same calls are only recorded, and Module.generate() then writes +# @NAME@.cpp: a nanobind module that #includes @NAME@_go.h (cgo's own header) +# and calls the shim functions directly. This is a fork of pybind11_build.py +# (nanobind's API mirrors pybind11's): keep the two in step, differing only +# where nanobind itself does. + + +def retval(ctype, *a, **kw): + return ctype + + +def param(ctype, name, *a, **kw): + return (ctype, name) + + +class Module(object): + def __init__(self, name, cpp_name): + self.name = name # the compiled extension's import name, e.g. "_hi" + self.cpp_name = cpp_name # source file to write, e.g. "hi.cpp" + self.header = None + self.funcs = [] + + def add_include(self, inc): + self.header = inc.strip('"') + + def add_function(self, name, ret, params, *a, **kw): + self.funcs.append((name, ret, params)) + + def generate(self): + import os + + here = os.path.dirname(os.path.abspath(__file__)) + # a callback's C++ trampoline is shared by every callable of the same + # shape (see pybind11_callback.go, which nanobind shares), keyed and + # numbered here in the same first-seen order Go numbered them in, so + # "gopy_cb_" means the same thing on both sides without the two + # ever exchanging it. + callback_kinds = {} + defs = [d for d in (wrapper(name, ret, params, callback_kinds) for name, ret, params in self.funcs) if d] + trampolines = "\n".join(callback_trampoline(ctype, i) for ctype, i in callback_kinds.items()) + cpp = ( + MODULE_TEMPLATE.replace("@HEADER@", self.header) + .replace("@CALLBACK_TRAMPOLINES@", trampolines) + .replace("@DEFS@", "\n".join(defs)) + ) + with open(os.path.join(here, self.cpp_name), "w") as f: + f.write(cpp) + + +def add_checked_function(mod, name, retval, params, failure_expression="", *a, **kw): + mod.add_function(name, retval, params) + + +add_checked_string_function = add_checked_function + + +def wrapper(name, ret, params, callback_kinds): + """Returns the m.def(...) call binding name, or "" if its signature + isn't supported yet (a raw PyObject*): the .cpp simply never binds it, so + calling it from python raises AttributeError instead of + NotImplementedError -- close enough for a function nothing in gopy's own + generated wrapper calls unconditionally. callback_kinds is shared across + every call from Module.generate, one entry per distinct callback shape + seen so far (see there and callback_trampoline). + """ + if ret == "PyObject*" or any(p[0] == "PyObject*" for p in params): + return "" + args = [] + setup = [] + call_args = [] + for ctype, pname in params: + if ctype == "char*": + args.append("const std::string& " + pname) + call_args.append("const_cast(%s.c_str())" % pname) + elif ctype == "bool": + args.append("bool " + pname) + call_args.append("(char)(%s ? 1 : 0)" % pname) + elif ctype in ("complex64", "complex128"): + cxxfloat = "float" if ctype == "complex64" else "double" + args.append("std::complex<%s> %s" % (cxxfloat, pname)) + call_args.append("%s.real(), %s.imag()" % (pname, pname)) + elif ctype.startswith("callback:"): + i = callback_kinds.setdefault(ctype, len(callback_kinds)) + args.append("nb::callable " + pname) + setup.append( + "int64_t _h_%s = gopy_cb_register(%s);\n" + " GopyCBGuard _g_%s{_h_%s};" % (pname, pname, pname, pname) + ) + call_args.append("_h_%s" % pname) + else: + args.append(ctype + " " + pname) + call_args.append(pname) + call = "%s(%s)" % (name, ", ".join(call_args)) + # Releasing the GIL only around the call itself (not the setup/result + # handling around it, which need it) matches what cffi gets for free + # from ctypes/cffi's own default behavior, and is what makes a callback + # arrive correctly rather than deadlock: Go may run it from a goroutine + # (see InGoroutine in _examples/callbacks) while this call's own thread + # blocks waiting for that goroutine, so it must not be left holding the + # only GIL there is. + if ret is None: + call = "[&]{ nb::gil_scoped_release _rel; %s; }()" % call + else: + call = "[&]{ nb::gil_scoped_release _rel; return %s; }()" % call + if ret is None: + body, cpptype = "%s;\n _check();" % call, "void" + elif ret == "char*": + body = ( + "char* _r = %s;\n" + " std::string _s(_r ? _r : \"\");\n" + " free(_r);\n" + " _check();\n" + " return _s;" % call + ) + cpptype = "std::string" + elif ret == "bool": + body = "auto _r = %s;\n _check();\n return _r != 0;" % call + cpptype = "bool" + elif ret in ("complex64", "complex128"): + cxxfloat = "float" if ret == "complex64" else "double" + cpptype = "std::complex<%s>" % cxxfloat + body = ( + "auto _r = %s;\n" + " _check();\n" + " return %s(_r.r0, _r.r1);" % (call, cpptype) + ) + else: + body = "auto _r = %s;\n _check();\n return _r;" % call + cpptype = ret + if setup: + body = "\n ".join(setup) + "\n " + body + return ' m.def("%s", [](%s) -> %s {\n %s\n });' % ( + name, + ", ".join(args), + cpptype, + body, + ) + + +def callback_trampoline(ctype, idx): + """Returns the static gopy_cb_ trampoline for the callback shape in + ctype ("callback:()", see cffiCallback + in cffi_callback.go): the Go closure for every callable of this shape + calls gopy_cb_, passing its own registry handle as the first + argument (see pybind11CallbackLit in pybind11_callback.go). + """ + ret, _, rest = ctype[len("callback:") :].partition("(") + ctypes_ = [t for t in rest[:-1].split(",") if t] + names = ["a%d" % i for i in range(len(ctypes_))] + + def cxxparam(t): + return "unsigned char" if t == "bool" else t + + params = "".join(", %s %s" % (cxxparam(t), n) for t, n in zip(ctypes_, names)) + call_args = [] + for t, n in zip(ctypes_, names): + if t == "char*": + call_args.append("nb::str(%s ? %s : \"\")" % (n, n)) + elif t == "bool": + call_args.append("nb::bool_(%s != 0)" % n) + else: + call_args.append(n) + call = "fn(%s)" % ", ".join(call_args) + cxxret = "void" if ret == "void" else cxxparam(ret) + zero = "" if ret == "void" else " 0" + if ret == "void": + body = "%s;" % call + elif ret == "bool": + body = "return nb::cast(%s) ? 1 : 0;" % call + else: + body = "return nb::cast<%s>(%s);" % (ret, call) + # A raised exception must not reach the extern "C" boundary as a C++ + # exception: unwinding through Go's compiled call frames is undefined + # behavior (a hard crash in practice). Printing it and returning the + # zero value instead matches what cffi's ffi.callback does by default. + body = ( + "try {\n" + " %s\n" + " } catch (nb::python_error& e) {\n" + " e.restore();\n" + " PyErr_Print();\n" + " return%s;\n" + " }" % (body, zero) + ) + return ( + # gil must be declared (and so acquired) before fn: C++ destroys + # locals in reverse declaration order, and fn (an nb::callable) needs + # the GIL held for its own destructor -- declared the other way + # around, gil would release it first, and fn would decref without it. + 'extern "C" %s gopy_cb_%d(int64_t h%s) {\n' + " nb::gil_scoped_acquire gil;\n" + " nb::callable fn;\n" + " if (!gopy_cb_lookup(h, fn)) {\n" + " return%s;\n" + " }\n" + " %s\n" + "}" % (cxxret, idx, params, zero, body) + ) + + +MODULE_TEMPLATE = '''// python bindings for package @NAME@ using nanobind. +// File is generated by gopy version @VERSION@. Do not edit. +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace nb = nanobind; + +extern "C" { +#include "@HEADER@" +} + +// Raises the exception, if any, that the last Go call on this thread +// recorded (see gopySetError/GopyTakeError, shared with the cffi backend). +static inline void _check() { + char* e = GopyTakeError(); + if (!e) { + return; + } + std::string s(e); + free(e); + auto sep = s.find(':'); + std::string kind = sep == std::string::npos ? s : s.substr(0, sep); + std::string msg = sep == std::string::npos ? "" : s.substr(sep + 1); + PyObject* exc = PyExc_RuntimeError; + if (kind == "ValueError") exc = PyExc_ValueError; + else if (kind == "TypeError") exc = PyExc_TypeError; + else if (kind == "KeyError") exc = PyExc_KeyError; + else if (kind == "IndexError") exc = PyExc_IndexError; + else if (kind == "AttributeError") exc = PyExc_AttributeError; + PyErr_SetString(exc, msg.c_str()); + throw nb::python_error(); +} + +// A python callable passed as a func-typed argument is registered here for +// the duration of the call it was passed to (see pybind11_callback.go for +// why a registry rather than one C function pointer per callable), and the +// trampolines below (one per callback shape, see callback_trampoline in +// nanobind_build.py) look it up by handle each time Go calls back in. +static std::mutex gopy_cb_mutex; +static std::unordered_map gopy_cb_registry; +static int64_t gopy_cb_next = 1; + +static int64_t gopy_cb_register(nb::callable fn) { + std::lock_guard lock(gopy_cb_mutex); + int64_t h = gopy_cb_next++; + gopy_cb_registry[h] = std::move(fn); + return h; +} + +static void gopy_cb_unregister(int64_t h) { + std::lock_guard lock(gopy_cb_mutex); + gopy_cb_registry.erase(h); +} + +// Unregisters a callback's handle once the call it was passed to returns, +// even if that call raised: playing the same role gopyCallbackScope plays +// for cffi. +struct GopyCBGuard { + int64_t h; + ~GopyCBGuard() { gopy_cb_unregister(h); } +}; + +// Looks up the callable registered under h, or returns false if the call it +// was passed to has already returned (h was never valid, or was already +// unregistered). The caller must already hold the GIL (see +// callback_trampoline in nanobind_build.py for why it acquires that itself, +// rather than here). +static bool gopy_cb_lookup(int64_t h, nb::callable& out) { + std::lock_guard lock(gopy_cb_mutex); + auto it = gopy_cb_registry.find(h); + if (it == gopy_cb_registry.end()) { + return false; + } + out = it->second; + return true; +} + +@CALLBACK_TRAMPOLINES@ + +NB_MODULE(_@NAME@, m) { + // gen_slice.go always exports these 4 (under noAPIShim()) for the + // built-in byte slice, regardless of whether the package uses []byte; + // they exchange a raw pointer+length rather than a PyObject*, same as + // the cffi and pybind11 backends. + m.def("Slice_byte_from_bytes", [](nb::bytes b) -> int64_t { + return Slice_byte_from_bytes(const_cast(b.c_str()), (long long)b.size()); + }); + m.def("Slice_byte_to_bytes", [](int64_t handle) -> nb::bytes { + long long n = Slice_byte_to_bytes_len(handle); + if (n == 0) { + return nb::bytes("", 0); + } + void* ptr = Slice_byte_to_bytes_ptr(handle); + nb::bytes result(ptr, (size_t)n); + Slice_byte_free_ptr(ptr); + return result; + }); +@DEFS@ +} +''' + +mod = Module('_@NAME@', '@NAME@.cpp') +mod.add_include('"@NAME@_go.h"') +mod.add_function('GoPyInit', None, []) +mod.add_function('DecRef', None, [param('int64_t', 'handle')]) +mod.add_function('IncRef', None, [param('int64_t', 'handle')]) +mod.add_function('NumHandles', retval('int'), []) +mod.add_function('RequestGC', None, []) diff --git a/bind/noapi.go b/bind/noapi.go index 1115fe77..4b18dc8c 100644 --- a/bind/noapi.go +++ b/bind/noapi.go @@ -8,13 +8,14 @@ import "fmt" // A "no-API" backend generates exported Go functions that never touch the // CPython C API, because their python-facing wrapper is produced outside the -// Go build: cffi loads a plain shared library at runtime; pybind11 compiles -// a C++ file against that same library. This file holds what the two share -// about the shape of that library -- backend detection, error reporting -// (gopySetError, defined in cffi.go's preamble and used by both), and how a -// complex64/128 value crosses the boundary -- leaving each backend's own file -// for what only it needs (cffiBuildPreamble in cffi.go, pybind11BuildPreamble -// in pybind11.go). +// Go build: cffi loads a plain shared library at runtime; pybind11 and +// nanobind compile a C++ file against that same library. This file holds +// what they share about the shape of that library -- backend detection, +// error reporting (gopySetError, defined in cffi.go's preamble and used by +// all of them), and how a complex64/128 value crosses the boundary -- +// leaving each backend's own file for what only it needs (cffiBuildPreamble +// in cffi.go, pybind11BuildPreamble in pybind11.go, nanobindBuildPreamble in +// nanobind.go). func (g *pyGen) isCFFI() bool { return g.cfg.Backend == BackendCFFI @@ -24,10 +25,21 @@ func (g *pyGen) isPyBind11() bool { return g.cfg.Backend == BackendPyBind11 } +func (g *pyGen) isNanobind() bool { + return g.cfg.Backend == BackendNanobind +} + +// isCXXShim reports whether the shim's consumer is a C++ module compiled +// against it (pybind11 or nanobind), which also means callbacks cross as +// registry handles (see pybind11_callback.go). +func (g *pyGen) isCXXShim() bool { + return g.isPyBind11() || g.isNanobind() +} + // noAPIShim reports whether the exported Go functions must avoid the // CPython C API (see the file comment above). func (g *pyGen) noAPIShim() bool { - return g.isCFFI() || g.isPyBind11() + return g.isCFFI() || g.isCXXShim() } // goSetError returns Go code that records an error for Python to raise. @@ -40,15 +52,15 @@ func isComplexSym(sym *symbol) bool { return sym != nil && (sym.goname == "complex64" || sym.goname == "complex128") } -// A complex64/complex128 value has no single C type that cffi or pybind11 -// can declare (cgo's is _Complex), and cgo won't export a struct, so under -// either it crosses as two floats: as two parameters (_re, _im), +// A complex64/complex128 value has no single C type that cffi or a C++ +// module can declare (cgo's is _Complex), and cgo won't export a struct, so +// under any of them it crosses as two floats: as two parameters (_re, _im), // and as a result in cgo's two-value return, which it exports as a plain C // struct {r0; r1;}. The methods below say how a value of a given symbol // crosses, so that the generators only differ from the default backend here. // isComplexShim reports whether sym crosses as two floats, under either -// no-API backend (cffi or pybind11). +// no-API backend (cffi, pybind11 or nanobind). func (g *pyGen) isComplexShim(sym *symbol) bool { return g.noAPIShim() && isComplexSym(sym) } @@ -83,7 +95,8 @@ func (g *pyGen) cgoResult(sym *symbol) string { } // cpyName returns the type that build.py records for a value of sym. -// wrapper in cffi_build.py and pybind11_build.py expands complex64/128. +// wrapper in cffi_build.py, pybind11_build.py and nanobind_build.py expands +// complex64/128. func (g *pyGen) cpyName(sym *symbol) string { if g.isComplexShim(sym) { return sym.goname diff --git a/bind/pybind11_callback.go b/bind/pybind11_callback.go index e613c029..01327552 100644 --- a/bind/pybind11_callback.go +++ b/bind/pybind11_callback.go @@ -27,12 +27,15 @@ import ( // cffiCallbackParam, cffiCallbackResult in cffi_callback.go): the same Go // types are supported, crossing as the same int64_t/uint64_t/double/bool/ // char* vocabulary either way. +// +// Everything in this file serves the nanobind backend too (see isCXXShim), +// whose nanobind_build.py defines the same registry and trampolines. // pybind11TrampolinesKey stands in for the extern declarations in the cgo // preamble, which are written before the callback types that need them are // known. Unlike cffiTrampolinesKey, these are declarations only: the // trampolines themselves are defined in the .cpp pybind11_build.py writes -// (callback_trampoline), not here -- see the buildPyBind11 doc comment +// (callback_trampoline), not here -- see the buildCXXModule doc comment // (cmd_build.go) for why Go and that .cpp can't be two separate libraries // with a dependency in each direction. const pybind11TrampolinesKey = "@@GOPY_PYBIND11_TRAMPOLINES@@" @@ -40,10 +43,10 @@ const pybind11TrampolinesKey = "@@GOPY_PYBIND11_TRAMPOLINES@@" // splicePyBind11Trampolines writes the extern declarations into the cgo // preamble, so the C compiler accepts calls to a function it never sees // defined; the actual gopy_cb_N functions are resolved at the final link -// step in buildPyBind11, against the object code pybind11_build.py's +// step in buildCXXModule, against the object code pybind11_build.py's // generated .cpp compiles to. func (g *pyGen) splicePyBind11Trampolines() { - if !g.isPyBind11() { + if !g.isCXXShim() { return } var c strings.Builder diff --git a/cmd_build.go b/cmd_build.go index d9f732b9..9e66bcfd 100644 --- a/cmd_build.go +++ b/cmd_build.go @@ -133,8 +133,11 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { return err } - if cfg.Backend == bind.BackendPyBind11 { + switch cfg.Backend { + case bind.BackendPyBind11: return buildPyBind11(cfg, buildname, pycfg) + case bind.BackendNanobind: + return buildNanobind(cfg, buildname, pycfg) } if mode == bind.ModeExe { @@ -373,9 +376,49 @@ func buildCFFI(cfg *BuildCfg, buildLib string) error { return err } -// buildPyBind11 builds the cgo shim as a static archive, runs build.py to -// write a pybind11 C++ module wrapping it, and compiles+links that with a -// C++ compiler. The current directory is the output directory. +// buildPyBind11 builds the pybind11 backend's module (see buildCXXModule). +func buildPyBind11(cfg *BuildCfg, buildname string, pycfg bind.PyConfig) error { + cmdout, err := exec.Command(cfg.VM, "-m", "pybind11", "--includes").CombinedOutput() + if err != nil { + fmt.Printf("cmd had error: %v output:\n%v\n(is pybind11 installed? pip install pybind11)\n", err, string(cmdout)) + return err + } + return buildCXXModule(cfg, buildname, pycfg, strings.Fields(strings.TrimSpace(string(cmdout))), nil) +} + +// buildNanobind builds the nanobind backend's module (see buildCXXModule). +// Unlike pybind11, nanobind isn't header-only: its own runtime (libnanobind) +// ships as source, meant to be compiled into each extension alongside the +// extension's own code, which nb_combined.cpp does in one translation unit. +func buildNanobind(cfg *BuildCfg, buildname string, pycfg bind.PyConfig) error { + cmdout, err := exec.Command(cfg.VM, "-c", + "import nanobind; print(nanobind.include_dir()); print(nanobind.source_dir())").CombinedOutput() + if err != nil { + fmt.Printf("cmd had error: %v output:\n%v\n(is nanobind installed? pip install nanobind)\n", err, string(cmdout)) + return err + } + dirs := strings.Split(strings.TrimSpace(string(cmdout)), "\n") + if len(dirs) != 2 { + return fmt.Errorf("gopy: unexpected output locating nanobind: %q", string(cmdout)) + } + incdir, srcdir := strings.TrimSpace(dirs[0]), strings.TrimSpace(dirs[1]) + // robin_map is a dependency nanobind vendors next to its own headers. + robinmap := filepath.Join(filepath.Dir(incdir), "ext", "robin_map", "include") + flags := []string{ + "-I" + incdir, "-I" + robinmap, + // as nanobind's own build does for libnanobind (see the comment at + // the top of nb_combined.cpp); harmless for the generated .cpp too. + "-DNDEBUG", "-DNB_COMPACT_ASSERTIONS", "-fno-strict-aliasing", + } + return buildCXXModule(cfg, buildname, pycfg, flags, []string{filepath.Join(srcdir, "nb_combined.cpp")}) +} + +// buildCXXModule builds the cgo shim as a static archive, runs build.py to +// write a C++ module wrapping it (pybind11 or nanobind), and compiles+links +// that with a C++ compiler, passing it cxxflags (the C++ library's include +// directories, and any flags of its own) and, besides the generated .cpp, +// the C++ library's own sources, if any. The current directory is the +// output directory. // // Unlike cffi (buildCFFI), the wrapper Go generates (pybind11_callback.go) // has Go call INTO the wrapper's own C++ code (the per-callback-shape @@ -386,7 +429,7 @@ func buildCFFI(cfg *BuildCfg, buildLib string) error { // static archive (-buildmode=c-archive), with its symbols left unresolved // until the single final link below, alongside the C++ object code that // defines them. -func buildPyBind11(cfg *BuildCfg, buildname string, pycfg bind.PyConfig) error { +func buildCXXModule(cfg *BuildCfg, buildname string, pycfg bind.PyConfig, cxxflags, srcs []string) error { archive := buildname + ".a" args := []string{"build", "-mod=mod", "-buildmode=c-archive"} if cfg.BuildTags != "" { @@ -410,13 +453,6 @@ func buildPyBind11(cfg *BuildCfg, buildname string, pycfg bind.PyConfig) error { return err } - cmdout, err = exec.Command(cfg.VM, "-m", "pybind11", "--includes").CombinedOutput() - if err != nil { - fmt.Printf("cmd had error: %v output:\n%v\n(is pybind11 installed? pip install pybind11)\n", err, string(cmdout)) - return err - } - pyinc := strings.Fields(strings.TrimSpace(string(cmdout))) - extext := libExt if runtime.GOOS == "windows" { extext = ".pyd" @@ -469,7 +505,7 @@ func buildPyBind11(cfg *BuildCfg, buildname string, pycfg bind.PyConfig) error { case "windows": // No rpath equivalent; modlib depends on nothing but libpython now // that the Go side is a static archive, not a separate DLL of its - // own (see the buildPyBind11 doc comment). MinGW's own runtime + // own (see the buildCXXModule doc comment). MinGW's own runtime // (libstdc++/libgcc/libwinpthread), which g++ links dynamically by // default, has no such fix available -- it isn't found by name // alone unless its directory happens to be on PATH -- so link it in @@ -483,9 +519,10 @@ func buildPyBind11(cfg *BuildCfg, buildname string, pycfg bind.PyConfig) error { cxxArgs = append(cxxArgs, "-Wl,-rpath,"+libdir) } } - cxxArgs = append(cxxArgs, pyinc...) + cxxArgs = append(cxxArgs, cxxflags...) cxxArgs = append(cxxArgs, unquote(strings.Fields(pycfg.CFlags))...) cxxArgs = append(cxxArgs, cfg.Name+".cpp") + cxxArgs = append(cxxArgs, srcs...) cxxArgs = append(cxxArgs, archiveArgs...) cxxArgs = append(cxxArgs, unquote(strings.Fields(pycfg.LdFlags))...) // c-archive mode (unlike c-shared) doesn't resolve the Go runtime's own diff --git a/gen.go b/gen.go index 4b41eabe..7970c0cc 100644 --- a/gen.go +++ b/gen.go @@ -66,10 +66,11 @@ func genPkg(mode bind.BuildMode, cfg *BuildCfg) error { if cfg.Backend, err = bind.BackendFromEnv(); err != nil { return err } - if (cfg.Backend == bind.BackendCFFI || cfg.Backend == bind.BackendPyBind11) && mode == bind.ModeExe { + if (cfg.Backend == bind.BackendCFFI || cfg.Backend == bind.BackendPyBind11 || cfg.Backend == bind.BackendNanobind) && mode == bind.ModeExe { // exe mode embeds the Python interpreter into the Go binary via the // CPython C API (see goExePreambleC/Go in bind/gen.go), unrelated to - // how the bindings themselves are generated; neither backend supports it. + // how the bindings themselves are generated; none of these backends + // supports it. return fmt.Errorf("gopy: %s=%s does not support gopy exe", bind.BackendEnvVar, cfg.Backend) } cfg.OutputDir, err = genOutDir(cfg.OutputDir)