From 2865990873e94c53d51a9eddffb2f1ca0c12a518 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 14:17:18 +0800 Subject: [PATCH 01/34] Remove deprecated VERSION_TUPLE and bump version to 4.0.0.dev0 --- CHANGELOG.rst | 1 + xxhash/__init__.py | 3 +-- xxhash/__init__.pyi | 3 --- xxhash/version.py | 4 +--- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1208b35..577453e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,7 @@ NEXT ~~~~~~~~~~~~~~~~~ - Drop support for Python 3.8 +- Remove deprecated ``xxhash.VERSION_TUPLE`` v3.7.0 2025-04-25 ~~~~~~~~~~~~~~~~~ diff --git a/xxhash/__init__.py b/xxhash/__init__.py index 5fe0087..4825901 100644 --- a/xxhash/__init__.py +++ b/xxhash/__init__.py @@ -18,7 +18,7 @@ XXHASH_VERSION, ) -from .version import VERSION, VERSION_TUPLE +from .version import VERSION xxh128 = xxh3_128 @@ -59,7 +59,6 @@ "xxh128_intdigest", "xxh128_hexdigest", "VERSION", - "VERSION_TUPLE", "XXHASH_VERSION", "algorithms_available", "algorithms_guaranteed", diff --git a/xxhash/__init__.pyi b/xxhash/__init__.pyi index 7490e30..91741d2 100644 --- a/xxhash/__init__.pyi +++ b/xxhash/__init__.pyi @@ -8,8 +8,6 @@ _DataType = _Buffer VERSION: str XXHASH_VERSION: str -#: Deprecated, will be removed in the next major release -VERSION_TUPLE: tuple[int, ...] algorithms_available: set[str] algorithms_guaranteed: set[str] @@ -36,7 +34,6 @@ __all__: list[str] = [ "xxh128_intdigest", "xxh128_hexdigest", "VERSION", - "VERSION_TUPLE", "XXHASH_VERSION", "algorithms_available", "algorithms_guaranteed", diff --git a/xxhash/version.py b/xxhash/version.py index 17952b8..1f5645f 100644 --- a/xxhash/version.py +++ b/xxhash/version.py @@ -1,3 +1 @@ -VERSION = "3.8.0.dev9" -#: Deprecated, will be removed in the next major release -VERSION_TUPLE = (3, 8, 0) +VERSION = "4.0.0.dev0" From 4304fa7334ca29b35c62cc424b4dedd2897df080 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 14:36:12 +0800 Subject: [PATCH 02/34] Make _xxhash.c lock behaviour controllable via XXHASH_WITH_LOCK and support a parameterized module name --- src/_xxhash.c | 135 +++++++++++++++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 52 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index d79f006..890baaf 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -35,60 +35,71 @@ /* ------------------------------------------------------------------ */ /* Lock type & helpers */ /* ------------------------------------------------------------------ */ -#if PY_VERSION_HEX >= 0x030d0000 /* Python 3.13+: always-on PyMutex (3.15+ style) */ -# define XXHASH_LOCK_FIELD PyMutex mutex; -# define XXHASH_LOCK_INIT(o) ((void)((o)->mutex = (PyMutex){0})) -# define XXHASH_LOCK_IS_ACTIVE(o) 1 -# define XXHASH_LOCK_MAYBE_INIT(o, len) ((void)0) -# define XXHASH_LOCK_FINI(o) ((void)0) -# define XXHASH_LOCK_ACQUIRE(o) PyMutex_Lock(&(o)->mutex) -# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) XXHASH_LOCK_ACQUIRE(o) -# define XXHASH_LOCK_RELEASE(o) PyMutex_Unlock(&(o)->mutex) -#else /* Python 3.9-3.12: PyThread_type_lock */ -# define XXHASH_LOCK_FIELD PyThread_type_lock lock; -# define XXHASH_LOCK_INIT(o) ((o)->lock = NULL) -# define XXHASH_LOCK_IS_ACTIVE(o) ((o)->lock != NULL) +#ifdef XXHASH_WITH_LOCK +# if PY_VERSION_HEX >= 0x030d0000 /* Python 3.13+: always-on PyMutex (3.15+ style) */ +# define XXHASH_LOCK_FIELD PyMutex mutex; +# define XXHASH_LOCK_INIT(o) ((void)((o)->mutex = (PyMutex){0})) +# define XXHASH_LOCK_IS_ACTIVE(o) 1 +# define XXHASH_LOCK_MAYBE_INIT(o, len) ((void)0) +# define XXHASH_LOCK_FINI(o) ((void)0) +# define XXHASH_LOCK_ACQUIRE(o) PyMutex_Lock(&(o)->mutex) +# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) XXHASH_LOCK_ACQUIRE(o) +# define XXHASH_LOCK_RELEASE(o) PyMutex_Unlock(&(o)->mutex) +# else /* Python 3.9-3.12: PyThread_type_lock */ +# define XXHASH_LOCK_FIELD PyThread_type_lock lock; +# define XXHASH_LOCK_INIT(o) ((o)->lock = NULL) +# define XXHASH_LOCK_IS_ACTIVE(o) ((o)->lock != NULL) /* Lazy allocation on first large update */ -# define XXHASH_LOCK_MAYBE_INIT(o, len) \ - do { \ - if ((o)->lock == NULL && (len) >= XXHASH_GIL_MINSIZE) { \ - (o)->lock = PyThread_allocate_lock(); \ - /* fail? lock stays NULL, fall back to non-threaded code. */ \ - } \ - } while (0) -# define XXHASH_LOCK_FINI(o) do { if ((o)->lock) \ - PyThread_free_lock((o)->lock); \ - } while (0) +# define XXHASH_LOCK_MAYBE_INIT(o, len) \ + do { \ + if ((o)->lock == NULL && (len) >= XXHASH_GIL_MINSIZE) { \ + (o)->lock = PyThread_allocate_lock(); \ + /* fail? lock stays NULL, fall back to non-threaded code. */ \ + } \ + } while (0) +# define XXHASH_LOCK_FINI(o) do { if ((o)->lock) \ + PyThread_free_lock((o)->lock); \ + } while (0) /* Acquire lock when GIL is already released — simple blocking acquire. * Only acquires if lock has been allocated (lazy init). */ -# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) \ - do { \ - if ((o)->lock) { \ - PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ - } \ - } while (0) +# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) \ + do { \ + if ((o)->lock) { \ + PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ + } \ + } while (0) /* Acquire lock with the GIL held — non-blocking try first, then release * GIL and block if contested (matches hashlib's ENTER_HASHLIB in 3.9-3.12). * Only acquires if lock has been allocated (lazy init). */ -# define XXHASH_LOCK_ACQUIRE(o) \ - do { \ - if ((o)->lock) { \ - if (!PyThread_acquire_lock((o)->lock, NOWAIT_LOCK)) { \ - /* Lock contested – release GIL while waiting. */ \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ - Py_END_ALLOW_THREADS \ - } \ - } \ - } while (0) - -# define XXHASH_LOCK_RELEASE(o) \ - do { \ - if ((o)->lock) { \ - PyThread_release_lock((o)->lock); \ - } \ - } while (0) +# define XXHASH_LOCK_ACQUIRE(o) \ + do { \ + if ((o)->lock) { \ + if (!PyThread_acquire_lock((o)->lock, NOWAIT_LOCK)) { \ + /* Lock contested – release GIL while waiting. */ \ + Py_BEGIN_ALLOW_THREADS \ + PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ + Py_END_ALLOW_THREADS \ + } \ + } \ + } while (0) + +# define XXHASH_LOCK_RELEASE(o) \ + do { \ + if ((o)->lock) { \ + PyThread_release_lock((o)->lock); \ + } \ + } while (0) +# endif +#else /* !XXHASH_WITH_LOCK */ +# define XXHASH_LOCK_FIELD +# define XXHASH_LOCK_INIT(o) ((void)0) +# define XXHASH_LOCK_IS_ACTIVE(o) 0 +# define XXHASH_LOCK_MAYBE_INIT(o, len) ((void)0) +# define XXHASH_LOCK_FINI(o) ((void)0) +# define XXHASH_LOCK_ACQUIRE(o) ((void)0) +# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) ((void)0) +# define XXHASH_LOCK_RELEASE(o) ((void)0) #endif /* Data size threshold for releasing the GIL during hash. */ @@ -98,6 +109,15 @@ #define VALUE_TO_STRING(x) TOSTRING(x) #define XXHASH_VERSION XXH_VERSION_MAJOR.XXH_VERSION_MINOR.XXH_VERSION_RELEASE +/* Module name is parameterised so the same source file can be compiled as + * both xxhash._xxhash and xxhash._xxhash_threadsafe. */ +#ifndef XXHASH_MODULE_NAME +# define XXHASH_MODULE_NAME _xxhash +#endif +#define XXHASH_PASTE2(a, b) a ## b +#define XXHASH_PASTE(a, b) XXHASH_PASTE2(a, b) +#define XXHASH_PYINIT(name) XXHASH_PASTE(PyInit__, name) + #define XXH32_DIGESTSIZE 4 #define XXH32_BLOCKSIZE 16 #define XXH64_DIGESTSIZE 8 @@ -647,8 +667,14 @@ PY##type##_do_update(PY##type##Object *self, Py_buffer *buf) \ XXHASH_LOCK_RELEASE(self); \ } \ } else { \ - /* No lock: hash directly, no GIL release. */ \ - update_fn(self->xxhash_state, buf->buf, buf->len); \ + /* No lock: release GIL for large data to avoid blocking other threads. */ \ + if (buf->len > XXHASH_GIL_MINSIZE) { \ + Py_BEGIN_ALLOW_THREADS \ + update_fn(self->xxhash_state, buf->buf, buf->len); \ + Py_END_ALLOW_THREADS \ + } else { \ + update_fn(self->xxhash_state, buf->buf, buf->len); \ + } \ } \ PyBuffer_Release(buf); \ } @@ -2230,8 +2256,13 @@ static PyModuleDef_Slot slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, #endif #if PY_VERSION_HEX >= 0x030d0000 - /* Python 3.13+: module is thread-safe with per-object lock */ +#ifdef XXHASH_WITH_LOCK + /* Thread-safe variant declares it manages its own synchronisation. */ {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#else + /* Default variant relies on the GIL (or caller synchronisation on free-threading builds). */ + {Py_mod_gil, Py_MOD_GIL_USED}, +#endif #endif {0, NULL} }; @@ -2254,7 +2285,7 @@ static PyMethodDef methods[] = { static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, - "_xxhash", + VALUE_TO_STRING(XXHASH_MODULE_NAME), NULL, 0, methods, @@ -2265,7 +2296,7 @@ static struct PyModuleDef moduledef = { }; PyMODINIT_FUNC -PyInit__xxhash(void) +XXHASH_PYINIT(XXHASH_MODULE_NAME)(void) { return PyModuleDef_Init(&moduledef); } From e91b0c6fb961319eb7938338dfede63e7fd86f9d Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 14:36:44 +0800 Subject: [PATCH 03/34] Build both _xxhash and _xxhash_threadsafe extensions; default is locked on free-threading builds --- setup.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 6fed6f6..cc16ee7 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ import os +import sysconfig from pathlib import Path from setuptools import Extension, setup @@ -12,13 +13,30 @@ source = ["src/_xxhash.c", "deps/xxhash/xxhash.c"] include_dirs = ["deps/xxhash"] +# In free-threading builds the default module must be thread-safe, because +# there is no GIL protecting the internal xxHash state. On regular GIL +# builds the default module is unlocked for maximum performance; users who +# need to share a streaming hash object across threads can use +# xxhash.threadsafe. +_is_free_threading = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + +_ext_kwargs = { + "sources": source, + "include_dirs": include_dirs, + "libraries": libraries, +} + ext_modules = [ Extension( "_xxhash", - source, - include_dirs=include_dirs, - libraries=libraries, - ) + define_macros=[("XXHASH_WITH_LOCK", "1")] if _is_free_threading else [], + **_ext_kwargs, + ), + Extension( + "_xxhash_threadsafe", + define_macros=[("XXHASH_WITH_LOCK", "1")], + **_ext_kwargs, + ), ] d = Path(__file__).parent From b92d9ba5eb5650a951b188bff4b926bdd6b9f7c6 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 14:46:38 +0800 Subject: [PATCH 04/34] Add xxhash.threadsafe submodule backed by the locked _xxhash_threadsafe extension --- setup.py | 22 +++++++++- xxhash/threadsafe.py | 62 +++++++++++++++++++++++++++++ xxhash/threadsafe.pyi | 93 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 xxhash/threadsafe.py create mode 100644 xxhash/threadsafe.pyi diff --git a/setup.py b/setup.py index cc16ee7..fd9933e 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ from pathlib import Path from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext as _build_ext if os.getenv("XXHASH_LINK_SO"): libraries = ["xxhash"] @@ -34,11 +35,29 @@ ), Extension( "_xxhash_threadsafe", - define_macros=[("XXHASH_WITH_LOCK", "1")], + define_macros=[("XXHASH_WITH_LOCK", "1"), ("XXHASH_MODULE_NAME", "_xxhash_threadsafe")], **_ext_kwargs, ), ] + +class build_ext(_build_ext): + """Build each extension in its own temp directory. + + Both extensions are built from the same ``src/_xxhash.c`` source file. + Without separate temp directories their object files would overwrite + each other, causing one variant to be linked with the wrong macros. + """ + + def build_extension(self, ext): + old_build_temp = self.build_temp + self.build_temp = os.path.join(old_build_temp, ext.name) + try: + super().build_extension(ext) + finally: + self.build_temp = old_build_temp + + d = Path(__file__).parent long_description = d.joinpath("README.rst").read_text() + "\n" + d.joinpath("CHANGELOG.rst").read_text() @@ -76,5 +95,6 @@ ], python_requires=">=3.9", ext_modules=ext_modules, + cmdclass={"build_ext": build_ext}, package_data={"xxhash": ["py.typed", "**.pyi"]}, ) diff --git a/xxhash/threadsafe.py b/xxhash/threadsafe.py new file mode 100644 index 0000000..0b42c73 --- /dev/null +++ b/xxhash/threadsafe.py @@ -0,0 +1,62 @@ +from xxhash.version import VERSION +from xxhash._xxhash_threadsafe import ( + xxh32, + xxh32_digest, + xxh32_intdigest, + xxh32_hexdigest, + xxh64, + xxh64_digest, + xxh64_intdigest, + xxh64_hexdigest, + xxh3_64, + xxh3_64_digest, + xxh3_64_intdigest, + xxh3_64_hexdigest, + xxh3_128, + xxh3_128_digest, + xxh3_128_intdigest, + xxh3_128_hexdigest, + XXHASH_VERSION, +) + +xxh128 = xxh3_128 +xxh128_digest = xxh3_128_digest +xxh128_intdigest = xxh3_128_intdigest +xxh128_hexdigest = xxh3_128_hexdigest + +algorithms_available = set([ + "xxh32", + "xxh64", + "xxh3_64", + "xxh128", + "xxh3_128", +]) + +algorithms_guaranteed = algorithms_available + +__all__ = [ + "xxh32", + "xxh32_digest", + "xxh32_intdigest", + "xxh32_hexdigest", + "xxh64", + "xxh64_digest", + "xxh64_intdigest", + "xxh64_hexdigest", + "xxh3_64", + "xxh3_64_digest", + "xxh3_64_intdigest", + "xxh3_64_hexdigest", + "xxh3_128", + "xxh3_128_digest", + "xxh3_128_intdigest", + "xxh3_128_hexdigest", + "xxh128", + "xxh128_digest", + "xxh128_intdigest", + "xxh128_hexdigest", + "VERSION", + "XXHASH_VERSION", + "algorithms_available", + "algorithms_guaranteed", +] diff --git a/xxhash/threadsafe.pyi b/xxhash/threadsafe.pyi new file mode 100644 index 0000000..e90bffe --- /dev/null +++ b/xxhash/threadsafe.pyi @@ -0,0 +1,93 @@ +from typing import Protocol, final + +class _Buffer(Protocol): + """Objects that support the buffer protocol (PEP 688).""" + def __buffer__(self, flags: int, /) -> memoryview: ... + +_DataType = _Buffer + +VERSION: str +XXHASH_VERSION: str + +algorithms_available: set[str] +algorithms_guaranteed: set[str] + +__all__: list[str] = [ + "xxh32", + "xxh32_digest", + "xxh32_intdigest", + "xxh32_hexdigest", + "xxh64", + "xxh64_digest", + "xxh64_intdigest", + "xxh64_hexdigest", + "xxh3_64", + "xxh3_64_digest", + "xxh3_64_intdigest", + "xxh3_64_hexdigest", + "xxh3_128", + "xxh3_128_digest", + "xxh3_128_intdigest", + "xxh3_128_hexdigest", + "xxh128", + "xxh128_digest", + "xxh128_intdigest", + "xxh128_hexdigest", + "VERSION", + "XXHASH_VERSION", + "algorithms_available", + "algorithms_guaranteed", +] + +class _Hasher: + def __init__(self, data: _DataType = ..., seed: int = ...) -> None: ... + def update(self, data: _DataType) -> None: ... + def digest(self) -> bytes: ... + def hexdigest(self) -> str: ... + def intdigest(self) -> int: ... + def copy(self) -> _Hasher: ... + def reset(self) -> None: ... + @property + def digestsize(self) -> int: ... + @property + def digest_size(self) -> int: ... + @property + def block_size(self) -> int: ... + @property + def name(self) -> str: ... + @property + def seed(self) -> int: ... + +@final +class xxh32(_Hasher): ... + +@final +class xxh64(_Hasher): ... + +@final +class xxh3_64(_Hasher): ... + +@final +class xxh3_128(_Hasher): ... + +xxh128 = xxh3_128 + +def xxh32_digest(data: _DataType, seed: int = ...) -> bytes: ... +def xxh32_hexdigest(data: _DataType, seed: int = ...) -> str: ... +def xxh32_intdigest(data: _DataType, seed: int = ...) -> int: ... + +def xxh64_digest(data: _DataType, seed: int = ...) -> bytes: ... +def xxh64_hexdigest(data: _DataType, seed: int = ...) -> str: ... +def xxh64_intdigest(data: _DataType, seed: int = ...) -> int: ... + +def xxh3_64_digest(data: _DataType, seed: int = ...) -> bytes: ... +def xxh3_64_hexdigest(data: _DataType, seed: int = ...) -> str: ... +def xxh3_64_intdigest(data: _DataType, seed: int = ...) -> int: ... + +def xxh3_128_digest(data: _DataType, seed: int = ...) -> bytes: ... +def xxh3_128_hexdigest(data: _DataType, seed: int = ...) -> str: ... +def xxh3_128_intdigest(data: _DataType, seed: int = ...) -> int: ... + +xxh128_digest = xxh3_128_digest +xxh128_hexdigest = xxh3_128_hexdigest +xxh128_intdigest = xxh3_128_intdigest From 38fa45ec698447ca384d459bbb34220279e68b9f Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 14:51:55 +0800 Subject: [PATCH 05/34] Update thread-safety tests to use xxhash.threadsafe --- tests/test_thread_safety.py | 45 +++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/tests/test_thread_safety.py b/tests/test_thread_safety.py index 24852ba..c254a4d 100644 --- a/tests/test_thread_safety.py +++ b/tests/test_thread_safety.py @@ -1,18 +1,15 @@ """ -Thread-safety tests for xxhash. +Thread-safety tests for xxhash.threadsafe. -Previously, the C extension released the GIL inside ``_do_update`` (via -``Py_BEGIN_ALLOW_THREADS`` / ``Py_END_ALLOW_THREADS``) while calling -``XXH*_update(self->xxhash_state, ...)``, and all other methods accessed the -same ``xxhash_state`` without any per-object lock — creating a data race. - -This has been fixed by adding a per-object ``PyThread_type_lock`` that protects -all access to ``xxhash_state``. The lock is acquired **after** releasing the -GIL in ``update()`` to avoid ABBA deadlocks. +The default ``xxhash`` module is optimized for speed and does not protect +streaming hash objects with a per-object lock. Concurrent access to the same +hash object from multiple threads is only safe when using the +``xxhash.threadsafe`` submodule, which adds a per-object lock around every +operation that touches the internal xxHash state. The tests below verify that: - * no crashes occur under concurrent access - * hash results are now deterministic (no data races) + * no crashes occur under concurrent access to ``threadsafe`` hash objects + * hash results are deterministic (no data races) """ import os @@ -20,7 +17,7 @@ import subprocess import signal import unittest -import xxhash +from xxhash import threadsafe as xxhash # --------------------------------------------------------------------------- @@ -47,7 +44,7 @@ def _run_in_subprocess(code: str, timeout: float = 60.0): # --------------------------------------------------------------------------- CONCURRENT_DIGEST_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh32() BLOCK = b'x' * (4 * 1024 * 1024) # 4 MiB @@ -93,7 +90,7 @@ def digester(): # --------------------------------------------------------------------------- CONCURRENT_RESET_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh32() BLOCK = b'x' * (4 * 1024 * 1024) @@ -139,7 +136,7 @@ def reseter(): # --------------------------------------------------------------------------- CONCURRENT_UPDATE_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh32() BLOCK = b'x' * (4 * 1024 * 1024) @@ -184,7 +181,7 @@ def updater(): # --------------------------------------------------------------------------- NON_DETERMINISM_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh32() BLOCK = b'x' * (4 * 1024 * 1024) @@ -224,7 +221,7 @@ def updater(): # --------------------------------------------------------------------------- XXH128_UPDATE_RESET_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh128() BLOCK = b'x' * 16 # tiny block: more calls = more race windows @@ -278,7 +275,7 @@ def reseter(): # --------------------------------------------------------------------------- XXH128_UPDATE_COPY_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh128() copies = [] @@ -337,7 +334,7 @@ def copier(): # --------------------------------------------------------------------------- XXH128_ALL_METHODS_CODE = r""" -import sys, random, threading, xxhash +import sys, random, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh128() BLOCK = b'x' * 32 @@ -386,7 +383,7 @@ def worker(): # --------------------------------------------------------------------------- XXH64_AGGRESSIVE_RACE_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh64() BLOCK = b'x' * 16 @@ -423,9 +420,10 @@ def worker(): class TestThreadSafety(unittest.TestCase): - """Verify that concurrent access to a single hash object does not crash. + """Verify that concurrent access to a single threadsafe hash object works. - With per-object locking in place, concurrent access is now safe. + These tests import ``xxhash.threadsafe`` (not the default ``xxhash`` + module), which uses a per-object lock around all streaming operations. We run each scenario many times (``REPETITIONS``) to verify that no crashes, deadlocks, or unexpected exceptions occur. """ @@ -509,8 +507,7 @@ class TestNonDeterminism(unittest.TestCase): TIMEOUT = int(os.environ.get("XXHASH_TEST_TIMEOUT", "120")) def test_concurrent_update_is_deterministic(self): - """Concurrent update() should be deterministic (FIXED: per-object - lock now prevents data races).""" + """Concurrent update() on a threadsafe object should be deterministic.""" digests = set() for i in range(self.SAMPLES): rc, out, err = _run_in_subprocess( From 330c42c305168d10d190ec9085b3ae2b1591ec62 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 14:55:34 +0800 Subject: [PATCH 06/34] Document thread-safety model and the new xxhash.threadsafe submodule --- CHANGELOG.rst | 8 ++++++++ README.rst | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 577453e..dfe3780 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,14 @@ NEXT - Drop support for Python 3.8 - Remove deprecated ``xxhash.VERSION_TUPLE`` +- Default streaming hash objects (``xxh32``, ``xxh64``, ``xxh3_64``, + ``xxh3_128``) are no longer thread-safe by default; this removes + per-object locking overhead and restores performance as the primary goal +- Add ``xxhash.threadsafe`` submodule for users who need to share a + streaming hash object across threads; it provides the same API with a + per-object lock +- On free-threading (no-GIL) Python builds the default module remains + locked, because there is no GIL to protect the internal xxHash state v3.7.0 2025-04-25 ~~~~~~~~~~~~~~~~~ diff --git a/README.rst b/README.rst index 3ec4e9c..54ce791 100644 --- a/README.rst +++ b/README.rst @@ -251,6 +251,32 @@ And aliases: | xxh128_intdigest = xxh3_128_intdigest | xxh128_hexdigest = xxh3_128_hexdigest +Thread safety +------------- + +The default ``xxhash`` module is optimized for speed. Streaming hash objects +(``xxh32``, ``xxh64``, ``xxh3_64``, ``xxh3_128`` / ``xxh128``) are **not** +thread-safe: do not call ``update()``, ``digest()``, ``copy()``, ``reset()``, +or any other mutating method on the same object from multiple threads without +external synchronization. + +One-shot functions (``xxh32_digest``, ``xxh64_hexdigest``, ``xxh3_128_digest``, +etc.) are stateless and always safe to call concurrently. + +If you need to share a streaming hash object across threads, use the +``xxhash.threadsafe`` submodule. It provides the same API with a per-object +lock that protects all access to the internal xxHash state: + +.. code-block:: python + + >>> from xxhash import threadsafe + >>> h = threadsafe.xxh64() + >>> # h can now be safely updated from multiple threads + +On free-threading (no-GIL) Python builds the default ``xxhash`` module is +also locked, because there is no GIL to protect the internal state. + + Caveats ------- From 6d0bc6839bb681471c464e8cc7963985778a6ce0 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 15:20:35 +0800 Subject: [PATCH 07/34] Unify behaviour across GIL and free-threading builds: default module is always unlocked --- CHANGELOG.rst | 5 +++-- README.rst | 5 +++-- setup.py | 13 +++++-------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dfe3780..5bcd2dc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,8 +12,9 @@ NEXT - Add ``xxhash.threadsafe`` submodule for users who need to share a streaming hash object across threads; it provides the same API with a per-object lock -- On free-threading (no-GIL) Python builds the default module remains - locked, because there is no GIL to protect the internal xxHash state +- Both the default module and ``xxhash.threadsafe`` are provided on + free-threading (no-GIL) Python builds, matching the API on regular GIL + builds v3.7.0 2025-04-25 ~~~~~~~~~~~~~~~~~ diff --git a/README.rst b/README.rst index 54ce791..4496b7e 100644 --- a/README.rst +++ b/README.rst @@ -273,8 +273,9 @@ lock that protects all access to the internal xxHash state: >>> h = threadsafe.xxh64() >>> # h can now be safely updated from multiple threads -On free-threading (no-GIL) Python builds the default ``xxhash`` module is -also locked, because there is no GIL to protect the internal state. +The same two-module split is provided on free-threading (no-GIL) Python +builds: the default module is unlocked, and ``xxhash.threadsafe`` provides a +locked variant. Caveats diff --git a/setup.py b/setup.py index fd9933e..6376ee0 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ import os -import sysconfig from pathlib import Path from setuptools import Extension, setup @@ -14,12 +13,11 @@ source = ["src/_xxhash.c", "deps/xxhash/xxhash.c"] include_dirs = ["deps/xxhash"] -# In free-threading builds the default module must be thread-safe, because -# there is no GIL protecting the internal xxHash state. On regular GIL -# builds the default module is unlocked for maximum performance; users who -# need to share a streaming hash object across threads can use -# xxhash.threadsafe. -_is_free_threading = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) +# The default ``xxhash._xxhash`` extension is built without per-object locks +# for maximum performance. Users who need to share a streaming hash object +# across threads can use ``xxhash._xxhash_threadsafe`` (exposed as the public +# ``xxhash.threadsafe`` submodule), which is compiled from the same source +# with locking enabled. _ext_kwargs = { "sources": source, @@ -30,7 +28,6 @@ ext_modules = [ Extension( "_xxhash", - define_macros=[("XXHASH_WITH_LOCK", "1")] if _is_free_threading else [], **_ext_kwargs, ), Extension( From 5817b29d46674477e42c37c2547254a296802d83 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 15:28:25 +0800 Subject: [PATCH 08/34] Declare both extension variants as Py_MOD_GIL_NOT_USED on Python 3.13+ --- src/_xxhash.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index 890baaf..737d206 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -116,7 +116,7 @@ #endif #define XXHASH_PASTE2(a, b) a ## b #define XXHASH_PASTE(a, b) XXHASH_PASTE2(a, b) -#define XXHASH_PYINIT(name) XXHASH_PASTE(PyInit__, name) +#define XXHASH_PYINIT(name) XXHASH_PASTE(PyInit_, name) #define XXH32_DIGESTSIZE 4 #define XXH32_BLOCKSIZE 16 @@ -2256,13 +2256,11 @@ static PyModuleDef_Slot slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, #endif #if PY_VERSION_HEX >= 0x030d0000 -#ifdef XXHASH_WITH_LOCK - /* Thread-safe variant declares it manages its own synchronisation. */ + /* Both variants manage their own synchronisation guarantees: + * the thread-safe variant uses a per-object lock; the default + * variant requires callers not to share streaming hash objects + * across threads. */ {Py_mod_gil, Py_MOD_GIL_NOT_USED}, -#else - /* Default variant relies on the GIL (or caller synchronisation on free-threading builds). */ - {Py_mod_gil, Py_MOD_GIL_USED}, -#endif #endif {0, NULL} }; From 4e196c9ad7cc57b946e54dab57074251ebeae348 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 15:40:37 +0800 Subject: [PATCH 09/34] Use set literal syntax instead of set([...]) --- xxhash/__init__.py | 4 ++-- xxhash/threadsafe.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/xxhash/__init__.py b/xxhash/__init__.py index 4825901..d1723b5 100644 --- a/xxhash/__init__.py +++ b/xxhash/__init__.py @@ -26,13 +26,13 @@ xxh128_intdigest = xxh3_128_intdigest xxh128_digest = xxh3_128_digest -algorithms_available = set([ +algorithms_available = { "xxh32", "xxh64", "xxh3_64", "xxh128", "xxh3_128", -]) +} algorithms_guaranteed = algorithms_available diff --git a/xxhash/threadsafe.py b/xxhash/threadsafe.py index 0b42c73..0e49785 100644 --- a/xxhash/threadsafe.py +++ b/xxhash/threadsafe.py @@ -24,13 +24,13 @@ xxh128_intdigest = xxh3_128_intdigest xxh128_hexdigest = xxh3_128_hexdigest -algorithms_available = set([ +algorithms_available = { "xxh32", "xxh64", "xxh3_64", "xxh128", "xxh3_128", -]) +} algorithms_guaranteed = algorithms_available From 6b4c2223759a2a85cfbe69e08a52f7dd263375ed Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 15:40:52 +0800 Subject: [PATCH 10/34] Remove empty if (!kwargs) branch in _parse_init_args --- src/_xxhash.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index 737d206..54f5ac8 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -763,9 +763,7 @@ _parse_init_args(PyObject *args, PyObject *kwargs, { Py_ssize_t nargs = PyTuple_GET_SIZE(args); - if (!kwargs) { - /* fast path: no keywords */ - } else { + if (kwargs) { Py_ssize_t pos = 0; PyObject *key, *val; while (PyDict_Next(kwargs, &pos, &key, &val)) { From d6d3c6eca92b659eb3cc938e87d921e6695288d8 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 15:41:05 +0800 Subject: [PATCH 11/34] Fix typo: synchronisation -> synchronization --- src/_xxhash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index 54f5ac8..9901e0b 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -2254,7 +2254,7 @@ static PyModuleDef_Slot slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, #endif #if PY_VERSION_HEX >= 0x030d0000 - /* Both variants manage their own synchronisation guarantees: + /* Both variants manage their own synchronization guarantees: * the thread-safe variant uses a per-object lock; the default * variant requires callers not to share streaming hash objects * across threads. */ From aaace1591ff36200b3a218cbd74463e35e3c1fd1 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 15:41:16 +0800 Subject: [PATCH 12/34] Update XXHASH_DO_UPDATE comment to describe both locked and unlocked paths --- src/_xxhash.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index 9901e0b..ce00838 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -644,9 +644,12 @@ static void PYXXH32_dealloc(PYXXH32Object *self) } /* Macro to generate _do_update for each hash type. - * Matches CPython 3.9-3.12 md5 pattern: release GIL first (for large data), - * then acquire lock, hash, release lock, re-acquire GIL. - * For small data, acquire lock with GIL held (try-then-block if contested). */ + * When XXHASH_WITH_LOCK is defined: matches CPython 3.9-3.12 md5 pattern, + * release GIL first (for large data), then acquire lock, hash, release lock, + * re-acquire GIL. For small data, acquire lock with GIL held + * (try-then-block if contested). + * When XXHASH_WITH_LOCK is not defined: no locking, but still release GIL + * for large data to avoid blocking other threads. */ #define XXHASH_DO_UPDATE(type, update_fn) \ static inline Py_ALWAYS_INLINE void \ PY##type##_do_update(PY##type##Object *self, Py_buffer *buf) \ From c1e5b745a9643aa74257ccea0deeb6d0ae57fbc7 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 15:41:41 +0800 Subject: [PATCH 13/34] Document why build_ext restores self.build_temp in try/finally --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 6376ee0..9b4d77d 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,9 @@ class build_ext(_build_ext): Both extensions are built from the same ``src/_xxhash.c`` source file. Without separate temp directories their object files would overwrite each other, causing one variant to be linked with the wrong macros. + + ``try/finally`` restores ``self.build_temp`` so that incremental builds + (where ``build_ext`` may be reused) still work correctly. """ def build_extension(self, ext): From 155c862b9bc72038ef57d51910fa4fbf54023110 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 15:47:25 +0800 Subject: [PATCH 14/34] Macro-generate the 4 update() methods via XXHASH_UPDATE_METHOD PYXXH32_update, PYXXH64_update, PYXXH3_64_update and PYXXH3_128_update were identical except for type prefixes and algorithm names. The new macro eliminates all four bodies, saving 148 lines. --- src/_xxhash.c | 268 +++++++++++--------------------------------------- 1 file changed, 60 insertions(+), 208 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index ce00838..92cb0b0 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -853,58 +853,63 @@ static int PY##type##_init(PY##type##Object *self, PyObject *args, \ XXHASH_INIT(XXH32, XXH32_reset, XXH32_update, XXH32_hash_t) -PyDoc_STRVAR( - PYXXH32_update_doc, - "update (data)\n\n" - "Update the xxh32 object with bytes-like data. Repeated calls are\n" - "equivalent to a single call with the concatenation of all the arguments."); - -static PyObject *PYXXH32_update(PYXXH32Object *self, PyObject *const *args, - Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *arg = NULL; - - /* validate keywords first */ - if (kwnames) { - Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < nkw; i++) { - PyObject *key = PyTuple_GET_ITEM(kwnames, i); - if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { - if (nargs >= 1) { - PyErr_SetString(PyExc_TypeError, - "xxh32.update() got multiple values for argument 'data'"); - return NULL; - } - arg = args[nargs + i]; - } else { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword argument for 'xxh32.update()'", - key); - return NULL; - } - } - } - - if (nargs >= 1) { - if (nargs > 1) { - PyErr_Format(PyExc_TypeError, - "xxh32.update() takes at most 1 positional argument (%zd given)", nargs); - return NULL; - } - arg = args[0]; - } - - if (!arg) { - PyErr_SetString(PyExc_TypeError, "xxh32.update() missing required argument 'data'"); - return NULL; - } - - Py_buffer buf; - if (_get_buffer_or_str(arg, &buf) < 0) - return NULL; - PYXXH32_do_update(self, &buf); - Py_RETURN_NONE; -} +#define XXHASH_UPDATE_METHOD(prefix, name) \ +PyDoc_STRVAR( \ + PY##prefix##_update_doc, \ + "update (data)\n\n" \ + "Update the " name " object with bytes-like data. Repeated calls are\n" \ + "equivalent to a single call with the concatenation of all the arguments."); \ + \ +static PyObject *PY##prefix##_update(PY##prefix##Object *self, \ + PyObject *const *args, \ + Py_ssize_t nargs, PyObject *kwnames) \ +{ \ + PyObject *arg = NULL; \ + \ + /* validate keywords first */ \ + if (kwnames) { \ + Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); \ + for (Py_ssize_t i = 0; i < nkw; i++) { \ + PyObject *key = PyTuple_GET_ITEM(kwnames, i); \ + if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { \ + if (nargs >= 1) { \ + PyErr_SetString(PyExc_TypeError, \ + name ".update() got multiple values for argument 'data'"); \ + return NULL; \ + } \ + arg = args[nargs + i]; \ + } else { \ + PyErr_Format(PyExc_TypeError, \ + "'%U' is an invalid keyword argument for '" name ".update()'", \ + key); \ + return NULL; \ + } \ + } \ + } \ + \ + if (nargs >= 1) { \ + if (nargs > 1) { \ + PyErr_Format(PyExc_TypeError, \ + name ".update() takes at most 1 positional argument (%zd given)", nargs); \ + return NULL; \ + } \ + arg = args[0]; \ + } \ + \ + if (!arg) { \ + PyErr_SetString(PyExc_TypeError, \ + name ".update() missing required argument 'data'"); \ + return NULL; \ + } \ + \ + Py_buffer buf; \ + if (_get_buffer_or_str(arg, &buf) < 0) \ + return NULL; \ + PY##prefix##_do_update(self, &buf); \ + Py_RETURN_NONE; \ +} + +XXHASH_UPDATE_METHOD(XXH32, "xxh32") PyDoc_STRVAR( PYXXH32_digest_doc, @@ -1205,58 +1210,7 @@ static PyObject *PYXXH64_new(PyTypeObject *type, PyObject *args, PyObject *kwarg XXHASH_INIT(XXH64, XXH64_reset, XXH64_update, XXH64_hash_t) -PyDoc_STRVAR( - PYXXH64_update_doc, - "update (data)\n\n" - "Update the xxh64 object with bytes-like data. Repeated calls are\n" - "equivalent to a single call with the concatenation of all the arguments."); - -static PyObject *PYXXH64_update(PYXXH64Object *self, PyObject *const *args, - Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *arg = NULL; - - /* validate keywords first */ - if (kwnames) { - Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < nkw; i++) { - PyObject *key = PyTuple_GET_ITEM(kwnames, i); - if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { - if (nargs >= 1) { - PyErr_SetString(PyExc_TypeError, - "xxh64.update() got multiple values for argument 'data'"); - return NULL; - } - arg = args[nargs + i]; - } else { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword argument for 'xxh64.update()'", - key); - return NULL; - } - } - } - - if (nargs >= 1) { - if (nargs > 1) { - PyErr_Format(PyExc_TypeError, - "xxh64.update() takes at most 1 positional argument (%zd given)", nargs); - return NULL; - } - arg = args[0]; - } - - if (!arg) { - PyErr_SetString(PyExc_TypeError, "xxh64.update() missing required argument 'data'"); - return NULL; - } - - Py_buffer buf; - if (_get_buffer_or_str(arg, &buf) < 0) - return NULL; - PYXXH64_do_update(self, &buf); - Py_RETURN_NONE; -} +XXHASH_UPDATE_METHOD(XXH64, "xxh64") PyDoc_STRVAR( PYXXH64_digest_doc, @@ -1557,58 +1511,7 @@ static PyObject *PYXXH3_64_new(PyTypeObject *type, PyObject *args, PyObject *kwa XXHASH_INIT(XXH3_64, XXH3_64bits_reset_withSeed, XXH3_64bits_update, XXH64_hash_t) -PyDoc_STRVAR( - PYXXH3_64_update_doc, - "update (data)\n\n" - "Update the xxh3_64 object with bytes-like data. Repeated calls are\n" - "equivalent to a single call with the concatenation of all the arguments."); - -static PyObject *PYXXH3_64_update(PYXXH3_64Object *self, PyObject *const *args, - Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *arg = NULL; - - /* validate keywords first */ - if (kwnames) { - Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < nkw; i++) { - PyObject *key = PyTuple_GET_ITEM(kwnames, i); - if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { - if (nargs >= 1) { - PyErr_SetString(PyExc_TypeError, - "xxh3_64.update() got multiple values for argument 'data'"); - return NULL; - } - arg = args[nargs + i]; - } else { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword argument for 'xxh3_64.update()'", - key); - return NULL; - } - } - } - - if (nargs >= 1) { - if (nargs > 1) { - PyErr_Format(PyExc_TypeError, - "xxh3_64.update() takes at most 1 positional argument (%zd given)", nargs); - return NULL; - } - arg = args[0]; - } - - if (!arg) { - PyErr_SetString(PyExc_TypeError, "xxh3_64.update() missing required argument 'data'"); - return NULL; - } - - Py_buffer buf; - if (_get_buffer_or_str(arg, &buf) < 0) - return NULL; - PYXXH3_64_do_update(self, &buf); - Py_RETURN_NONE; -} +XXHASH_UPDATE_METHOD(XXH3_64, "xxh3_64") PyDoc_STRVAR( PYXXH3_64_digest_doc, @@ -1916,58 +1819,7 @@ static PyObject *PYXXH3_128_new(PyTypeObject *type, PyObject *args, PyObject *kw XXHASH_INIT(XXH3_128, XXH3_128bits_reset_withSeed, XXH3_128bits_update, XXH64_hash_t) -PyDoc_STRVAR( - PYXXH3_128_update_doc, - "update (data)\n\n" - "Update the xxh3_128 object with bytes-like data. Repeated calls are\n" - "equivalent to a single call with the concatenation of all the arguments."); - -static PyObject *PYXXH3_128_update(PYXXH3_128Object *self, PyObject *const *args, - Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *arg = NULL; - - /* validate keywords first */ - if (kwnames) { - Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < nkw; i++) { - PyObject *key = PyTuple_GET_ITEM(kwnames, i); - if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { - if (nargs >= 1) { - PyErr_SetString(PyExc_TypeError, - "xxh3_128.update() got multiple values for argument 'data'"); - return NULL; - } - arg = args[nargs + i]; - } else { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword argument for 'xxh3_128.update()'", - key); - return NULL; - } - } - } - - if (nargs >= 1) { - if (nargs > 1) { - PyErr_Format(PyExc_TypeError, - "xxh3_128.update() takes at most 1 positional argument (%zd given)", nargs); - return NULL; - } - arg = args[0]; - } - - if (!arg) { - PyErr_SetString(PyExc_TypeError, "xxh3_128.update() missing required argument 'data'"); - return NULL; - } - - Py_buffer buf; - if (_get_buffer_or_str(arg, &buf) < 0) - return NULL; - PYXXH3_128_do_update(self, &buf); - Py_RETURN_NONE; -} +XXHASH_UPDATE_METHOD(XXH3_128, "xxh3_128") PyDoc_STRVAR( PYXXH3_128_digest_doc, From 83af085f37841ce2482e56083e2f4ae2656e731a Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 16:34:05 +0800 Subject: [PATCH 15/34] Align trailing backslashes in multi-line macros --- src/_xxhash.c | 118 +++++++++++++++++++++++++------------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index 92cb0b0..8fabae0 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -50,7 +50,7 @@ # define XXHASH_LOCK_INIT(o) ((o)->lock = NULL) # define XXHASH_LOCK_IS_ACTIVE(o) ((o)->lock != NULL) /* Lazy allocation on first large update */ -# define XXHASH_LOCK_MAYBE_INIT(o, len) \ +# define XXHASH_LOCK_MAYBE_INIT(o, len) \ do { \ if ((o)->lock == NULL && (len) >= XXHASH_GIL_MINSIZE) { \ (o)->lock = PyThread_allocate_lock(); \ @@ -62,7 +62,7 @@ } while (0) /* Acquire lock when GIL is already released — simple blocking acquire. * Only acquires if lock has been allocated (lazy init). */ -# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) \ +# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) \ do { \ if ((o)->lock) { \ PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ @@ -72,7 +72,7 @@ /* Acquire lock with the GIL held — non-blocking try first, then release * GIL and block if contested (matches hashlib's ENTER_HASHLIB in 3.9-3.12). * Only acquires if lock has been allocated (lazy init). */ -# define XXHASH_LOCK_ACQUIRE(o) \ +# define XXHASH_LOCK_ACQUIRE(o) \ do { \ if ((o)->lock) { \ if (!PyThread_acquire_lock((o)->lock, NOWAIT_LOCK)) { \ @@ -84,11 +84,11 @@ } \ } while (0) -# define XXHASH_LOCK_RELEASE(o) \ +# define XXHASH_LOCK_RELEASE(o) \ do { \ if ((o)->lock) { \ PyThread_release_lock((o)->lock); \ - } \ + } \ } while (0) # endif #else /* !XXHASH_WITH_LOCK */ @@ -670,7 +670,7 @@ PY##type##_do_update(PY##type##Object *self, Py_buffer *buf) \ XXHASH_LOCK_RELEASE(self); \ } \ } else { \ - /* No lock: release GIL for large data to avoid blocking other threads. */ \ + /* No lock */ \ if (buf->len > XXHASH_GIL_MINSIZE) { \ Py_BEGIN_ALLOW_THREADS \ update_fn(self->xxhash_state, buf->buf, buf->len); \ @@ -853,60 +853,60 @@ static int PY##type##_init(PY##type##Object *self, PyObject *args, \ XXHASH_INIT(XXH32, XXH32_reset, XXH32_update, XXH32_hash_t) -#define XXHASH_UPDATE_METHOD(prefix, name) \ -PyDoc_STRVAR( \ - PY##prefix##_update_doc, \ - "update (data)\n\n" \ - "Update the " name " object with bytes-like data. Repeated calls are\n" \ - "equivalent to a single call with the concatenation of all the arguments."); \ - \ -static PyObject *PY##prefix##_update(PY##prefix##Object *self, \ - PyObject *const *args, \ - Py_ssize_t nargs, PyObject *kwnames) \ -{ \ - PyObject *arg = NULL; \ - \ - /* validate keywords first */ \ - if (kwnames) { \ - Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); \ - for (Py_ssize_t i = 0; i < nkw; i++) { \ - PyObject *key = PyTuple_GET_ITEM(kwnames, i); \ - if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { \ - if (nargs >= 1) { \ - PyErr_SetString(PyExc_TypeError, \ - name ".update() got multiple values for argument 'data'"); \ - return NULL; \ - } \ - arg = args[nargs + i]; \ - } else { \ - PyErr_Format(PyExc_TypeError, \ - "'%U' is an invalid keyword argument for '" name ".update()'", \ - key); \ - return NULL; \ - } \ - } \ - } \ - \ - if (nargs >= 1) { \ - if (nargs > 1) { \ - PyErr_Format(PyExc_TypeError, \ +#define XXHASH_UPDATE_METHOD(prefix, name) \ +PyDoc_STRVAR( \ + PY##prefix##_update_doc, \ + "update (data)\n\n" \ + "Update the " name " object with bytes-like data. Repeated calls are\n" \ + "equivalent to a single call with the concatenation of all the arguments."); \ + \ +static PyObject *PY##prefix##_update(PY##prefix##Object *self, \ + PyObject *const *args, \ + Py_ssize_t nargs, PyObject *kwnames) \ +{ \ + PyObject *arg = NULL; \ + \ + /* validate keywords first */ \ + if (kwnames) { \ + Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); \ + for (Py_ssize_t i = 0; i < nkw; i++) { \ + PyObject *key = PyTuple_GET_ITEM(kwnames, i); \ + if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { \ + if (nargs >= 1) { \ + PyErr_SetString(PyExc_TypeError, \ + name ".update() got multiple values for argument 'data'"); \ + return NULL; \ + } \ + arg = args[nargs + i]; \ + } else { \ + PyErr_Format(PyExc_TypeError, \ + "'%U' is an invalid keyword argument for '" name ".update()'", \ + key); \ + return NULL; \ + } \ + } \ + } \ + \ + if (nargs >= 1) { \ + if (nargs > 1) { \ + PyErr_Format(PyExc_TypeError, \ name ".update() takes at most 1 positional argument (%zd given)", nargs); \ - return NULL; \ - } \ - arg = args[0]; \ - } \ - \ - if (!arg) { \ - PyErr_SetString(PyExc_TypeError, \ - name ".update() missing required argument 'data'"); \ - return NULL; \ - } \ - \ - Py_buffer buf; \ - if (_get_buffer_or_str(arg, &buf) < 0) \ - return NULL; \ - PY##prefix##_do_update(self, &buf); \ - Py_RETURN_NONE; \ + return NULL; \ + } \ + arg = args[0]; \ + } \ + \ + if (!arg) { \ + PyErr_SetString(PyExc_TypeError, \ + name ".update() missing required argument 'data'"); \ + return NULL; \ + } \ + \ + Py_buffer buf; \ + if (_get_buffer_or_str(arg, &buf) < 0) \ + return NULL; \ + PY##prefix##_do_update(self, &buf); \ + Py_RETURN_NONE; \ } XXHASH_UPDATE_METHOD(XXH32, "xxh32") From e8d9cb29a02784ed7abfe4d51698645e58ea9dc8 Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 16:36:51 +0800 Subject: [PATCH 16/34] Remove Py_ALWAYS_INLINE --- src/_xxhash.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index 8fabae0..5c3b025 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -125,13 +125,10 @@ #define XXH128_DIGESTSIZE 16 #define XXH128_BLOCKSIZE 64 -#ifndef Py_ALWAYS_INLINE -# define Py_ALWAYS_INLINE -#endif /* Hex lookup table for hexdigest(). */ /* Get a buffer from an object. Rejects str with hashlib-compatible error. */ -static inline Py_ALWAYS_INLINE int +static inline int _get_buffer_or_str(PyObject *obj, Py_buffer *buf) { if (obj == Py_None) { @@ -651,7 +648,7 @@ static void PYXXH32_dealloc(PYXXH32Object *self) * When XXHASH_WITH_LOCK is not defined: no locking, but still release GIL * for large data to avoid blocking other threads. */ #define XXHASH_DO_UPDATE(type, update_fn) \ -static inline Py_ALWAYS_INLINE void \ +static inline void \ PY##type##_do_update(PY##type##Object *self, Py_buffer *buf) \ { \ XXHASH_LOCK_MAYBE_INIT(self, buf->len); \ From be06f39295b93437820fc0b8bd50005c29466bed Mon Sep 17 00:00:00 2001 From: duyue Date: Sat, 20 Jun 2026 16:42:23 +0800 Subject: [PATCH 17/34] Discourage concurrent update/reset even with xxhash.threadsafe --- README.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 4496b7e..c1abbb1 100644 --- a/README.rst +++ b/README.rst @@ -263,15 +263,18 @@ external synchronization. One-shot functions (``xxh32_digest``, ``xxh64_hexdigest``, ``xxh3_128_digest``, etc.) are stateless and always safe to call concurrently. -If you need to share a streaming hash object across threads, use the +Concurrent ``update()`` / ``reset()`` on a shared streaming hash object is +discouraged even with locking — prefer one-shot functions or per-thread hash +objects. If you must share a streaming hash across threads, use the ``xxhash.threadsafe`` submodule. It provides the same API with a per-object -lock that protects all access to the internal xxHash state: +lock that serializes all access to the internal xxHash state: .. code-block:: python >>> from xxhash import threadsafe >>> h = threadsafe.xxh64() - >>> # h can now be safely updated from multiple threads + >>> # h can be updated from multiple threads, but concurrent update/reset + >>> # still adds overhead and is not recommended The same two-module split is provided on free-threading (no-GIL) Python builds: the default module is unlocked, and ``xxhash.threadsafe`` provides a From 07e9186086c7f7f57795fa20fc261c2b3d7d4301 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Mon, 22 Jun 2026 15:48:41 +0800 Subject: [PATCH 18/34] fix(xxhash/__init__.pyi): replace xxh64_* wrong aliases with proper declarations The stub file xxhash/__init__.pyi incorrectly defined xxh64_digest, xxh64_hexdigest, xxh64_intdigest as aliases of xxh3_64_* functions. xxh64 and xxh3_64 are separate algorithms with potentially different hash outputs, so they should have independent function declarations. This aligns with the already-correct threadsafe.pyi. --- xxhash/__init__.pyi | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xxhash/__init__.pyi b/xxhash/__init__.pyi index 91741d2..e90bffe 100644 --- a/xxhash/__init__.pyi +++ b/xxhash/__init__.pyi @@ -76,6 +76,10 @@ def xxh32_digest(data: _DataType, seed: int = ...) -> bytes: ... def xxh32_hexdigest(data: _DataType, seed: int = ...) -> str: ... def xxh32_intdigest(data: _DataType, seed: int = ...) -> int: ... +def xxh64_digest(data: _DataType, seed: int = ...) -> bytes: ... +def xxh64_hexdigest(data: _DataType, seed: int = ...) -> str: ... +def xxh64_intdigest(data: _DataType, seed: int = ...) -> int: ... + def xxh3_64_digest(data: _DataType, seed: int = ...) -> bytes: ... def xxh3_64_hexdigest(data: _DataType, seed: int = ...) -> str: ... def xxh3_64_intdigest(data: _DataType, seed: int = ...) -> int: ... @@ -84,10 +88,6 @@ def xxh3_128_digest(data: _DataType, seed: int = ...) -> bytes: ... def xxh3_128_hexdigest(data: _DataType, seed: int = ...) -> str: ... def xxh3_128_intdigest(data: _DataType, seed: int = ...) -> int: ... -xxh64_digest = xxh3_64_digest -xxh64_hexdigest = xxh3_64_hexdigest -xxh64_intdigest = xxh3_64_intdigest - xxh128_digest = xxh3_128_digest xxh128_hexdigest = xxh3_128_hexdigest xxh128_intdigest = xxh3_128_intdigest From 501a1f76b3c95eecb05368af5bda4aa2e0cd1a6c Mon Sep 17 00:00:00 2001 From: Yue Du Date: Mon, 22 Jun 2026 15:53:16 +0800 Subject: [PATCH 19/34] fix(_xxhash.c): always allocate lock at init for Python 3.9-3.12 The previous lazy allocation in XXHASH_LOCK_MAYBE_INIT only created a PyThread_type_lock if update() was called with data >= 64KB. This meant that if a thread-safe hash object was only used with small data (or never called update()), digest(), copy(), reset(), etc. all ran without any synchronization because XXHASH_LOCK_ACQUIRE was a no-op when lock == NULL. Fix: always allocate the lock at initialization via XXHASH_LOCK_INIT, matching hashlib's approach of always-initialized mutex (HASHLIB_INIT_MUTEX). This eliminates the security window entirely. Changes: - XXHASH_LOCK_INIT now calls PyThread_allocate_lock() instead of setting NULL - XXHASH_LOCK_MAYBE_INIT reduced to a no-op (lock always allocated) - Simplified _do_update by removing the lazy init call and dead 'no lock' branch - locking is now unconditional (no-op in non-lock build) - NULL guards in acquire/release/fini retained as defensive safety Python 3.13+ (PyMutex path) was not affected as it was already always-on. --- src/_xxhash.c | 66 +++++++++++++++++---------------------------------- 1 file changed, 22 insertions(+), 44 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index 5c3b025..4a0fe05 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -45,23 +45,18 @@ # define XXHASH_LOCK_ACQUIRE(o) PyMutex_Lock(&(o)->mutex) # define XXHASH_LOCK_ACQUIRE_BLOCKING(o) XXHASH_LOCK_ACQUIRE(o) # define XXHASH_LOCK_RELEASE(o) PyMutex_Unlock(&(o)->mutex) -# else /* Python 3.9-3.12: PyThread_type_lock */ +# else /* Python 3.9-3.12: PyThread_type_lock (always-on, no lazy init) */ # define XXHASH_LOCK_FIELD PyThread_type_lock lock; -# define XXHASH_LOCK_INIT(o) ((o)->lock = NULL) +# define XXHASH_LOCK_INIT(o) ((o)->lock = PyThread_allocate_lock()) +/* Always-on: lock is allocated in __init__, matching hashlib's approach. + * NULL is only possible if memory allocation fails; guards are defensive. */ # define XXHASH_LOCK_IS_ACTIVE(o) ((o)->lock != NULL) -/* Lazy allocation on first large update */ -# define XXHASH_LOCK_MAYBE_INIT(o, len) \ - do { \ - if ((o)->lock == NULL && (len) >= XXHASH_GIL_MINSIZE) { \ - (o)->lock = PyThread_allocate_lock(); \ - /* fail? lock stays NULL, fall back to non-threaded code. */ \ - } \ - } while (0) +/* Lazy init removed: lock is always allocated. Safety check retained. */ +# define XXHASH_LOCK_MAYBE_INIT(o, len) ((void)0) # define XXHASH_LOCK_FINI(o) do { if ((o)->lock) \ PyThread_free_lock((o)->lock); \ } while (0) -/* Acquire lock when GIL is already released — simple blocking acquire. - * Only acquires if lock has been allocated (lazy init). */ +/* Acquire lock when GIL is already released — simple blocking acquire. */ # define XXHASH_LOCK_ACQUIRE_BLOCKING(o) \ do { \ if ((o)->lock) { \ @@ -70,13 +65,12 @@ } while (0) /* Acquire lock with the GIL held — non-blocking try first, then release - * GIL and block if contested (matches hashlib's ENTER_HASHLIB in 3.9-3.12). - * Only acquires if lock has been allocated (lazy init). */ + * GIL and block if contested (matches hashlib's ENTER_HASHLIB in 3.9-3.12). */ # define XXHASH_LOCK_ACQUIRE(o) \ do { \ if ((o)->lock) { \ if (!PyThread_acquire_lock((o)->lock, NOWAIT_LOCK)) { \ - /* Lock contested – release GIL while waiting. */ \ + /* Lock contested — release GIL while waiting. */ \ Py_BEGIN_ALLOW_THREADS \ PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ Py_END_ALLOW_THREADS \ @@ -641,40 +635,24 @@ static void PYXXH32_dealloc(PYXXH32Object *self) } /* Macro to generate _do_update for each hash type. - * When XXHASH_WITH_LOCK is defined: matches CPython 3.9-3.12 md5 pattern, - * release GIL first (for large data), then acquire lock, hash, release lock, - * re-acquire GIL. For small data, acquire lock with GIL held - * (try-then-block if contested). - * When XXHASH_WITH_LOCK is not defined: no locking, but still release GIL - * for large data to avoid blocking other threads. */ + * Lock is always acquired (no-op in non-lock build). For large data, + * release GIL while blocking on lock to avoid stalling other threads. */ #define XXHASH_DO_UPDATE(type, update_fn) \ static inline void \ PY##type##_do_update(PY##type##Object *self, Py_buffer *buf) \ { \ - XXHASH_LOCK_MAYBE_INIT(self, buf->len); \ - if (XXHASH_LOCK_IS_ACTIVE(self)) { \ - if (buf->len > XXHASH_GIL_MINSIZE) { \ - /* Release GIL first, then acquire lock. */ \ - Py_BEGIN_ALLOW_THREADS \ - XXHASH_LOCK_ACQUIRE_BLOCKING(self); \ - update_fn(self->xxhash_state, buf->buf, buf->len); \ - XXHASH_LOCK_RELEASE(self); \ - Py_END_ALLOW_THREADS \ - } else { \ - /* Acquire lock with GIL held. */ \ - XXHASH_LOCK_ACQUIRE(self); \ - update_fn(self->xxhash_state, buf->buf, buf->len); \ - XXHASH_LOCK_RELEASE(self); \ - } \ + if (buf->len > XXHASH_GIL_MINSIZE) { \ + /* Release GIL first, then acquire lock. */ \ + Py_BEGIN_ALLOW_THREADS \ + XXHASH_LOCK_ACQUIRE_BLOCKING(self); \ + update_fn(self->xxhash_state, buf->buf, buf->len); \ + XXHASH_LOCK_RELEASE(self); \ + Py_END_ALLOW_THREADS \ } else { \ - /* No lock */ \ - if (buf->len > XXHASH_GIL_MINSIZE) { \ - Py_BEGIN_ALLOW_THREADS \ - update_fn(self->xxhash_state, buf->buf, buf->len); \ - Py_END_ALLOW_THREADS \ - } else { \ - update_fn(self->xxhash_state, buf->buf, buf->len); \ - } \ + /* Acquire lock with GIL held (try-then-block if contested). */ \ + XXHASH_LOCK_ACQUIRE(self); \ + update_fn(self->xxhash_state, buf->buf, buf->len); \ + XXHASH_LOCK_RELEASE(self); \ } \ PyBuffer_Release(buf); \ } From 863972418ae9d026752b9a73423c308fd4ede5dd Mon Sep 17 00:00:00 2001 From: Yue Du Date: Mon, 22 Jun 2026 16:11:19 +0800 Subject: [PATCH 20/34] refactor: align lock design with CPython 3.15's HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED Two changes: 1. Simplify lock macros to match 3.15's unconditional acquire/release: - PyMutex path (3.13+): retain only FIELD/INIT/FINI/ACQUIRE/RELEASE; remove IS_ACTIVE, MAYBE_INIT, ACQUIRE_BLOCKING (no longer needed) - PyThread_type_lock path (3.9-3.12): ACQUIRE/RELEASE are now unconditional blocking calls (no NULL guards, no try-then-block), matching 3.15's HASHLIB_ACQUIRE_LOCK - Non-lock path: keep only mandatory macros 2. Simplify _do_update to 3.15's pattern: - Both large and small data paths use the SAME XXHASH_LOCK_ACQUIRE - Size threshold (XXHASH_GIL_MINSIZE) only controls GIL release, NOT whether locking happens - Matches 3.15's HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED exactly --- src/_xxhash.c | 50 ++++++++------------------------------------------ 1 file changed, 8 insertions(+), 42 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index 4a0fe05..8046c70 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -39,60 +39,26 @@ # if PY_VERSION_HEX >= 0x030d0000 /* Python 3.13+: always-on PyMutex (3.15+ style) */ # define XXHASH_LOCK_FIELD PyMutex mutex; # define XXHASH_LOCK_INIT(o) ((void)((o)->mutex = (PyMutex){0})) -# define XXHASH_LOCK_IS_ACTIVE(o) 1 -# define XXHASH_LOCK_MAYBE_INIT(o, len) ((void)0) # define XXHASH_LOCK_FINI(o) ((void)0) # define XXHASH_LOCK_ACQUIRE(o) PyMutex_Lock(&(o)->mutex) -# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) XXHASH_LOCK_ACQUIRE(o) # define XXHASH_LOCK_RELEASE(o) PyMutex_Unlock(&(o)->mutex) -# else /* Python 3.9-3.12: PyThread_type_lock (always-on, no lazy init) */ +# else /* Python 3.9-3.12: PyThread_type_lock (always-on) */ # define XXHASH_LOCK_FIELD PyThread_type_lock lock; # define XXHASH_LOCK_INIT(o) ((o)->lock = PyThread_allocate_lock()) -/* Always-on: lock is allocated in __init__, matching hashlib's approach. - * NULL is only possible if memory allocation fails; guards are defensive. */ -# define XXHASH_LOCK_IS_ACTIVE(o) ((o)->lock != NULL) -/* Lazy init removed: lock is always allocated. Safety check retained. */ -# define XXHASH_LOCK_MAYBE_INIT(o, len) ((void)0) # define XXHASH_LOCK_FINI(o) do { if ((o)->lock) \ PyThread_free_lock((o)->lock); \ } while (0) -/* Acquire lock when GIL is already released — simple blocking acquire. */ -# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) \ - do { \ - if ((o)->lock) { \ - PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ - } \ - } while (0) - -/* Acquire lock with the GIL held — non-blocking try first, then release - * GIL and block if contested (matches hashlib's ENTER_HASHLIB in 3.9-3.12). */ -# define XXHASH_LOCK_ACQUIRE(o) \ - do { \ - if ((o)->lock) { \ - if (!PyThread_acquire_lock((o)->lock, NOWAIT_LOCK)) { \ - /* Lock contested — release GIL while waiting. */ \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ - Py_END_ALLOW_THREADS \ - } \ - } \ - } while (0) - -# define XXHASH_LOCK_RELEASE(o) \ - do { \ - if ((o)->lock) { \ - PyThread_release_lock((o)->lock); \ - } \ - } while (0) +/* Unconditional blocking acquire, matching 3.15's HASHLIB_ACQUIRE_LOCK. + * Under GIL: if lock contended, the holder has released GIL (large data path), + * so we block here without risking deadlock. */ +# define XXHASH_LOCK_ACQUIRE(o) PyThread_acquire_lock((o)->lock, WAIT_LOCK) +# define XXHASH_LOCK_RELEASE(o) PyThread_release_lock((o)->lock) # endif #else /* !XXHASH_WITH_LOCK */ # define XXHASH_LOCK_FIELD # define XXHASH_LOCK_INIT(o) ((void)0) -# define XXHASH_LOCK_IS_ACTIVE(o) 0 -# define XXHASH_LOCK_MAYBE_INIT(o, len) ((void)0) # define XXHASH_LOCK_FINI(o) ((void)0) # define XXHASH_LOCK_ACQUIRE(o) ((void)0) -# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) ((void)0) # define XXHASH_LOCK_RELEASE(o) ((void)0) #endif @@ -644,12 +610,12 @@ PY##type##_do_update(PY##type##Object *self, Py_buffer *buf) \ if (buf->len > XXHASH_GIL_MINSIZE) { \ /* Release GIL first, then acquire lock. */ \ Py_BEGIN_ALLOW_THREADS \ - XXHASH_LOCK_ACQUIRE_BLOCKING(self); \ + XXHASH_LOCK_ACQUIRE(self); \ update_fn(self->xxhash_state, buf->buf, buf->len); \ XXHASH_LOCK_RELEASE(self); \ Py_END_ALLOW_THREADS \ } else { \ - /* Acquire lock with GIL held (try-then-block if contested). */ \ + /* Acquire lock with GIL held. */ \ XXHASH_LOCK_ACQUIRE(self); \ update_fn(self->xxhash_state, buf->buf, buf->len); \ XXHASH_LOCK_RELEASE(self); \ From 54b7d5fbff9cdb321cd091f64b7bb5e10ebe9059 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Mon, 22 Jun 2026 16:20:42 +0800 Subject: [PATCH 21/34] fix: restore try-then-block for PyThread_type_lock to prevent deadlock under GIL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 4443f28 incorrectly simplified the 3.9-3.12 PyThread_type_lock path to unconditional WAIT_LOCK. This is unsafe under GIL: PyThread_acquire_lock(lock, WAIT_LOCK) does NOT release the GIL while waiting, unlike PyMutex_Lock (3.13+) which passes _PY_LOCK_DETACH and releases GIL when parking. Deadlock scenario: 1. T1 holds object lock (large data path, GIL released) 2. T2 holds GIL, enters small data path, calls WAIT_LOCK → blocks with GIL 3. T1 finishes hash, tries to reacquire GIL → blocked 4. T2 waits for object lock, T1 waits for GIL → deadlock Fix: restore the try-then-block in XXHASH_LOCK_ACQUIRE (used under GIL): - First try NOWAIT_LOCK (fast path) - If contested, release GIL (Py_BEGIN_ALLOW_THREADS), then WAIT_LOCK - Reacquire GIL (Py_END_ALLOW_THREADS) XXHASH_LOCK_ACQUIRE_BLOCKING (used without GIL, large data path) keeps simple WAIT_LOCK since GIL is already released. 3.13+ PyMutex path remains unconditional (safe due to _PY_LOCK_DETACH). --- src/_xxhash.c | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index 8046c70..fca3345 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -41,6 +41,7 @@ # define XXHASH_LOCK_INIT(o) ((void)((o)->mutex = (PyMutex){0})) # define XXHASH_LOCK_FINI(o) ((void)0) # define XXHASH_LOCK_ACQUIRE(o) PyMutex_Lock(&(o)->mutex) +# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) XXHASH_LOCK_ACQUIRE(o) # define XXHASH_LOCK_RELEASE(o) PyMutex_Unlock(&(o)->mutex) # else /* Python 3.9-3.12: PyThread_type_lock (always-on) */ # define XXHASH_LOCK_FIELD PyThread_type_lock lock; @@ -48,17 +49,44 @@ # define XXHASH_LOCK_FINI(o) do { if ((o)->lock) \ PyThread_free_lock((o)->lock); \ } while (0) -/* Unconditional blocking acquire, matching 3.15's HASHLIB_ACQUIRE_LOCK. - * Under GIL: if lock contended, the holder has released GIL (large data path), - * so we block here without risking deadlock. */ -# define XXHASH_LOCK_ACQUIRE(o) PyThread_acquire_lock((o)->lock, WAIT_LOCK) -# define XXHASH_LOCK_RELEASE(o) PyThread_release_lock((o)->lock) +/* Acquire lock when GIL is already released — simple blocking acquire. */ +# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) \ + do { \ + if ((o)->lock) { \ + PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ + } \ + } while (0) + +/* Acquire lock with the GIL held — non-blocking try first, then release + * GIL and block if contested (matches hashlib's ENTER_HASHLIB in 3.9-3.12). + * WAIT_LOCK under GIL would deadlock: if another thread holds the object lock + * (large data path, GIL released), this thread blocks with GIL held, preventing + * the holder from re-acquiring GIL to release the lock. */ +# define XXHASH_LOCK_ACQUIRE(o) \ + do { \ + if ((o)->lock) { \ + if (!PyThread_acquire_lock((o)->lock, NOWAIT_LOCK)) { \ + /* Lock contested — release GIL while waiting. */ \ + Py_BEGIN_ALLOW_THREADS \ + PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ + Py_END_ALLOW_THREADS \ + } \ + } \ + } while (0) + +# define XXHASH_LOCK_RELEASE(o) \ + do { \ + if ((o)->lock) { \ + PyThread_release_lock((o)->lock); \ + } \ + } while (0) # endif -#else /* !XXHASH_WITH_LOCK */ +# else /* !XXHASH_WITH_LOCK */ # define XXHASH_LOCK_FIELD # define XXHASH_LOCK_INIT(o) ((void)0) # define XXHASH_LOCK_FINI(o) ((void)0) # define XXHASH_LOCK_ACQUIRE(o) ((void)0) +# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) ((void)0) # define XXHASH_LOCK_RELEASE(o) ((void)0) #endif @@ -608,14 +636,14 @@ static inline void \ PY##type##_do_update(PY##type##Object *self, Py_buffer *buf) \ { \ if (buf->len > XXHASH_GIL_MINSIZE) { \ - /* Release GIL first, then acquire lock. */ \ + /* Release GIL first, then acquire lock (blocking, no GIL). */ \ Py_BEGIN_ALLOW_THREADS \ - XXHASH_LOCK_ACQUIRE(self); \ + XXHASH_LOCK_ACQUIRE_BLOCKING(self); \ update_fn(self->xxhash_state, buf->buf, buf->len); \ XXHASH_LOCK_RELEASE(self); \ Py_END_ALLOW_THREADS \ } else { \ - /* Acquire lock with GIL held. */ \ + /* Acquire lock with GIL held (try-then-block to avoid deadlock). */ \ XXHASH_LOCK_ACQUIRE(self); \ update_fn(self->xxhash_state, buf->buf, buf->len); \ XXHASH_LOCK_RELEASE(self); \ From 5126e271deb1c9944ea0ebececcbd722a8bfca5a Mon Sep 17 00:00:00 2001 From: Yue Du Date: Mon, 22 Jun 2026 16:48:09 +0800 Subject: [PATCH 22/34] fix: distinguish default vs threadsafe types in repr() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The C extension type specs hardcoded tp_name as "xxhash.xxh32" etc., so _xxhash and _xxhash_threadsafe types had identical repr() output: Now tp_name uses the C module name as prefix via XXHASH_TP_NAME macro: _xxhash.xxh32 → default module _xxhash_threadsafe.xxh32 → threadsafe module Also updates _parse_fastcall_args error message strings for consistency. --- src/_xxhash.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index fca3345..8cdd43e 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -105,6 +105,8 @@ #define XXHASH_PASTE2(a, b) a ## b #define XXHASH_PASTE(a, b) XXHASH_PASTE2(a, b) #define XXHASH_PYINIT(name) XXHASH_PASTE(PyInit_, name) +/* Type name for repr(), e.g. "_xxhash.xxh32" or "_xxhash_threadsafe.xxh32". */ +#define XXHASH_TP_NAME(base) VALUE_TO_STRING(XXHASH_MODULE_NAME) "." base #define XXH32_DIGESTSIZE 4 #define XXH32_BLOCKSIZE 16 @@ -662,7 +664,7 @@ PYXXH32_vectorcall(PyObject *type, PyObject *const *args, Py_buffer buf; unsigned long long raw_seed; - if (_parse_fastcall_args(args, nargs, kwnames, "xxhash.xxh32", 0, + if (_parse_fastcall_args(args, nargs, kwnames, XXHASH_TP_NAME("xxh32"), 0, &buf, &raw_seed) < 0) return NULL; seed = (XXH32_hash_t)raw_seed; @@ -1078,7 +1080,7 @@ static PyType_Slot XXH32Type_slots[] = { }; static PyType_Spec XXH32Type_spec = { - .name = "xxhash.xxh32", + .name = XXHASH_TP_NAME("xxh32"), .basicsize = sizeof(PYXXH32Object), .flags = Py_TPFLAGS_DEFAULT #if PY_VERSION_HEX >= 0x030c0000 @@ -1119,7 +1121,7 @@ PYXXH64_vectorcall(PyObject *type, PyObject *const *args, Py_buffer buf; unsigned long long raw_seed; - if (_parse_fastcall_args(args, nargs, kwnames, "xxhash.xxh64", 0, + if (_parse_fastcall_args(args, nargs, kwnames, XXHASH_TP_NAME("xxh64"), 0, &buf, &raw_seed) < 0) return NULL; seed = (XXH64_hash_t)raw_seed; @@ -1379,7 +1381,7 @@ static PyType_Slot XXH64Type_slots[] = { }; static PyType_Spec XXH64Type_spec = { - .name = "xxhash.xxh64", + .name = XXHASH_TP_NAME("xxh64"), .basicsize = sizeof(PYXXH64Object), .flags = Py_TPFLAGS_DEFAULT #if PY_VERSION_HEX >= 0x030c0000 @@ -1420,7 +1422,7 @@ PYXXH3_64_vectorcall(PyObject *type, PyObject *const *args, Py_buffer buf; unsigned long long raw_seed; - if (_parse_fastcall_args(args, nargs, kwnames, "xxhash.xxh3_64", 0, + if (_parse_fastcall_args(args, nargs, kwnames, XXHASH_TP_NAME("xxh3_64"), 0, &buf, &raw_seed) < 0) return NULL; seed = (XXH64_hash_t)raw_seed; @@ -1687,7 +1689,7 @@ static PyType_Slot XXH3_64Type_slots[] = { }; static PyType_Spec XXH3_64Type_spec = { - .name = "xxhash.xxh3_64", + .name = XXHASH_TP_NAME("xxh3_64"), .basicsize = sizeof(PYXXH3_64Object), .flags = Py_TPFLAGS_DEFAULT #if PY_VERSION_HEX >= 0x030c0000 @@ -1728,7 +1730,7 @@ PYXXH3_128_vectorcall(PyObject *type, PyObject *const *args, Py_buffer buf; unsigned long long raw_seed; - if (_parse_fastcall_args(args, nargs, kwnames, "xxhash.xxh3_128", 0, + if (_parse_fastcall_args(args, nargs, kwnames, XXHASH_TP_NAME("xxh3_128"), 0, &buf, &raw_seed) < 0) return NULL; seed = (XXH64_hash_t)raw_seed; @@ -2014,7 +2016,7 @@ static PyType_Slot XXH3_128Type_slots[] = { }; static PyType_Spec XXH3_128Type_spec = { - .name = "xxhash.xxh3_128", + .name = XXHASH_TP_NAME("xxh3_128"), .basicsize = sizeof(PYXXH3_128Object), .flags = Py_TPFLAGS_DEFAULT #if PY_VERSION_HEX >= 0x030c0000 From bed6ae12933d479cc378c4411fdfa3d9411d0620 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Mon, 22 Jun 2026 16:52:16 +0800 Subject: [PATCH 23/34] fix: handle PyThread_allocate_lock() failure with MemoryError XXHASH_LOCK_INIT now returns 0 on success, -1 on failure (exception set). When PyThread_allocate_lock() fails (OOM), PyErr_NoMemory() is raised and the object construction fails cleanly instead of silently falling back to unlocked operation. All 14 call sites (new, vectorcall, copy for all 4 hash types) check the return value and cleanup properly: if (XXHASH_LOCK_INIT(self) < 0) { Py_DECREF(self); PyBuffer_Release(&buf); // vectorcall only return NULL; } CPython 3.9-3.12 hashlib does not handle this (lock=NULL = silent fallback), but since xxhash allocates eagerly at construction time, failing early with a clear MemoryError is more user-friendly. --- src/_xxhash.c | 72 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index 8cdd43e..bf5e60d 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -38,14 +38,16 @@ #ifdef XXHASH_WITH_LOCK # if PY_VERSION_HEX >= 0x030d0000 /* Python 3.13+: always-on PyMutex (3.15+ style) */ # define XXHASH_LOCK_FIELD PyMutex mutex; -# define XXHASH_LOCK_INIT(o) ((void)((o)->mutex = (PyMutex){0})) +# define XXHASH_LOCK_INIT(o) ( ((o)->mutex = (PyMutex){0}), 0 ) # define XXHASH_LOCK_FINI(o) ((void)0) # define XXHASH_LOCK_ACQUIRE(o) PyMutex_Lock(&(o)->mutex) # define XXHASH_LOCK_ACQUIRE_BLOCKING(o) XXHASH_LOCK_ACQUIRE(o) # define XXHASH_LOCK_RELEASE(o) PyMutex_Unlock(&(o)->mutex) # else /* Python 3.9-3.12: PyThread_type_lock (always-on) */ # define XXHASH_LOCK_FIELD PyThread_type_lock lock; -# define XXHASH_LOCK_INIT(o) ((o)->lock = PyThread_allocate_lock()) +/* Returns 0 on success, -1 on allocation failure (exception set). */ +# define XXHASH_LOCK_INIT(o) \ + (((o)->lock = PyThread_allocate_lock()) ? 0 : (PyErr_NoMemory(), -1)) # define XXHASH_LOCK_FINI(o) do { if ((o)->lock) \ PyThread_free_lock((o)->lock); \ } while (0) @@ -83,7 +85,7 @@ # endif # else /* !XXHASH_WITH_LOCK */ # define XXHASH_LOCK_FIELD -# define XXHASH_LOCK_INIT(o) ((void)0) +# define XXHASH_LOCK_INIT(o) (0) # define XXHASH_LOCK_FINI(o) ((void)0) # define XXHASH_LOCK_ACQUIRE(o) ((void)0) # define XXHASH_LOCK_ACQUIRE_BLOCKING(o) ((void)0) @@ -676,7 +678,11 @@ PYXXH32_vectorcall(PyObject *type, PyObject *const *args, return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + PyBuffer_Release(&buf); + return NULL; + } self->xxhash_state = XXH32_createState(); if (self->xxhash_state == NULL) { @@ -711,7 +717,10 @@ static PyObject *PYXXH32_new(PyTypeObject *type, PyObject *args, PyObject *kwarg return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + return NULL; + } if ((self->xxhash_state = XXH32_createState()) == NULL) { Py_DECREF(self); @@ -960,7 +969,10 @@ static PyObject *PYXXH32_copy(PYXXH32Object *self) return NULL; } - XXHASH_LOCK_INIT(p); + if (XXHASH_LOCK_INIT(p) < 0) { + Py_DECREF(p); + return NULL; + } if ((p->xxhash_state = XXH32_createState()) == NULL) { Py_DECREF(p); @@ -1133,7 +1145,11 @@ PYXXH64_vectorcall(PyObject *type, PyObject *const *args, return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + PyBuffer_Release(&buf); + return NULL; + } self->xxhash_state = XXH64_createState(); if (self->xxhash_state == NULL) { @@ -1165,7 +1181,10 @@ static PyObject *PYXXH64_new(PyTypeObject *type, PyObject *args, PyObject *kwarg return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + return NULL; + } if ((self->xxhash_state = XXH64_createState()) == NULL) { Py_DECREF(self); @@ -1261,7 +1280,10 @@ static PyObject *PYXXH64_copy(PYXXH64Object *self) return NULL; } - XXHASH_LOCK_INIT(p); + if (XXHASH_LOCK_INIT(p) < 0) { + Py_DECREF(p); + return NULL; + } if ((p->xxhash_state = XXH64_createState()) == NULL) { Py_DECREF(p); @@ -1434,7 +1456,11 @@ PYXXH3_64_vectorcall(PyObject *type, PyObject *const *args, return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + PyBuffer_Release(&buf); + return NULL; + } self->xxhash_state = XXH3_createState(); if (self->xxhash_state == NULL) { @@ -1466,7 +1492,10 @@ static PyObject *PYXXH3_64_new(PyTypeObject *type, PyObject *args, PyObject *kwa return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + return NULL; + } if ((self->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(self); @@ -1562,7 +1591,10 @@ static PyObject *PYXXH3_64_copy(PYXXH3_64Object *self) return NULL; } - XXHASH_LOCK_INIT(p); + if (XXHASH_LOCK_INIT(p) < 0) { + Py_DECREF(p); + return NULL; + } if ((p->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(p); @@ -1742,7 +1774,11 @@ PYXXH3_128_vectorcall(PyObject *type, PyObject *const *args, return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + PyBuffer_Release(&buf); + return NULL; + } self->xxhash_state = XXH3_createState(); if (self->xxhash_state == NULL) { @@ -1774,7 +1810,10 @@ static PyObject *PYXXH3_128_new(PyTypeObject *type, PyObject *args, PyObject *kw return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + return NULL; + } if ((self->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(self); @@ -1889,7 +1928,10 @@ static PyObject *PYXXH3_128_copy(PYXXH3_128Object *self) return NULL; } - XXHASH_LOCK_INIT(p); + if (XXHASH_LOCK_INIT(p) < 0) { + Py_DECREF(p); + return NULL; + } if ((p->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(p); From 4fa9b6620070c6b33ef473b9d4feb65af0725b93 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Mon, 22 Jun 2026 20:59:16 +0800 Subject: [PATCH 24/34] fix: use public module name for tp_name (repr/__module__) - Add XXHASH_TP_NAME_PREFIX macro distinct from XXHASH_MODULE_NAME - Default module: tp_name = "xxhash.xxh64" (was "_xxhash.xxh64") - Threadsafe module: tp_name = "xxhash.threadsafe.xxh64" (was "_xxhash_threadsafe.xxh64") - type.__module__ now points to a resolvable public module name Previously the internal C extension module name (_xxhash) leaked into repr() and __module__, even though the public API is xxhash.xxh64. --- setup.py | 2 +- src/_xxhash.c | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9b4d77d..5d4c1f9 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ ), Extension( "_xxhash_threadsafe", - define_macros=[("XXHASH_WITH_LOCK", "1"), ("XXHASH_MODULE_NAME", "_xxhash_threadsafe")], + define_macros=[("XXHASH_WITH_LOCK", "1"), ("XXHASH_MODULE_NAME", "_xxhash_threadsafe"), ("XXHASH_TP_NAME_PREFIX", "xxhash.threadsafe")], **_ext_kwargs, ), ] diff --git a/src/_xxhash.c b/src/_xxhash.c index bf5e60d..f399b7c 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -104,11 +104,18 @@ #ifndef XXHASH_MODULE_NAME # define XXHASH_MODULE_NAME _xxhash #endif +/* Display prefix for tp_name (repr), separate from the C extension module name. + * Default: "xxhash". Threadsafe build: "xxhash.threadsafe". */ +#ifndef XXHASH_TP_NAME_PREFIX +# define XXHASH_TP_NAME_PREFIX xxhash +#endif #define XXHASH_PASTE2(a, b) a ## b #define XXHASH_PASTE(a, b) XXHASH_PASTE2(a, b) #define XXHASH_PYINIT(name) XXHASH_PASTE(PyInit_, name) -/* Type name for repr(), e.g. "_xxhash.xxh32" or "_xxhash_threadsafe.xxh32". */ -#define XXHASH_TP_NAME(base) VALUE_TO_STRING(XXHASH_MODULE_NAME) "." base +/* Type name for repr(), e.g. "xxhash.xxh32" or "xxhash.threadsafe.xxh32". + * Uses XXHASH_TP_NAME_PREFIX (public module name) instead of + * XXHASH_MODULE_NAME (internal C extension name). */ +#define XXHASH_TP_NAME(base) VALUE_TO_STRING(XXHASH_TP_NAME_PREFIX) "." base #define XXH32_DIGESTSIZE 4 #define XXH32_BLOCKSIZE 16 From 03f4180535110487a444d1c5bb30a9f260eac311 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Mon, 22 Jun 2026 21:03:04 +0800 Subject: [PATCH 25/34] test: add threadsafe basic API tests Covers all 4 hash types (xxh32/64/3_64/128): - Object construction with/without seed - Streaming: update, digest, hexdigest, intdigest, copy, reset - Top-level convenience functions (xxh32_digest, etc.) - Results match default xxhash module - repr() and __module__ use public module name - Large data paths (64KB GIL release threshold) - Type attributes: name, seed, digest_size, block_size --- tests/test_threadsafe_api.py | 307 +++++++++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 tests/test_threadsafe_api.py diff --git a/tests/test_threadsafe_api.py b/tests/test_threadsafe_api.py new file mode 100644 index 0000000..0585ccc --- /dev/null +++ b/tests/test_threadsafe_api.py @@ -0,0 +1,307 @@ +""" +Basic API tests for the ``xxhash.threadsafe`` module. + +Verifies that: + * the ``xxhash.threadsafe`` submodule is importable + * all 4 hash types (xxh32, xxh64, xxh3_64, xxh128/xxh3_128) work + * streaming methods (update, digest, hexdigest, intdigest, copy, reset) work + * results match the default ``xxhash`` module exactly + * ``repr()`` and ``type.__module__`` reflect the public module name +""" + +import os +import unittest +import xxhash +from xxhash import threadsafe + + +# Known good values from the default xxhash module. +XXH32_A = (b'a', 1426945110) +XXH64_A = (b'a', 15154266338359012955) +XXH3_64_A = (b'a', 16629034431890738719) +XXH3_128_A = (b'a', 225219434562328483135862406050043285023) + +class TestThreadsafeTypes(unittest.TestCase): + """Verify that all 4 hash types exist and produce correct results.""" + + def test_xxh32_object(self): + h = threadsafe.xxh32(b'a') + self.assertEqual(h.intdigest(), XXH32_A[1]) + + def test_xxh32_seed(self): + h = threadsafe.xxh32(b'a', seed=42) + self.assertEqual(h.intdigest(), threadsafe.xxh32(b'a', 42).intdigest()) + + def test_xxh32_matches_default(self): + default = xxhash.xxh32(b'hello world', 123) + safe = threadsafe.xxh32(b'hello world', 123) + self.assertEqual(default.digest(), safe.digest()) + self.assertEqual(default.hexdigest(), safe.hexdigest()) + self.assertEqual(default.intdigest(), safe.intdigest()) + + def test_xxh64_object(self): + h = threadsafe.xxh64(b'a') + self.assertEqual(h.intdigest(), XXH64_A[1]) + + def test_xxh64_matches_default(self): + default = xxhash.xxh64(b'hello world', 123) + safe = threadsafe.xxh64(b'hello world', 123) + self.assertEqual(default.digest(), safe.digest()) + self.assertEqual(default.hexdigest(), safe.hexdigest()) + self.assertEqual(default.intdigest(), safe.intdigest()) + + def test_xxh3_64_object(self): + h = threadsafe.xxh3_64(b'a') + self.assertEqual(h.intdigest(), XXH3_64_A[1]) + + def test_xxh3_64_matches_default(self): + default = xxhash.xxh3_64(b'hello world', 123) + safe = threadsafe.xxh3_64(b'hello world', 123) + self.assertEqual(default.digest(), safe.digest()) + self.assertEqual(default.hexdigest(), safe.hexdigest()) + self.assertEqual(default.intdigest(), safe.intdigest()) + + def test_xxh128_object(self): + h = threadsafe.xxh128(b'a') + self.assertEqual(h.intdigest(), XXH3_128_A[1]) + + def test_xxh128_matches_default(self): + default = xxhash.xxh128(b'hello world', 123) + safe = threadsafe.xxh128(b'hello world', 123) + self.assertEqual(default.digest(), safe.digest()) + self.assertEqual(default.hexdigest(), safe.hexdigest()) + self.assertEqual(default.intdigest(), safe.intdigest()) + + +class TestThreadsafeStreaming(unittest.TestCase): + """Verify that streaming operations work correctly.""" + + def test_update(self): + h = threadsafe.xxh32() + h.update(b'a') + self.assertEqual(h.digest(), threadsafe.xxh32(b'a').digest()) + h.update(b'b') + self.assertEqual(h.digest(), threadsafe.xxh32(b'ab').digest()) + h.update(b'c') + self.assertEqual(h.digest(), threadsafe.xxh32(b'abc').digest()) + + def test_update_chain(self): + for typ, data in [ + (threadsafe.xxh32, b'x' * 100000), + (threadsafe.xxh64, b'x' * 100000), + (threadsafe.xxh3_64, b'x' * 100000), + (threadsafe.xxh128, b'x' * 100000), + ]: + with self.subTest(typ=typ): + h = typ() + h.update(data) + self.assertEqual(h.digest(), typ(data).digest()) + + def test_reset(self): + for typ in [threadsafe.xxh32, threadsafe.xxh64, + threadsafe.xxh3_64, threadsafe.xxh128]: + with self.subTest(typ=typ): + h = typ() + initial = h.intdigest() + for _ in range(10): + h.update(os.urandom(64)) + h.reset() + self.assertEqual(initial, h.intdigest()) + + def test_copy(self): + for typ in [threadsafe.xxh32, threadsafe.xxh64, + threadsafe.xxh3_64, threadsafe.xxh128]: + with self.subTest(typ=typ): + a = typ() + a.update(b'hello world') + b = a.copy() + self.assertEqual(a.digest(), b.digest()) + self.assertEqual(a.intdigest(), b.intdigest()) + self.assertEqual(a.hexdigest(), b.hexdigest()) + b.update(b'more data') + self.assertNotEqual(a.digest(), b.digest()) + + def test_digest_hexdigest_intdigest(self): + for typ, known in [ + (threadsafe.xxh32, XXH32_A), + (threadsafe.xxh64, XXH64_A), + (threadsafe.xxh3_64, XXH3_64_A), + (threadsafe.xxh128, XXH3_128_A), + ]: + data, expected = known + with self.subTest(typ=typ): + h = typ(data) + self.assertEqual(h.intdigest(), expected) + self.assertIsInstance(h.digest(), bytes) + self.assertIsInstance(h.hexdigest(), str) + self.assertIsInstance(h.intdigest(), int) + + +class TestThreadsafeTopLevelFunctions(unittest.TestCase): + """Verify that top-level convenience functions work.""" + + def test_xxh32_digest(self): + self.assertEqual( + threadsafe.xxh32_digest(b'a'), + xxhash.xxh32_digest(b'a'), + ) + + def test_xxh32_hexdigest(self): + self.assertEqual( + threadsafe.xxh32_hexdigest(b'a'), + xxhash.xxh32_hexdigest(b'a'), + ) + + def test_xxh32_intdigest(self): + self.assertEqual( + threadsafe.xxh32_intdigest(b'a'), + xxhash.xxh32_intdigest(b'a'), + ) + + def test_xxh64_digest(self): + self.assertEqual( + threadsafe.xxh64_digest(b'a'), + xxhash.xxh64_digest(b'a'), + ) + + def test_xxh64_hexdigest(self): + self.assertEqual( + threadsafe.xxh64_hexdigest(b'a'), + xxhash.xxh64_hexdigest(b'a'), + ) + + def test_xxh64_intdigest(self): + self.assertEqual( + threadsafe.xxh64_intdigest(b'a'), + xxhash.xxh64_intdigest(b'a'), + ) + + def test_xxh3_64_digest(self): + self.assertEqual( + threadsafe.xxh3_64_digest(b'a'), + xxhash.xxh3_64_digest(b'a'), + ) + + def test_xxh3_64_hexdigest(self): + self.assertEqual( + threadsafe.xxh3_64_hexdigest(b'a'), + xxhash.xxh3_64_hexdigest(b'a'), + ) + + def test_xxh3_64_intdigest(self): + self.assertEqual( + threadsafe.xxh3_64_intdigest(b'a'), + xxhash.xxh3_64_intdigest(b'a'), + ) + + def test_xxh128_digest(self): + self.assertEqual( + threadsafe.xxh128_digest(b'a'), + xxhash.xxh128_digest(b'a'), + ) + + def test_xxh128_hexdigest(self): + self.assertEqual( + threadsafe.xxh128_hexdigest(b'a'), + xxhash.xxh128_hexdigest(b'a'), + ) + + def test_xxh128_intdigest(self): + self.assertEqual( + threadsafe.xxh128_intdigest(b'a'), + xxhash.xxh128_intdigest(b'a'), + ) + + +class TestThreadsafeRepr(unittest.TestCase): + """Verify that repr() and __module__ use the public module name.""" + + def test_repr_xxh32(self): + h = threadsafe.xxh32() + self.assertIn("xxhash.threadsafe", repr(h)) + self.assertEqual(type(h).__module__, "xxhash.threadsafe") + + def test_repr_xxh64(self): + h = threadsafe.xxh64() + self.assertIn("xxhash.threadsafe", repr(h)) + self.assertEqual(type(h).__module__, "xxhash.threadsafe") + + def test_repr_xxh3_64(self): + h = threadsafe.xxh3_64() + self.assertIn("xxhash.threadsafe", repr(h)) + self.assertEqual(type(h).__module__, "xxhash.threadsafe") + + def test_repr_xxh128(self): + h = threadsafe.xxh128() + self.assertIn("xxhash.threadsafe", repr(h)) + self.assertEqual(type(h).__module__, "xxhash.threadsafe") + + +class TestThreadsafeLargeData(unittest.TestCase): + """Verify that large data paths (≥64KB GIL release threshold) work.""" + + SIZE = 64 * 1024 # exactly the GIL release threshold + + def test_xxh32_large(self): + data = b'x' * self.SIZE + self.assertEqual( + threadsafe.xxh32(data).digest(), + xxhash.xxh32(data).digest(), + ) + + def test_xxh64_large(self): + data = b'x' * self.SIZE + self.assertEqual( + threadsafe.xxh64(data).digest(), + xxhash.xxh64(data).digest(), + ) + + def test_xxh3_64_large(self): + data = b'x' * self.SIZE + self.assertEqual( + threadsafe.xxh3_64(data).digest(), + xxhash.xxh3_64(data).digest(), + ) + + def test_xxh128_large(self): + data = b'x' * self.SIZE + self.assertEqual( + threadsafe.xxh128(data).digest(), + xxhash.xxh128(data).digest(), + ) + + def test_streaming_update_large_chunks(self): + """Verify that streaming with large chunks works.""" + h = threadsafe.xxh64() + for _ in range(5): + h.update(b'x' * self.SIZE) + expected = xxhash.xxh64(b'x' * (5 * self.SIZE)).digest() + self.assertEqual(h.digest(), expected) + + +class TestThreadsafeAttributes(unittest.TestCase): + """Verify that threadsafe types have the expected attributes.""" + + def test_name(self): + self.assertEqual(threadsafe.xxh32().name, "XXH32") + self.assertEqual(threadsafe.xxh64().name, "XXH64") + self.assertEqual(threadsafe.xxh3_64().name, "XXH3_64") + self.assertEqual(threadsafe.xxh128().name, "XXH3_128") + + def test_seed_attribute(self): + h = threadsafe.xxh32(b'test', 42) + self.assertEqual(h.seed, 42) + + def test_digest_size(self): + self.assertEqual(threadsafe.xxh32().digest_size, 4) + self.assertEqual(threadsafe.xxh64().digest_size, 8) + self.assertEqual(threadsafe.xxh3_64().digest_size, 8) + self.assertEqual(threadsafe.xxh128().digest_size, 16) + + def test_block_size(self): + self.assertEqual(threadsafe.xxh32().block_size, 16) + self.assertEqual(threadsafe.xxh64().block_size, 32) + + +if __name__ == '__main__': + unittest.main() From ec53c687b04857f39c0ffd40903df61de2df0fcf Mon Sep 17 00:00:00 2001 From: Yue Du Date: Mon, 22 Jun 2026 21:08:51 +0800 Subject: [PATCH 26/34] refactor: group all #define macros together at file top Move XXHASH_DO_UPDATE, XXHASH_INIT, XXHASH_UPDATE_METHOD code generation macros from scattered locations to the macro section, grouped into clear sections: - Lock types & helpers (XXHASH_WITH_LOCK) - Stringification helpers (TOSTRING, VALUE_TO_STRING) - Build configuration (XXHASH_MODULE_NAME, XXHASH_TP_NAME_PREFIX, ...) - xxHash constants (DIGESTSIZE, BLOCKSIZE, VERSION) - Code generation (XXHASH_DO_UPDATE, XXHASH_INIT, XXHASH_UPDATE_METHOD) --- src/_xxhash.c | 253 +++++++++++++++++++++++++++----------------------- 1 file changed, 138 insertions(+), 115 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index f399b7c..5763f7b 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -92,39 +92,165 @@ # define XXHASH_LOCK_RELEASE(o) ((void)0) #endif -/* Data size threshold for releasing the GIL during hash. */ +/* Data size threshold for releasing the GIL during hash update. */ #define XXHASH_GIL_MINSIZE 65536 + +/* ------------------------------------------------------------------ */ +/* Stringification helpers */ +/* ------------------------------------------------------------------ */ #define TOSTRING(x) #x #define VALUE_TO_STRING(x) TOSTRING(x) -#define XXHASH_VERSION XXH_VERSION_MAJOR.XXH_VERSION_MINOR.XXH_VERSION_RELEASE -/* Module name is parameterised so the same source file can be compiled as - * both xxhash._xxhash and xxhash._xxhash_threadsafe. */ + +/* ------------------------------------------------------------------ */ +/* Build configuration: module & type names */ +/* ------------------------------------------------------------------ */ +/* Module name is parameterised so the same source file can be compiled + * as both xxhash._xxhash and xxhash._xxhash_threadsafe. */ #ifndef XXHASH_MODULE_NAME # define XXHASH_MODULE_NAME _xxhash #endif -/* Display prefix for tp_name (repr), separate from the C extension module name. - * Default: "xxhash". Threadsafe build: "xxhash.threadsafe". */ +/* Display prefix for tp_name (repr), separate from the C extension + * module name. Default: "xxhash". Threadsafe build: "xxhash.threadsafe". */ #ifndef XXHASH_TP_NAME_PREFIX # define XXHASH_TP_NAME_PREFIX xxhash #endif +/* Token concatenation helpers for PyInit_. */ #define XXHASH_PASTE2(a, b) a ## b #define XXHASH_PASTE(a, b) XXHASH_PASTE2(a, b) #define XXHASH_PYINIT(name) XXHASH_PASTE(PyInit_, name) -/* Type name for repr(), e.g. "xxhash.xxh32" or "xxhash.threadsafe.xxh32". - * Uses XXHASH_TP_NAME_PREFIX (public module name) instead of - * XXHASH_MODULE_NAME (internal C extension name). */ +/* Type name for repr(), e.g. "xxhash.xxh32" or "xxhash.threadsafe.xxh32". */ #define XXHASH_TP_NAME(base) VALUE_TO_STRING(XXHASH_TP_NAME_PREFIX) "." base + +/* ------------------------------------------------------------------ */ +/* xxHash constants */ +/* ------------------------------------------------------------------ */ #define XXH32_DIGESTSIZE 4 #define XXH32_BLOCKSIZE 16 #define XXH64_DIGESTSIZE 8 #define XXH64_BLOCKSIZE 32 #define XXH128_DIGESTSIZE 16 #define XXH128_BLOCKSIZE 64 +#define XXHASH_VERSION XXH_VERSION_MAJOR.XXH_VERSION_MINOR.XXH_VERSION_RELEASE +/* ------------------------------------------------------------------ */ +/* Code generation helpers */ +/* ------------------------------------------------------------------ */ + +/* Generate _do_update for each hash type. + * Lock is always acquired (no-op in non-lock build). For large data + * release GIL while blocking on lock to avoid stalling other threads. */ +#define XXHASH_DO_UPDATE(type, update_fn) \ +static inline void \ +PY##type##_do_update(PY##type##Object *self, Py_buffer *buf) \ +{ \ + if (buf->len > XXHASH_GIL_MINSIZE) { \ + /* Release GIL first, then block on lock. */ \ + Py_BEGIN_ALLOW_THREADS \ + XXHASH_LOCK_ACQUIRE_BLOCKING(self); \ + update_fn(self->xxhash_state, buf->buf, buf->len); \ + XXHASH_LOCK_RELEASE(self); \ + Py_END_ALLOW_THREADS \ + } else { \ + /* Acquire lock with GIL held (try-then-block). */ \ + XXHASH_LOCK_ACQUIRE(self); \ + update_fn(self->xxhash_state, buf->buf, buf->len); \ + XXHASH_LOCK_RELEASE(self); \ + } \ + PyBuffer_Release(buf); \ +} + +/* Generate __init__ for each hash type. */ +#define XXHASH_INIT(type, reset_fn, update_fn, seed_cast) \ +static int PY##type##_init(PY##type##Object *self, \ + PyObject *args, PyObject *kwargs) \ +{ \ + unsigned long long seed_val = 0; \ + PyObject *data_obj = NULL; \ + Py_buffer buf = {NULL, NULL}; \ + \ + if (_parse_init_args(args, kwargs, &data_obj, &seed_val,\ + "__init__()") < 0) \ + return -1; \ + \ + if (data_obj) { \ + if (_get_buffer_or_str(data_obj, &buf) < 0) \ + return -1; \ + } \ + \ + XXHASH_LOCK_ACQUIRE(self); \ + self->seed = (seed_cast)seed_val; \ + reset_fn(self->xxhash_state, self->seed); \ + \ + if (buf.obj) { \ + update_fn(self->xxhash_state, buf.buf, buf.len); \ + PyBuffer_Release(&buf); \ + } \ + XXHASH_LOCK_RELEASE(self); \ + return 0; \ +} + +/* Generate update() method for each hash type (with keyword support). */ +#define XXHASH_UPDATE_METHOD(prefix, name) \ +PyDoc_STRVAR( \ + PY##prefix##_update_doc, \ + "update (data)\n\n" \ + "Update the " name " object with bytes-like data. Repeated calls are\n" \ + "equivalent to a single call with the concatenation of all the arguments."); \ + \ +static PyObject *PY##prefix##_update(PY##prefix##Object *self, \ + PyObject *const *args, \ + Py_ssize_t nargs, PyObject *kwnames) \ +{ \ + PyObject *arg = NULL; \ + \ + /* validate keywords first */ \ + if (kwnames) { \ + Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); \ + for (Py_ssize_t i = 0; i < nkw; i++) { \ + PyObject *key = PyTuple_GET_ITEM(kwnames, i); \ + if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { \ + if (nargs >= 1) { \ + PyErr_SetString(PyExc_TypeError, \ + name ".update() got multiple values for argument 'data'"); \ + return NULL; \ + } \ + arg = args[nargs + i]; \ + } else { \ + PyErr_Format(PyExc_TypeError, \ + "'%U' is an invalid keyword argument for '" name ".update()'", \ + key); \ + return NULL; \ + } \ + } \ + } \ + \ + if (nargs >= 1) { \ + if (nargs > 1) { \ + PyErr_Format(PyExc_TypeError, \ + name ".update() takes at most 1 positional argument (%zd given)", \ + nargs); \ + return NULL; \ + } \ + arg = args[0]; \ + } \ + \ + if (!arg) { \ + PyErr_SetString(PyExc_TypeError, \ + name ".update() missing required argument 'data'"); \ + return NULL; \ + } \ + \ + Py_buffer buf; \ + if (_get_buffer_or_str(arg, &buf) < 0) \ + return NULL; \ + PY##prefix##_do_update(self, &buf); \ + Py_RETURN_NONE; \ +} + /* Hex lookup table for hexdigest(). */ /* Get a buffer from an object. Rejects str with hashlib-compatible error. */ static inline int @@ -639,28 +765,7 @@ static void PYXXH32_dealloc(PYXXH32Object *self) Py_DECREF(tp); } -/* Macro to generate _do_update for each hash type. - * Lock is always acquired (no-op in non-lock build). For large data, - * release GIL while blocking on lock to avoid stalling other threads. */ -#define XXHASH_DO_UPDATE(type, update_fn) \ -static inline void \ -PY##type##_do_update(PY##type##Object *self, Py_buffer *buf) \ -{ \ - if (buf->len > XXHASH_GIL_MINSIZE) { \ - /* Release GIL first, then acquire lock (blocking, no GIL). */ \ - Py_BEGIN_ALLOW_THREADS \ - XXHASH_LOCK_ACQUIRE_BLOCKING(self); \ - update_fn(self->xxhash_state, buf->buf, buf->len); \ - XXHASH_LOCK_RELEASE(self); \ - Py_END_ALLOW_THREADS \ - } else { \ - /* Acquire lock with GIL held (try-then-block to avoid deadlock). */ \ - XXHASH_LOCK_ACQUIRE(self); \ - update_fn(self->xxhash_state, buf->buf, buf->len); \ - XXHASH_LOCK_RELEASE(self); \ - } \ - PyBuffer_Release(buf); \ -} +/* Invoke _do_update generator for each hash type. */ XXHASH_DO_UPDATE(XXH32, XXH32_update) @@ -808,93 +913,11 @@ _parse_init_args(PyObject *args, PyObject *kwargs, return 0; } -/* Macro to generate __init__ for each hash type. */ -#define XXHASH_INIT(type, reset_fn, update_fn, seed_cast) \ -static int PY##type##_init(PY##type##Object *self, PyObject *args, \ - PyObject *kwargs) \ -{ \ - unsigned long long seed_val = 0; \ - PyObject *data_obj = NULL; \ - Py_buffer buf = {NULL, NULL}; \ - \ - if (_parse_init_args(args, kwargs, &data_obj, &seed_val, \ - "__init__()") < 0) \ - return -1; \ - \ - if (data_obj) { \ - if (_get_buffer_or_str(data_obj, &buf) < 0) \ - return -1; \ - } \ - \ - XXHASH_LOCK_ACQUIRE(self); \ - self->seed = (seed_cast)seed_val; \ - reset_fn(self->xxhash_state, self->seed); \ - \ - if (buf.obj) { \ - update_fn(self->xxhash_state, buf.buf, buf.len); \ - PyBuffer_Release(&buf); \ - } \ - XXHASH_LOCK_RELEASE(self); \ - return 0; \ -} +/* Generate __init__ for each hash type. */ XXHASH_INIT(XXH32, XXH32_reset, XXH32_update, XXH32_hash_t) -#define XXHASH_UPDATE_METHOD(prefix, name) \ -PyDoc_STRVAR( \ - PY##prefix##_update_doc, \ - "update (data)\n\n" \ - "Update the " name " object with bytes-like data. Repeated calls are\n" \ - "equivalent to a single call with the concatenation of all the arguments."); \ - \ -static PyObject *PY##prefix##_update(PY##prefix##Object *self, \ - PyObject *const *args, \ - Py_ssize_t nargs, PyObject *kwnames) \ -{ \ - PyObject *arg = NULL; \ - \ - /* validate keywords first */ \ - if (kwnames) { \ - Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); \ - for (Py_ssize_t i = 0; i < nkw; i++) { \ - PyObject *key = PyTuple_GET_ITEM(kwnames, i); \ - if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { \ - if (nargs >= 1) { \ - PyErr_SetString(PyExc_TypeError, \ - name ".update() got multiple values for argument 'data'"); \ - return NULL; \ - } \ - arg = args[nargs + i]; \ - } else { \ - PyErr_Format(PyExc_TypeError, \ - "'%U' is an invalid keyword argument for '" name ".update()'", \ - key); \ - return NULL; \ - } \ - } \ - } \ - \ - if (nargs >= 1) { \ - if (nargs > 1) { \ - PyErr_Format(PyExc_TypeError, \ - name ".update() takes at most 1 positional argument (%zd given)", nargs); \ - return NULL; \ - } \ - arg = args[0]; \ - } \ - \ - if (!arg) { \ - PyErr_SetString(PyExc_TypeError, \ - name ".update() missing required argument 'data'"); \ - return NULL; \ - } \ - \ - Py_buffer buf; \ - if (_get_buffer_or_str(arg, &buf) < 0) \ - return NULL; \ - PY##prefix##_do_update(self, &buf); \ - Py_RETURN_NONE; \ -} +/* Generate update() for each hash type. */ XXHASH_UPDATE_METHOD(XXH32, "xxh32") From 87a8d4f728cda783dd0a0accbd2850b04ba095c9 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Mon, 22 Jun 2026 21:41:55 +0800 Subject: [PATCH 27/34] chore: reformat setup.py define_macros for readability --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5d4c1f9..539b40f 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,11 @@ ), Extension( "_xxhash_threadsafe", - define_macros=[("XXHASH_WITH_LOCK", "1"), ("XXHASH_MODULE_NAME", "_xxhash_threadsafe"), ("XXHASH_TP_NAME_PREFIX", "xxhash.threadsafe")], + define_macros=[ + ("XXHASH_WITH_LOCK", "1"), + ("XXHASH_MODULE_NAME", "_xxhash_threadsafe"), + ("XXHASH_TP_NAME_PREFIX", "xxhash.threadsafe"), + ], **_ext_kwargs, ), ] From 9b66e855dac11e7ec2b3b48d92ff30f55ac63431 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Mon, 22 Jun 2026 21:51:06 +0800 Subject: [PATCH 28/34] test: merge test_name and test_algorithms_available into test_hashlib_compat --- tests/test_algorithms_available.py | 28 ---------------------------- tests/test_hashlib_compat.py | 26 +++++++++++++++++++------- tests/test_name.py | 24 ------------------------ 3 files changed, 19 insertions(+), 59 deletions(-) delete mode 100644 tests/test_algorithms_available.py delete mode 100644 tests/test_name.py diff --git a/tests/test_algorithms_available.py b/tests/test_algorithms_available.py deleted file mode 100644 index 3c3f1fd..0000000 --- a/tests/test_algorithms_available.py +++ /dev/null @@ -1,28 +0,0 @@ -import xxhash -import unittest - - -class TestAlgorithmExists(unittest.TestCase): - def test_xxh32(self): - xxhash.xxh32 - assert "xxh32" in xxhash.algorithms_available - - def test_xxh64(self): - xxhash.xxh64 - assert "xxh64" in xxhash.algorithms_available - - def test_xxh3_64(self): - xxhash.xxh3_64 - assert "xxh3_64" in xxhash.algorithms_available - - def test_xxh128(self): - xxhash.xxh128 - assert "xxh128" in xxhash.algorithms_available - - def test_xxh3_128(self): - xxhash.xxh3_128 - assert "xxh3_128" in xxhash.algorithms_available - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_hashlib_compat.py b/tests/test_hashlib_compat.py index fa21b90..f2be6e3 100644 --- a/tests/test_hashlib_compat.py +++ b/tests/test_hashlib_compat.py @@ -10,12 +10,26 @@ class TestHashlibCompat(unittest.TestCase): def test_algorithms_available(self): self.assertIsInstance(xxhash.algorithms_available, set) - for a in ('xxh32', 'xxh64', 'xxh3_64', 'xxh3_128', 'xxh128'): + expected = {'xxh32', 'xxh64', 'xxh3_64', 'xxh3_128', 'xxh128'} + self.assertGreaterEqual(xxhash.algorithms_available, expected) + for a in expected: self.assertIn(a, xxhash.algorithms_available) def test_algorithms_guaranteed(self): self.assertEqual(xxhash.algorithms_guaranteed, xxhash.algorithms_available) + def test_name(self): + expected = { + 'xxh32': 'XXH32', + 'xxh64': 'XXH64', + 'xxh3_64': 'XXH3_64', + 'xxh3_128': 'XXH3_128', + 'xxh128': 'XXH3_128', + } + for algo, name in expected.items(): + with self.subTest(algo=algo): + self.assertEqual(getattr(xxhash, algo)().name, name) + # ── str rejection ────────────────────────────────────────────── def test_str_rejected(self): @@ -120,12 +134,6 @@ def test_block_size(self): self.assertEqual(xxhash.xxh3_64().block_size, 32) self.assertEqual(xxhash.xxh3_128().block_size, 64) - def test_name(self): - self.assertEqual(xxhash.xxh32().name, 'XXH32') - self.assertEqual(xxhash.xxh64().name, 'XXH64') - self.assertEqual(xxhash.xxh3_64().name, 'XXH3_64') - self.assertEqual(xxhash.xxh3_128().name, 'XXH3_128') - # ── digest / hexdigest ───────────────────────────────────────── def test_digest(self): @@ -162,3 +170,7 @@ def test_copy(self): self.assertEqual(a.digest(), b.digest()) b.update(b'more') self.assertNotEqual(a.digest(), b.digest()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_name.py b/tests/test_name.py deleted file mode 100644 index f8fd479..0000000 --- a/tests/test_name.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import unittest -import random -import xxhash - - -class TestName(unittest.TestCase): - def test_xxh32(self): - self.assertEqual(xxhash.xxh32().name, "XXH32") - - def test_xxh64(self): - self.assertEqual(xxhash.xxh64().name, "XXH64") - - def test_xxh3_64(self): - self.assertEqual(xxhash.xxh3_64().name, "XXH3_64") - - def test_xxh128(self): - self.assertEqual(xxhash.xxh128().name, "XXH3_128") - - def test_xxh3_128(self): - self.assertEqual(xxhash.xxh3_128().name, "XXH3_128") - -if __name__ == '__main__': - unittest.main() From 015826529c3728efc6847cb57d3d253ac164fccb Mon Sep 17 00:00:00 2001 From: Yue Du Date: Tue, 23 Jun 2026 00:32:12 +0800 Subject: [PATCH 29/34] docs: fix xxh3_64 copy docstrings that incorrectly referred to xxh64 --- src/_xxhash.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/_xxhash.c b/src/_xxhash.c index 5763f7b..a322728 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -1610,8 +1610,8 @@ static PyObject *PYXXH3_64_intdigest(PYXXH3_64Object *self) PyDoc_STRVAR( PYXXH3_64_copy_doc, - "copy() -> xxh64 object\n\n" - "Return a copy (``clone'') of the xxh64 object."); + "copy() -> xxh3_64 object\n\n" + "Return a copy (``clone'') of the xxh3_64 object."); static PyObject *PYXXH3_64_copy(PYXXH3_64Object *self) { @@ -1738,7 +1738,7 @@ PyDoc_STRVAR( "digest() -- return the current digest value\n" "hexdigest() -- return the current digest as a string of hexadecimal digits\n" "intdigest() -- return the current digest as an integer\n" - "copy() -- return a copy of the current xxh64 object"); + "copy() -- return a copy of the current xxh3_64 object"); static PyType_Slot XXH3_64Type_slots[] = { {Py_tp_dealloc, PYXXH3_64_dealloc}, From dc9e21056cfe7e2317d36cd770d0563e7bfa3d09 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Tue, 23 Jun 2026 00:32:18 +0800 Subject: [PATCH 30/34] chore: bump version to 4.0.0.dev1 --- xxhash/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xxhash/version.py b/xxhash/version.py index 1f5645f..69cb0bd 100644 --- a/xxhash/version.py +++ b/xxhash/version.py @@ -1 +1 @@ -VERSION = "4.0.0.dev0" +VERSION = "4.0.0.dev1" From 5131776d5bb4acc361221a90906b4a24d8bd6453 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Tue, 23 Jun 2026 17:11:46 +0800 Subject: [PATCH 31/34] test: add block_size assertions for xxh3_64/xxh128 and type-distinctness tests Extend test_block_size to cover all four hash types (xxh3_64=32, xxh128=64). Add TestThreadsafeTypesAreDistinct to verify that the threadsafe module exports distinct type objects from the default module. A regression where both modules accidentally pointed at the same type would silently break thread safety with no other test catching it. --- tests/test_threadsafe_api.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_threadsafe_api.py b/tests/test_threadsafe_api.py index 0585ccc..4dbdc73 100644 --- a/tests/test_threadsafe_api.py +++ b/tests/test_threadsafe_api.py @@ -301,6 +301,33 @@ def test_digest_size(self): def test_block_size(self): self.assertEqual(threadsafe.xxh32().block_size, 16) self.assertEqual(threadsafe.xxh64().block_size, 32) + self.assertEqual(threadsafe.xxh3_64().block_size, 32) + self.assertEqual(threadsafe.xxh128().block_size, 64) + + +class TestThreadsafeTypesAreDistinct(unittest.TestCase): + """Verify that threadsafe types are distinct from the default types. + + A regression here (e.g. both modules accidentally pointing at the same + type object) would silently break thread safety with no other test + catching it. + """ + + def test_types_are_distinct(self): + for name in ('xxh32', 'xxh64', 'xxh3_64', 'xxh128'): + with self.subTest(name=name): + default_type = getattr(xxhash, name) + safe_type = getattr(threadsafe, name) + self.assertIsNot(default_type, safe_type) + self.assertIsNot(type(default_type()), type(safe_type())) + + def test_instances_are_not_instances_of_each_other(self): + for name in ('xxh32', 'xxh64', 'xxh3_64', 'xxh128'): + with self.subTest(name=name): + default_type = getattr(xxhash, name) + safe_type = getattr(threadsafe, name) + self.assertFalse(isinstance(default_type(), safe_type)) + self.assertFalse(isinstance(safe_type(), default_type)) if __name__ == '__main__': From ed212aadfbce1bbe995774c104b6df6bbffbbec1 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Tue, 23 Jun 2026 17:46:16 +0800 Subject: [PATCH 32/34] style: clean up comments, error messages, and classifiers - Remove stale 'Hex lookup table' comment above _get_buffer_or_str - Initialize Py_buffer in XXHASH_UPDATE_METHOD macro defensively - Unify PyErr_NoMemory() return style across _new and _copy paths - Include funcname in _parse_init_args invalid-keyword error message - Add Python 3.15 classifier to setup.py --- setup.py | 1 + src/_xxhash.c | 30 +++++++++++------------------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 539b40f..38b21e6 100644 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ def build_extension(self, ext): "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Free Threading :: 1 - Unstable", ], diff --git a/src/_xxhash.c b/src/_xxhash.c index a322728..fb9f001 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -244,14 +244,13 @@ static PyObject *PY##prefix##_update(PY##prefix##Object *self, return NULL; \ } \ \ - Py_buffer buf; \ + Py_buffer buf = {NULL, NULL}; \ if (_get_buffer_or_str(arg, &buf) < 0) \ return NULL; \ PY##prefix##_do_update(self, &buf); \ Py_RETURN_NONE; \ } -/* Hex lookup table for hexdigest(). */ /* Get a buffer from an object. Rejects str with hashlib-compatible error. */ static inline int _get_buffer_or_str(PyObject *obj, Py_buffer *buf) @@ -836,8 +835,7 @@ static PyObject *PYXXH32_new(PyTypeObject *type, PyObject *args, PyObject *kwarg if ((self->xxhash_state = XXH32_createState()) == NULL) { Py_DECREF(self); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } self->seed = 0; @@ -866,7 +864,8 @@ _parse_init_args(PyObject *args, PyObject *kwargs, PyUnicode_CompareWithASCIIString(key, "seed") == 0) continue; PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword argument for this function", key); + "'%U' is an invalid keyword argument for '%s()'", + key, funcname); return -1; } } @@ -1006,8 +1005,7 @@ static PyObject *PYXXH32_copy(PYXXH32Object *self) if ((p->xxhash_state = XXH32_createState()) == NULL) { Py_DECREF(p); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } XXHASH_LOCK_ACQUIRE(self); @@ -1218,8 +1216,7 @@ static PyObject *PYXXH64_new(PyTypeObject *type, PyObject *args, PyObject *kwarg if ((self->xxhash_state = XXH64_createState()) == NULL) { Py_DECREF(self); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } self->seed = 0; @@ -1317,8 +1314,7 @@ static PyObject *PYXXH64_copy(PYXXH64Object *self) if ((p->xxhash_state = XXH64_createState()) == NULL) { Py_DECREF(p); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } XXHASH_LOCK_ACQUIRE(self); @@ -1529,8 +1525,7 @@ static PyObject *PYXXH3_64_new(PyTypeObject *type, PyObject *args, PyObject *kwa if ((self->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(self); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } self->seed = 0; @@ -1628,8 +1623,7 @@ static PyObject *PYXXH3_64_copy(PYXXH3_64Object *self) if ((p->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(p); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } XXHASH_LOCK_ACQUIRE(self); @@ -1847,8 +1841,7 @@ static PyObject *PYXXH3_128_new(PyTypeObject *type, PyObject *args, PyObject *kw if ((self->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(self); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } self->seed = 0; @@ -1965,8 +1958,7 @@ static PyObject *PYXXH3_128_copy(PYXXH3_128Object *self) if ((p->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(p); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } XXHASH_LOCK_ACQUIRE(self); From 24889b448697ae164f926d3f278bd6790c71e6a2 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Tue, 23 Jun 2026 21:58:30 +0800 Subject: [PATCH 33/34] style: fix misleading #else indentation and update copyright year --- README.rst | 2 +- src/_xxhash.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c1abbb1..8527cb8 100644 --- a/README.rst +++ b/README.rst @@ -312,6 +312,6 @@ functions are required. Copyright and License --------------------- -Copyright (c) 2014-2025 Yue Du - https://github.com/ifduyue +Copyright (c) 2014-2026 Yue Du - https://github.com/ifduyue Licensed under `BSD 2-Clause License `_ diff --git a/src/_xxhash.c b/src/_xxhash.c index fb9f001..b091bfc 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -83,7 +83,7 @@ } \ } while (0) # endif -# else /* !XXHASH_WITH_LOCK */ +#else /* !XXHASH_WITH_LOCK */ # define XXHASH_LOCK_FIELD # define XXHASH_LOCK_INIT(o) (0) # define XXHASH_LOCK_FINI(o) ((void)0) From e4cfc4017c7379e206099fd93ca18851428855d8 Mon Sep 17 00:00:00 2001 From: Yue Du Date: Tue, 23 Jun 2026 22:18:57 +0800 Subject: [PATCH 34/34] chore(ci): test against Python 3.15.0-beta.2 --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90f95b6..4858a3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,7 @@ jobs: - "3.13t" - "3.14" - "3.14t" + - "3.15.0-beta.2" - "pypy3.9" - "pypy3.10" - "pypy3.11"