diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90f95b6..4858a3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,7 @@ jobs: - "3.13t" - "3.14" - "3.14t" + - "3.15.0-beta.2" - "pypy3.9" - "pypy3.10" - "pypy3.11" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1208b35..5bcd2dc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,16 @@ NEXT ~~~~~~~~~~~~~~~~~ - Drop support for Python 3.8 +- Remove deprecated ``xxhash.VERSION_TUPLE`` +- Default streaming hash objects (``xxh32``, ``xxh64``, ``xxh3_64``, + ``xxh3_128``) are no longer thread-safe by default; this removes + per-object locking overhead and restores performance as the primary goal +- Add ``xxhash.threadsafe`` submodule for users who need to share a + streaming hash object across threads; it provides the same API with a + per-object lock +- Both the default module and ``xxhash.threadsafe`` are provided on + free-threading (no-GIL) Python builds, matching the API on regular GIL + builds v3.7.0 2025-04-25 ~~~~~~~~~~~~~~~~~ diff --git a/README.rst b/README.rst index 3ec4e9c..8527cb8 100644 --- a/README.rst +++ b/README.rst @@ -251,6 +251,36 @@ And aliases: | xxh128_intdigest = xxh3_128_intdigest | xxh128_hexdigest = xxh3_128_hexdigest +Thread safety +------------- + +The default ``xxhash`` module is optimized for speed. Streaming hash objects +(``xxh32``, ``xxh64``, ``xxh3_64``, ``xxh3_128`` / ``xxh128``) are **not** +thread-safe: do not call ``update()``, ``digest()``, ``copy()``, ``reset()``, +or any other mutating method on the same object from multiple threads without +external synchronization. + +One-shot functions (``xxh32_digest``, ``xxh64_hexdigest``, ``xxh3_128_digest``, +etc.) are stateless and always safe to call concurrently. + +Concurrent ``update()`` / ``reset()`` on a shared streaming hash object is +discouraged even with locking — prefer one-shot functions or per-thread hash +objects. If you must share a streaming hash across threads, use the +``xxhash.threadsafe`` submodule. It provides the same API with a per-object +lock that serializes all access to the internal xxHash state: + +.. code-block:: python + + >>> from xxhash import threadsafe + >>> h = threadsafe.xxh64() + >>> # h can be updated from multiple threads, but concurrent update/reset + >>> # still adds overhead and is not recommended + +The same two-module split is provided on free-threading (no-GIL) Python +builds: the default module is unlocked, and ``xxhash.threadsafe`` provides a +locked variant. + + Caveats ------- @@ -282,6 +312,6 @@ functions are required. Copyright and License --------------------- -Copyright (c) 2014-2025 Yue Du - https://github.com/ifduyue +Copyright (c) 2014-2026 Yue Du - https://github.com/ifduyue Licensed under `BSD 2-Clause License `_ diff --git a/setup.py b/setup.py index 6fed6f6..38b21e6 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ from pathlib import Path from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext as _build_ext if os.getenv("XXHASH_LINK_SO"): libraries = ["xxhash"] @@ -12,15 +13,55 @@ source = ["src/_xxhash.c", "deps/xxhash/xxhash.c"] include_dirs = ["deps/xxhash"] +# The default ``xxhash._xxhash`` extension is built without per-object locks +# for maximum performance. Users who need to share a streaming hash object +# across threads can use ``xxhash._xxhash_threadsafe`` (exposed as the public +# ``xxhash.threadsafe`` submodule), which is compiled from the same source +# with locking enabled. + +_ext_kwargs = { + "sources": source, + "include_dirs": include_dirs, + "libraries": libraries, +} + ext_modules = [ Extension( "_xxhash", - source, - include_dirs=include_dirs, - libraries=libraries, - ) + **_ext_kwargs, + ), + Extension( + "_xxhash_threadsafe", + define_macros=[ + ("XXHASH_WITH_LOCK", "1"), + ("XXHASH_MODULE_NAME", "_xxhash_threadsafe"), + ("XXHASH_TP_NAME_PREFIX", "xxhash.threadsafe"), + ], + **_ext_kwargs, + ), ] + +class build_ext(_build_ext): + """Build each extension in its own temp directory. + + Both extensions are built from the same ``src/_xxhash.c`` source file. + Without separate temp directories their object files would overwrite + each other, causing one variant to be linked with the wrong macros. + + ``try/finally`` restores ``self.build_temp`` so that incremental builds + (where ``build_ext`` may be reused) still work correctly. + """ + + def build_extension(self, ext): + old_build_temp = self.build_temp + self.build_temp = os.path.join(old_build_temp, ext.name) + try: + super().build_extension(ext) + finally: + self.build_temp = old_build_temp + + d = Path(__file__).parent long_description = d.joinpath("README.rst").read_text() + "\n" + d.joinpath("CHANGELOG.rst").read_text() @@ -53,10 +94,12 @@ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Free Threading :: 1 - Unstable", ], python_requires=">=3.9", ext_modules=ext_modules, + cmdclass={"build_ext": build_ext}, package_data={"xxhash": ["py.typed", "**.pyi"]}, ) diff --git a/src/_xxhash.c b/src/_xxhash.c index d79f006..b091bfc 100644 --- a/src/_xxhash.c +++ b/src/_xxhash.c @@ -35,83 +35,224 @@ /* ------------------------------------------------------------------ */ /* Lock type & helpers */ /* ------------------------------------------------------------------ */ -#if PY_VERSION_HEX >= 0x030d0000 /* Python 3.13+: always-on PyMutex (3.15+ style) */ -# define XXHASH_LOCK_FIELD PyMutex mutex; -# define XXHASH_LOCK_INIT(o) ((void)((o)->mutex = (PyMutex){0})) -# define XXHASH_LOCK_IS_ACTIVE(o) 1 -# define XXHASH_LOCK_MAYBE_INIT(o, len) ((void)0) -# define XXHASH_LOCK_FINI(o) ((void)0) -# define XXHASH_LOCK_ACQUIRE(o) PyMutex_Lock(&(o)->mutex) -# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) XXHASH_LOCK_ACQUIRE(o) -# define XXHASH_LOCK_RELEASE(o) PyMutex_Unlock(&(o)->mutex) -#else /* Python 3.9-3.12: PyThread_type_lock */ -# define XXHASH_LOCK_FIELD PyThread_type_lock lock; -# define XXHASH_LOCK_INIT(o) ((o)->lock = NULL) -# define XXHASH_LOCK_IS_ACTIVE(o) ((o)->lock != NULL) -/* Lazy allocation on first large update */ -# define XXHASH_LOCK_MAYBE_INIT(o, len) \ - do { \ - if ((o)->lock == NULL && (len) >= XXHASH_GIL_MINSIZE) { \ - (o)->lock = PyThread_allocate_lock(); \ - /* fail? lock stays NULL, fall back to non-threaded code. */ \ - } \ - } while (0) -# define XXHASH_LOCK_FINI(o) do { if ((o)->lock) \ - PyThread_free_lock((o)->lock); \ - } while (0) -/* Acquire lock when GIL is already released — simple blocking acquire. - * Only acquires if lock has been allocated (lazy init). */ -# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) \ - do { \ - if ((o)->lock) { \ - PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ - } \ - } while (0) +#ifdef XXHASH_WITH_LOCK +# if PY_VERSION_HEX >= 0x030d0000 /* Python 3.13+: always-on PyMutex (3.15+ style) */ +# define XXHASH_LOCK_FIELD PyMutex mutex; +# define XXHASH_LOCK_INIT(o) ( ((o)->mutex = (PyMutex){0}), 0 ) +# define XXHASH_LOCK_FINI(o) ((void)0) +# define XXHASH_LOCK_ACQUIRE(o) PyMutex_Lock(&(o)->mutex) +# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) XXHASH_LOCK_ACQUIRE(o) +# define XXHASH_LOCK_RELEASE(o) PyMutex_Unlock(&(o)->mutex) +# else /* Python 3.9-3.12: PyThread_type_lock (always-on) */ +# define XXHASH_LOCK_FIELD PyThread_type_lock lock; +/* Returns 0 on success, -1 on allocation failure (exception set). */ +# define XXHASH_LOCK_INIT(o) \ + (((o)->lock = PyThread_allocate_lock()) ? 0 : (PyErr_NoMemory(), -1)) +# define XXHASH_LOCK_FINI(o) do { if ((o)->lock) \ + PyThread_free_lock((o)->lock); \ + } while (0) +/* Acquire lock when GIL is already released — simple blocking acquire. */ +# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) \ + do { \ + if ((o)->lock) { \ + PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ + } \ + } while (0) /* Acquire lock with the GIL held — non-blocking try first, then release * GIL and block if contested (matches hashlib's ENTER_HASHLIB in 3.9-3.12). - * Only acquires if lock has been allocated (lazy init). */ -# define XXHASH_LOCK_ACQUIRE(o) \ - do { \ - if ((o)->lock) { \ - if (!PyThread_acquire_lock((o)->lock, NOWAIT_LOCK)) { \ - /* Lock contested – release GIL while waiting. */ \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ - Py_END_ALLOW_THREADS \ - } \ - } \ - } while (0) - -# define XXHASH_LOCK_RELEASE(o) \ - do { \ - if ((o)->lock) { \ - PyThread_release_lock((o)->lock); \ - } \ - } while (0) + * WAIT_LOCK under GIL would deadlock: if another thread holds the object lock + * (large data path, GIL released), this thread blocks with GIL held, preventing + * the holder from re-acquiring GIL to release the lock. */ +# define XXHASH_LOCK_ACQUIRE(o) \ + do { \ + if ((o)->lock) { \ + if (!PyThread_acquire_lock((o)->lock, NOWAIT_LOCK)) { \ + /* Lock contested — release GIL while waiting. */ \ + Py_BEGIN_ALLOW_THREADS \ + PyThread_acquire_lock((o)->lock, WAIT_LOCK); \ + Py_END_ALLOW_THREADS \ + } \ + } \ + } while (0) + +# define XXHASH_LOCK_RELEASE(o) \ + do { \ + if ((o)->lock) { \ + PyThread_release_lock((o)->lock); \ + } \ + } while (0) +# endif +#else /* !XXHASH_WITH_LOCK */ +# define XXHASH_LOCK_FIELD +# define XXHASH_LOCK_INIT(o) (0) +# define XXHASH_LOCK_FINI(o) ((void)0) +# define XXHASH_LOCK_ACQUIRE(o) ((void)0) +# define XXHASH_LOCK_ACQUIRE_BLOCKING(o) ((void)0) +# define XXHASH_LOCK_RELEASE(o) ((void)0) #endif -/* Data size threshold for releasing the GIL during hash. */ +/* Data size threshold for releasing the GIL during hash update. */ #define XXHASH_GIL_MINSIZE 65536 + +/* ------------------------------------------------------------------ */ +/* Stringification helpers */ +/* ------------------------------------------------------------------ */ #define TOSTRING(x) #x #define VALUE_TO_STRING(x) TOSTRING(x) -#define XXHASH_VERSION XXH_VERSION_MAJOR.XXH_VERSION_MINOR.XXH_VERSION_RELEASE + +/* ------------------------------------------------------------------ */ +/* Build configuration: module & type names */ +/* ------------------------------------------------------------------ */ +/* Module name is parameterised so the same source file can be compiled + * as both xxhash._xxhash and xxhash._xxhash_threadsafe. */ +#ifndef XXHASH_MODULE_NAME +# define XXHASH_MODULE_NAME _xxhash +#endif +/* Display prefix for tp_name (repr), separate from the C extension + * module name. Default: "xxhash". Threadsafe build: "xxhash.threadsafe". */ +#ifndef XXHASH_TP_NAME_PREFIX +# define XXHASH_TP_NAME_PREFIX xxhash +#endif +/* Token concatenation helpers for PyInit_. */ +#define XXHASH_PASTE2(a, b) a ## b +#define XXHASH_PASTE(a, b) XXHASH_PASTE2(a, b) +#define XXHASH_PYINIT(name) XXHASH_PASTE(PyInit_, name) +/* Type name for repr(), e.g. "xxhash.xxh32" or "xxhash.threadsafe.xxh32". */ +#define XXHASH_TP_NAME(base) VALUE_TO_STRING(XXHASH_TP_NAME_PREFIX) "." base + + +/* ------------------------------------------------------------------ */ +/* xxHash constants */ +/* ------------------------------------------------------------------ */ #define XXH32_DIGESTSIZE 4 #define XXH32_BLOCKSIZE 16 #define XXH64_DIGESTSIZE 8 #define XXH64_BLOCKSIZE 32 #define XXH128_DIGESTSIZE 16 #define XXH128_BLOCKSIZE 64 +#define XXHASH_VERSION XXH_VERSION_MAJOR.XXH_VERSION_MINOR.XXH_VERSION_RELEASE -#ifndef Py_ALWAYS_INLINE -# define Py_ALWAYS_INLINE -#endif -/* Hex lookup table for hexdigest(). */ +/* ------------------------------------------------------------------ */ +/* Code generation helpers */ +/* ------------------------------------------------------------------ */ + +/* Generate _do_update for each hash type. + * Lock is always acquired (no-op in non-lock build). For large data + * release GIL while blocking on lock to avoid stalling other threads. */ +#define XXHASH_DO_UPDATE(type, update_fn) \ +static inline void \ +PY##type##_do_update(PY##type##Object *self, Py_buffer *buf) \ +{ \ + if (buf->len > XXHASH_GIL_MINSIZE) { \ + /* Release GIL first, then block on lock. */ \ + Py_BEGIN_ALLOW_THREADS \ + XXHASH_LOCK_ACQUIRE_BLOCKING(self); \ + update_fn(self->xxhash_state, buf->buf, buf->len); \ + XXHASH_LOCK_RELEASE(self); \ + Py_END_ALLOW_THREADS \ + } else { \ + /* Acquire lock with GIL held (try-then-block). */ \ + XXHASH_LOCK_ACQUIRE(self); \ + update_fn(self->xxhash_state, buf->buf, buf->len); \ + XXHASH_LOCK_RELEASE(self); \ + } \ + PyBuffer_Release(buf); \ +} + +/* Generate __init__ for each hash type. */ +#define XXHASH_INIT(type, reset_fn, update_fn, seed_cast) \ +static int PY##type##_init(PY##type##Object *self, \ + PyObject *args, PyObject *kwargs) \ +{ \ + unsigned long long seed_val = 0; \ + PyObject *data_obj = NULL; \ + Py_buffer buf = {NULL, NULL}; \ + \ + if (_parse_init_args(args, kwargs, &data_obj, &seed_val,\ + "__init__()") < 0) \ + return -1; \ + \ + if (data_obj) { \ + if (_get_buffer_or_str(data_obj, &buf) < 0) \ + return -1; \ + } \ + \ + XXHASH_LOCK_ACQUIRE(self); \ + self->seed = (seed_cast)seed_val; \ + reset_fn(self->xxhash_state, self->seed); \ + \ + if (buf.obj) { \ + update_fn(self->xxhash_state, buf.buf, buf.len); \ + PyBuffer_Release(&buf); \ + } \ + XXHASH_LOCK_RELEASE(self); \ + return 0; \ +} + +/* Generate update() method for each hash type (with keyword support). */ +#define XXHASH_UPDATE_METHOD(prefix, name) \ +PyDoc_STRVAR( \ + PY##prefix##_update_doc, \ + "update (data)\n\n" \ + "Update the " name " object with bytes-like data. Repeated calls are\n" \ + "equivalent to a single call with the concatenation of all the arguments."); \ + \ +static PyObject *PY##prefix##_update(PY##prefix##Object *self, \ + PyObject *const *args, \ + Py_ssize_t nargs, PyObject *kwnames) \ +{ \ + PyObject *arg = NULL; \ + \ + /* validate keywords first */ \ + if (kwnames) { \ + Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); \ + for (Py_ssize_t i = 0; i < nkw; i++) { \ + PyObject *key = PyTuple_GET_ITEM(kwnames, i); \ + if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { \ + if (nargs >= 1) { \ + PyErr_SetString(PyExc_TypeError, \ + name ".update() got multiple values for argument 'data'"); \ + return NULL; \ + } \ + arg = args[nargs + i]; \ + } else { \ + PyErr_Format(PyExc_TypeError, \ + "'%U' is an invalid keyword argument for '" name ".update()'", \ + key); \ + return NULL; \ + } \ + } \ + } \ + \ + if (nargs >= 1) { \ + if (nargs > 1) { \ + PyErr_Format(PyExc_TypeError, \ + name ".update() takes at most 1 positional argument (%zd given)", \ + nargs); \ + return NULL; \ + } \ + arg = args[0]; \ + } \ + \ + if (!arg) { \ + PyErr_SetString(PyExc_TypeError, \ + name ".update() missing required argument 'data'"); \ + return NULL; \ + } \ + \ + Py_buffer buf = {NULL, NULL}; \ + if (_get_buffer_or_str(arg, &buf) < 0) \ + return NULL; \ + PY##prefix##_do_update(self, &buf); \ + Py_RETURN_NONE; \ +} + /* Get a buffer from an object. Rejects str with hashlib-compatible error. */ -static inline Py_ALWAYS_INLINE int +static inline int _get_buffer_or_str(PyObject *obj, Py_buffer *buf) { if (obj == Py_None) { @@ -623,35 +764,7 @@ static void PYXXH32_dealloc(PYXXH32Object *self) Py_DECREF(tp); } -/* Macro to generate _do_update for each hash type. - * Matches CPython 3.9-3.12 md5 pattern: release GIL first (for large data), - * then acquire lock, hash, release lock, re-acquire GIL. - * For small data, acquire lock with GIL held (try-then-block if contested). */ -#define XXHASH_DO_UPDATE(type, update_fn) \ -static inline Py_ALWAYS_INLINE void \ -PY##type##_do_update(PY##type##Object *self, Py_buffer *buf) \ -{ \ - XXHASH_LOCK_MAYBE_INIT(self, buf->len); \ - if (XXHASH_LOCK_IS_ACTIVE(self)) { \ - if (buf->len > XXHASH_GIL_MINSIZE) { \ - /* Release GIL first, then acquire lock. */ \ - Py_BEGIN_ALLOW_THREADS \ - XXHASH_LOCK_ACQUIRE_BLOCKING(self); \ - update_fn(self->xxhash_state, buf->buf, buf->len); \ - XXHASH_LOCK_RELEASE(self); \ - Py_END_ALLOW_THREADS \ - } else { \ - /* Acquire lock with GIL held. */ \ - XXHASH_LOCK_ACQUIRE(self); \ - update_fn(self->xxhash_state, buf->buf, buf->len); \ - XXHASH_LOCK_RELEASE(self); \ - } \ - } else { \ - /* No lock: hash directly, no GIL release. */ \ - update_fn(self->xxhash_state, buf->buf, buf->len); \ - } \ - PyBuffer_Release(buf); \ -} +/* Invoke _do_update generator for each hash type. */ XXHASH_DO_UPDATE(XXH32, XXH32_update) @@ -664,7 +777,7 @@ PYXXH32_vectorcall(PyObject *type, PyObject *const *args, Py_buffer buf; unsigned long long raw_seed; - if (_parse_fastcall_args(args, nargs, kwnames, "xxhash.xxh32", 0, + if (_parse_fastcall_args(args, nargs, kwnames, XXHASH_TP_NAME("xxh32"), 0, &buf, &raw_seed) < 0) return NULL; seed = (XXH32_hash_t)raw_seed; @@ -676,7 +789,11 @@ PYXXH32_vectorcall(PyObject *type, PyObject *const *args, return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + PyBuffer_Release(&buf); + return NULL; + } self->xxhash_state = XXH32_createState(); if (self->xxhash_state == NULL) { @@ -711,12 +828,14 @@ static PyObject *PYXXH32_new(PyTypeObject *type, PyObject *args, PyObject *kwarg return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + return NULL; + } if ((self->xxhash_state = XXH32_createState()) == NULL) { Py_DECREF(self); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } self->seed = 0; @@ -737,9 +856,7 @@ _parse_init_args(PyObject *args, PyObject *kwargs, { Py_ssize_t nargs = PyTuple_GET_SIZE(args); - if (!kwargs) { - /* fast path: no keywords */ - } else { + if (kwargs) { Py_ssize_t pos = 0; PyObject *key, *val; while (PyDict_Next(kwargs, &pos, &key, &val)) { @@ -747,7 +864,8 @@ _parse_init_args(PyObject *args, PyObject *kwargs, PyUnicode_CompareWithASCIIString(key, "seed") == 0) continue; PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword argument for this function", key); + "'%U' is an invalid keyword argument for '%s()'", + key, funcname); return -1; } } @@ -794,90 +912,13 @@ _parse_init_args(PyObject *args, PyObject *kwargs, return 0; } -/* Macro to generate __init__ for each hash type. */ -#define XXHASH_INIT(type, reset_fn, update_fn, seed_cast) \ -static int PY##type##_init(PY##type##Object *self, PyObject *args, \ - PyObject *kwargs) \ -{ \ - unsigned long long seed_val = 0; \ - PyObject *data_obj = NULL; \ - Py_buffer buf = {NULL, NULL}; \ - \ - if (_parse_init_args(args, kwargs, &data_obj, &seed_val, \ - "__init__()") < 0) \ - return -1; \ - \ - if (data_obj) { \ - if (_get_buffer_or_str(data_obj, &buf) < 0) \ - return -1; \ - } \ - \ - XXHASH_LOCK_ACQUIRE(self); \ - self->seed = (seed_cast)seed_val; \ - reset_fn(self->xxhash_state, self->seed); \ - \ - if (buf.obj) { \ - update_fn(self->xxhash_state, buf.buf, buf.len); \ - PyBuffer_Release(&buf); \ - } \ - XXHASH_LOCK_RELEASE(self); \ - return 0; \ -} +/* Generate __init__ for each hash type. */ XXHASH_INIT(XXH32, XXH32_reset, XXH32_update, XXH32_hash_t) -PyDoc_STRVAR( - PYXXH32_update_doc, - "update (data)\n\n" - "Update the xxh32 object with bytes-like data. Repeated calls are\n" - "equivalent to a single call with the concatenation of all the arguments."); - -static PyObject *PYXXH32_update(PYXXH32Object *self, PyObject *const *args, - Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *arg = NULL; - - /* validate keywords first */ - if (kwnames) { - Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < nkw; i++) { - PyObject *key = PyTuple_GET_ITEM(kwnames, i); - if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { - if (nargs >= 1) { - PyErr_SetString(PyExc_TypeError, - "xxh32.update() got multiple values for argument 'data'"); - return NULL; - } - arg = args[nargs + i]; - } else { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword argument for 'xxh32.update()'", - key); - return NULL; - } - } - } - - if (nargs >= 1) { - if (nargs > 1) { - PyErr_Format(PyExc_TypeError, - "xxh32.update() takes at most 1 positional argument (%zd given)", nargs); - return NULL; - } - arg = args[0]; - } +/* Generate update() for each hash type. */ - if (!arg) { - PyErr_SetString(PyExc_TypeError, "xxh32.update() missing required argument 'data'"); - return NULL; - } - - Py_buffer buf; - if (_get_buffer_or_str(arg, &buf) < 0) - return NULL; - PYXXH32_do_update(self, &buf); - Py_RETURN_NONE; -} +XXHASH_UPDATE_METHOD(XXH32, "xxh32") PyDoc_STRVAR( PYXXH32_digest_doc, @@ -957,12 +998,14 @@ static PyObject *PYXXH32_copy(PYXXH32Object *self) return NULL; } - XXHASH_LOCK_INIT(p); + if (XXHASH_LOCK_INIT(p) < 0) { + Py_DECREF(p); + return NULL; + } if ((p->xxhash_state = XXH32_createState()) == NULL) { Py_DECREF(p); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } XXHASH_LOCK_ACQUIRE(self); @@ -1077,7 +1120,7 @@ static PyType_Slot XXH32Type_slots[] = { }; static PyType_Spec XXH32Type_spec = { - .name = "xxhash.xxh32", + .name = XXHASH_TP_NAME("xxh32"), .basicsize = sizeof(PYXXH32Object), .flags = Py_TPFLAGS_DEFAULT #if PY_VERSION_HEX >= 0x030c0000 @@ -1118,7 +1161,7 @@ PYXXH64_vectorcall(PyObject *type, PyObject *const *args, Py_buffer buf; unsigned long long raw_seed; - if (_parse_fastcall_args(args, nargs, kwnames, "xxhash.xxh64", 0, + if (_parse_fastcall_args(args, nargs, kwnames, XXHASH_TP_NAME("xxh64"), 0, &buf, &raw_seed) < 0) return NULL; seed = (XXH64_hash_t)raw_seed; @@ -1130,7 +1173,11 @@ PYXXH64_vectorcall(PyObject *type, PyObject *const *args, return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + PyBuffer_Release(&buf); + return NULL; + } self->xxhash_state = XXH64_createState(); if (self->xxhash_state == NULL) { @@ -1162,12 +1209,14 @@ static PyObject *PYXXH64_new(PyTypeObject *type, PyObject *args, PyObject *kwarg return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + return NULL; + } if ((self->xxhash_state = XXH64_createState()) == NULL) { Py_DECREF(self); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } self->seed = 0; @@ -1178,58 +1227,7 @@ static PyObject *PYXXH64_new(PyTypeObject *type, PyObject *args, PyObject *kwarg XXHASH_INIT(XXH64, XXH64_reset, XXH64_update, XXH64_hash_t) -PyDoc_STRVAR( - PYXXH64_update_doc, - "update (data)\n\n" - "Update the xxh64 object with bytes-like data. Repeated calls are\n" - "equivalent to a single call with the concatenation of all the arguments."); - -static PyObject *PYXXH64_update(PYXXH64Object *self, PyObject *const *args, - Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *arg = NULL; - - /* validate keywords first */ - if (kwnames) { - Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < nkw; i++) { - PyObject *key = PyTuple_GET_ITEM(kwnames, i); - if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { - if (nargs >= 1) { - PyErr_SetString(PyExc_TypeError, - "xxh64.update() got multiple values for argument 'data'"); - return NULL; - } - arg = args[nargs + i]; - } else { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword argument for 'xxh64.update()'", - key); - return NULL; - } - } - } - - if (nargs >= 1) { - if (nargs > 1) { - PyErr_Format(PyExc_TypeError, - "xxh64.update() takes at most 1 positional argument (%zd given)", nargs); - return NULL; - } - arg = args[0]; - } - - if (!arg) { - PyErr_SetString(PyExc_TypeError, "xxh64.update() missing required argument 'data'"); - return NULL; - } - - Py_buffer buf; - if (_get_buffer_or_str(arg, &buf) < 0) - return NULL; - PYXXH64_do_update(self, &buf); - Py_RETURN_NONE; -} +XXHASH_UPDATE_METHOD(XXH64, "xxh64") PyDoc_STRVAR( PYXXH64_digest_doc, @@ -1309,12 +1307,14 @@ static PyObject *PYXXH64_copy(PYXXH64Object *self) return NULL; } - XXHASH_LOCK_INIT(p); + if (XXHASH_LOCK_INIT(p) < 0) { + Py_DECREF(p); + return NULL; + } if ((p->xxhash_state = XXH64_createState()) == NULL) { Py_DECREF(p); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } XXHASH_LOCK_ACQUIRE(self); @@ -1429,7 +1429,7 @@ static PyType_Slot XXH64Type_slots[] = { }; static PyType_Spec XXH64Type_spec = { - .name = "xxhash.xxh64", + .name = XXHASH_TP_NAME("xxh64"), .basicsize = sizeof(PYXXH64Object), .flags = Py_TPFLAGS_DEFAULT #if PY_VERSION_HEX >= 0x030c0000 @@ -1470,7 +1470,7 @@ PYXXH3_64_vectorcall(PyObject *type, PyObject *const *args, Py_buffer buf; unsigned long long raw_seed; - if (_parse_fastcall_args(args, nargs, kwnames, "xxhash.xxh3_64", 0, + if (_parse_fastcall_args(args, nargs, kwnames, XXHASH_TP_NAME("xxh3_64"), 0, &buf, &raw_seed) < 0) return NULL; seed = (XXH64_hash_t)raw_seed; @@ -1482,7 +1482,11 @@ PYXXH3_64_vectorcall(PyObject *type, PyObject *const *args, return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + PyBuffer_Release(&buf); + return NULL; + } self->xxhash_state = XXH3_createState(); if (self->xxhash_state == NULL) { @@ -1514,12 +1518,14 @@ static PyObject *PYXXH3_64_new(PyTypeObject *type, PyObject *args, PyObject *kwa return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + return NULL; + } if ((self->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(self); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } self->seed = 0; @@ -1530,58 +1536,7 @@ static PyObject *PYXXH3_64_new(PyTypeObject *type, PyObject *args, PyObject *kwa XXHASH_INIT(XXH3_64, XXH3_64bits_reset_withSeed, XXH3_64bits_update, XXH64_hash_t) -PyDoc_STRVAR( - PYXXH3_64_update_doc, - "update (data)\n\n" - "Update the xxh3_64 object with bytes-like data. Repeated calls are\n" - "equivalent to a single call with the concatenation of all the arguments."); - -static PyObject *PYXXH3_64_update(PYXXH3_64Object *self, PyObject *const *args, - Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *arg = NULL; - - /* validate keywords first */ - if (kwnames) { - Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < nkw; i++) { - PyObject *key = PyTuple_GET_ITEM(kwnames, i); - if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { - if (nargs >= 1) { - PyErr_SetString(PyExc_TypeError, - "xxh3_64.update() got multiple values for argument 'data'"); - return NULL; - } - arg = args[nargs + i]; - } else { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword argument for 'xxh3_64.update()'", - key); - return NULL; - } - } - } - - if (nargs >= 1) { - if (nargs > 1) { - PyErr_Format(PyExc_TypeError, - "xxh3_64.update() takes at most 1 positional argument (%zd given)", nargs); - return NULL; - } - arg = args[0]; - } - - if (!arg) { - PyErr_SetString(PyExc_TypeError, "xxh3_64.update() missing required argument 'data'"); - return NULL; - } - - Py_buffer buf; - if (_get_buffer_or_str(arg, &buf) < 0) - return NULL; - PYXXH3_64_do_update(self, &buf); - Py_RETURN_NONE; -} +XXHASH_UPDATE_METHOD(XXH3_64, "xxh3_64") PyDoc_STRVAR( PYXXH3_64_digest_doc, @@ -1650,8 +1605,8 @@ static PyObject *PYXXH3_64_intdigest(PYXXH3_64Object *self) PyDoc_STRVAR( PYXXH3_64_copy_doc, - "copy() -> xxh64 object\n\n" - "Return a copy (``clone'') of the xxh64 object."); + "copy() -> xxh3_64 object\n\n" + "Return a copy (``clone'') of the xxh3_64 object."); static PyObject *PYXXH3_64_copy(PYXXH3_64Object *self) { @@ -1661,12 +1616,14 @@ static PyObject *PYXXH3_64_copy(PYXXH3_64Object *self) return NULL; } - XXHASH_LOCK_INIT(p); + if (XXHASH_LOCK_INIT(p) < 0) { + Py_DECREF(p); + return NULL; + } if ((p->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(p); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } XXHASH_LOCK_ACQUIRE(self); @@ -1775,7 +1732,7 @@ PyDoc_STRVAR( "digest() -- return the current digest value\n" "hexdigest() -- return the current digest as a string of hexadecimal digits\n" "intdigest() -- return the current digest as an integer\n" - "copy() -- return a copy of the current xxh64 object"); + "copy() -- return a copy of the current xxh3_64 object"); static PyType_Slot XXH3_64Type_slots[] = { {Py_tp_dealloc, PYXXH3_64_dealloc}, @@ -1788,7 +1745,7 @@ static PyType_Slot XXH3_64Type_slots[] = { }; static PyType_Spec XXH3_64Type_spec = { - .name = "xxhash.xxh3_64", + .name = XXHASH_TP_NAME("xxh3_64"), .basicsize = sizeof(PYXXH3_64Object), .flags = Py_TPFLAGS_DEFAULT #if PY_VERSION_HEX >= 0x030c0000 @@ -1829,7 +1786,7 @@ PYXXH3_128_vectorcall(PyObject *type, PyObject *const *args, Py_buffer buf; unsigned long long raw_seed; - if (_parse_fastcall_args(args, nargs, kwnames, "xxhash.xxh3_128", 0, + if (_parse_fastcall_args(args, nargs, kwnames, XXHASH_TP_NAME("xxh3_128"), 0, &buf, &raw_seed) < 0) return NULL; seed = (XXH64_hash_t)raw_seed; @@ -1841,7 +1798,11 @@ PYXXH3_128_vectorcall(PyObject *type, PyObject *const *args, return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + PyBuffer_Release(&buf); + return NULL; + } self->xxhash_state = XXH3_createState(); if (self->xxhash_state == NULL) { @@ -1873,12 +1834,14 @@ static PyObject *PYXXH3_128_new(PyTypeObject *type, PyObject *args, PyObject *kw return NULL; } - XXHASH_LOCK_INIT(self); + if (XXHASH_LOCK_INIT(self) < 0) { + Py_DECREF(self); + return NULL; + } if ((self->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(self); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } self->seed = 0; @@ -1889,58 +1852,7 @@ static PyObject *PYXXH3_128_new(PyTypeObject *type, PyObject *args, PyObject *kw XXHASH_INIT(XXH3_128, XXH3_128bits_reset_withSeed, XXH3_128bits_update, XXH64_hash_t) -PyDoc_STRVAR( - PYXXH3_128_update_doc, - "update (data)\n\n" - "Update the xxh3_128 object with bytes-like data. Repeated calls are\n" - "equivalent to a single call with the concatenation of all the arguments."); - -static PyObject *PYXXH3_128_update(PYXXH3_128Object *self, PyObject *const *args, - Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *arg = NULL; - - /* validate keywords first */ - if (kwnames) { - Py_ssize_t nkw = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < nkw; i++) { - PyObject *key = PyTuple_GET_ITEM(kwnames, i); - if (PyUnicode_CompareWithASCIIString(key, "data") == 0) { - if (nargs >= 1) { - PyErr_SetString(PyExc_TypeError, - "xxh3_128.update() got multiple values for argument 'data'"); - return NULL; - } - arg = args[nargs + i]; - } else { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword argument for 'xxh3_128.update()'", - key); - return NULL; - } - } - } - - if (nargs >= 1) { - if (nargs > 1) { - PyErr_Format(PyExc_TypeError, - "xxh3_128.update() takes at most 1 positional argument (%zd given)", nargs); - return NULL; - } - arg = args[0]; - } - - if (!arg) { - PyErr_SetString(PyExc_TypeError, "xxh3_128.update() missing required argument 'data'"); - return NULL; - } - - Py_buffer buf; - if (_get_buffer_or_str(arg, &buf) < 0) - return NULL; - PYXXH3_128_do_update(self, &buf); - Py_RETURN_NONE; -} +XXHASH_UPDATE_METHOD(XXH3_128, "xxh3_128") PyDoc_STRVAR( PYXXH3_128_digest_doc, @@ -2039,12 +1951,14 @@ static PyObject *PYXXH3_128_copy(PYXXH3_128Object *self) return NULL; } - XXHASH_LOCK_INIT(p); + if (XXHASH_LOCK_INIT(p) < 0) { + Py_DECREF(p); + return NULL; + } if ((p->xxhash_state = XXH3_createState()) == NULL) { Py_DECREF(p); - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } XXHASH_LOCK_ACQUIRE(self); @@ -2166,7 +2080,7 @@ static PyType_Slot XXH3_128Type_slots[] = { }; static PyType_Spec XXH3_128Type_spec = { - .name = "xxhash.xxh3_128", + .name = XXHASH_TP_NAME("xxh3_128"), .basicsize = sizeof(PYXXH3_128Object), .flags = Py_TPFLAGS_DEFAULT #if PY_VERSION_HEX >= 0x030c0000 @@ -2230,7 +2144,10 @@ static PyModuleDef_Slot slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, #endif #if PY_VERSION_HEX >= 0x030d0000 - /* Python 3.13+: module is thread-safe with per-object lock */ + /* Both variants manage their own synchronization guarantees: + * the thread-safe variant uses a per-object lock; the default + * variant requires callers not to share streaming hash objects + * across threads. */ {Py_mod_gil, Py_MOD_GIL_NOT_USED}, #endif {0, NULL} @@ -2254,7 +2171,7 @@ static PyMethodDef methods[] = { static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, - "_xxhash", + VALUE_TO_STRING(XXHASH_MODULE_NAME), NULL, 0, methods, @@ -2265,7 +2182,7 @@ static struct PyModuleDef moduledef = { }; PyMODINIT_FUNC -PyInit__xxhash(void) +XXHASH_PYINIT(XXHASH_MODULE_NAME)(void) { return PyModuleDef_Init(&moduledef); } diff --git a/tests/test_algorithms_available.py b/tests/test_algorithms_available.py deleted file mode 100644 index 3c3f1fd..0000000 --- a/tests/test_algorithms_available.py +++ /dev/null @@ -1,28 +0,0 @@ -import xxhash -import unittest - - -class TestAlgorithmExists(unittest.TestCase): - def test_xxh32(self): - xxhash.xxh32 - assert "xxh32" in xxhash.algorithms_available - - def test_xxh64(self): - xxhash.xxh64 - assert "xxh64" in xxhash.algorithms_available - - def test_xxh3_64(self): - xxhash.xxh3_64 - assert "xxh3_64" in xxhash.algorithms_available - - def test_xxh128(self): - xxhash.xxh128 - assert "xxh128" in xxhash.algorithms_available - - def test_xxh3_128(self): - xxhash.xxh3_128 - assert "xxh3_128" in xxhash.algorithms_available - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_hashlib_compat.py b/tests/test_hashlib_compat.py index fa21b90..f2be6e3 100644 --- a/tests/test_hashlib_compat.py +++ b/tests/test_hashlib_compat.py @@ -10,12 +10,26 @@ class TestHashlibCompat(unittest.TestCase): def test_algorithms_available(self): self.assertIsInstance(xxhash.algorithms_available, set) - for a in ('xxh32', 'xxh64', 'xxh3_64', 'xxh3_128', 'xxh128'): + expected = {'xxh32', 'xxh64', 'xxh3_64', 'xxh3_128', 'xxh128'} + self.assertGreaterEqual(xxhash.algorithms_available, expected) + for a in expected: self.assertIn(a, xxhash.algorithms_available) def test_algorithms_guaranteed(self): self.assertEqual(xxhash.algorithms_guaranteed, xxhash.algorithms_available) + def test_name(self): + expected = { + 'xxh32': 'XXH32', + 'xxh64': 'XXH64', + 'xxh3_64': 'XXH3_64', + 'xxh3_128': 'XXH3_128', + 'xxh128': 'XXH3_128', + } + for algo, name in expected.items(): + with self.subTest(algo=algo): + self.assertEqual(getattr(xxhash, algo)().name, name) + # ── str rejection ────────────────────────────────────────────── def test_str_rejected(self): @@ -120,12 +134,6 @@ def test_block_size(self): self.assertEqual(xxhash.xxh3_64().block_size, 32) self.assertEqual(xxhash.xxh3_128().block_size, 64) - def test_name(self): - self.assertEqual(xxhash.xxh32().name, 'XXH32') - self.assertEqual(xxhash.xxh64().name, 'XXH64') - self.assertEqual(xxhash.xxh3_64().name, 'XXH3_64') - self.assertEqual(xxhash.xxh3_128().name, 'XXH3_128') - # ── digest / hexdigest ───────────────────────────────────────── def test_digest(self): @@ -162,3 +170,7 @@ def test_copy(self): self.assertEqual(a.digest(), b.digest()) b.update(b'more') self.assertNotEqual(a.digest(), b.digest()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_name.py b/tests/test_name.py deleted file mode 100644 index f8fd479..0000000 --- a/tests/test_name.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import unittest -import random -import xxhash - - -class TestName(unittest.TestCase): - def test_xxh32(self): - self.assertEqual(xxhash.xxh32().name, "XXH32") - - def test_xxh64(self): - self.assertEqual(xxhash.xxh64().name, "XXH64") - - def test_xxh3_64(self): - self.assertEqual(xxhash.xxh3_64().name, "XXH3_64") - - def test_xxh128(self): - self.assertEqual(xxhash.xxh128().name, "XXH3_128") - - def test_xxh3_128(self): - self.assertEqual(xxhash.xxh3_128().name, "XXH3_128") - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_thread_safety.py b/tests/test_thread_safety.py index 24852ba..c254a4d 100644 --- a/tests/test_thread_safety.py +++ b/tests/test_thread_safety.py @@ -1,18 +1,15 @@ """ -Thread-safety tests for xxhash. +Thread-safety tests for xxhash.threadsafe. -Previously, the C extension released the GIL inside ``_do_update`` (via -``Py_BEGIN_ALLOW_THREADS`` / ``Py_END_ALLOW_THREADS``) while calling -``XXH*_update(self->xxhash_state, ...)``, and all other methods accessed the -same ``xxhash_state`` without any per-object lock — creating a data race. - -This has been fixed by adding a per-object ``PyThread_type_lock`` that protects -all access to ``xxhash_state``. The lock is acquired **after** releasing the -GIL in ``update()`` to avoid ABBA deadlocks. +The default ``xxhash`` module is optimized for speed and does not protect +streaming hash objects with a per-object lock. Concurrent access to the same +hash object from multiple threads is only safe when using the +``xxhash.threadsafe`` submodule, which adds a per-object lock around every +operation that touches the internal xxHash state. The tests below verify that: - * no crashes occur under concurrent access - * hash results are now deterministic (no data races) + * no crashes occur under concurrent access to ``threadsafe`` hash objects + * hash results are deterministic (no data races) """ import os @@ -20,7 +17,7 @@ import subprocess import signal import unittest -import xxhash +from xxhash import threadsafe as xxhash # --------------------------------------------------------------------------- @@ -47,7 +44,7 @@ def _run_in_subprocess(code: str, timeout: float = 60.0): # --------------------------------------------------------------------------- CONCURRENT_DIGEST_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh32() BLOCK = b'x' * (4 * 1024 * 1024) # 4 MiB @@ -93,7 +90,7 @@ def digester(): # --------------------------------------------------------------------------- CONCURRENT_RESET_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh32() BLOCK = b'x' * (4 * 1024 * 1024) @@ -139,7 +136,7 @@ def reseter(): # --------------------------------------------------------------------------- CONCURRENT_UPDATE_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh32() BLOCK = b'x' * (4 * 1024 * 1024) @@ -184,7 +181,7 @@ def updater(): # --------------------------------------------------------------------------- NON_DETERMINISM_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh32() BLOCK = b'x' * (4 * 1024 * 1024) @@ -224,7 +221,7 @@ def updater(): # --------------------------------------------------------------------------- XXH128_UPDATE_RESET_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh128() BLOCK = b'x' * 16 # tiny block: more calls = more race windows @@ -278,7 +275,7 @@ def reseter(): # --------------------------------------------------------------------------- XXH128_UPDATE_COPY_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh128() copies = [] @@ -337,7 +334,7 @@ def copier(): # --------------------------------------------------------------------------- XXH128_ALL_METHODS_CODE = r""" -import sys, random, threading, xxhash +import sys, random, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh128() BLOCK = b'x' * 32 @@ -386,7 +383,7 @@ def worker(): # --------------------------------------------------------------------------- XXH64_AGGRESSIVE_RACE_CODE = r""" -import sys, threading, xxhash +import sys, threading; from xxhash import threadsafe as xxhash h = xxhash.xxh64() BLOCK = b'x' * 16 @@ -423,9 +420,10 @@ def worker(): class TestThreadSafety(unittest.TestCase): - """Verify that concurrent access to a single hash object does not crash. + """Verify that concurrent access to a single threadsafe hash object works. - With per-object locking in place, concurrent access is now safe. + These tests import ``xxhash.threadsafe`` (not the default ``xxhash`` + module), which uses a per-object lock around all streaming operations. We run each scenario many times (``REPETITIONS``) to verify that no crashes, deadlocks, or unexpected exceptions occur. """ @@ -509,8 +507,7 @@ class TestNonDeterminism(unittest.TestCase): TIMEOUT = int(os.environ.get("XXHASH_TEST_TIMEOUT", "120")) def test_concurrent_update_is_deterministic(self): - """Concurrent update() should be deterministic (FIXED: per-object - lock now prevents data races).""" + """Concurrent update() on a threadsafe object should be deterministic.""" digests = set() for i in range(self.SAMPLES): rc, out, err = _run_in_subprocess( diff --git a/tests/test_threadsafe_api.py b/tests/test_threadsafe_api.py new file mode 100644 index 0000000..4dbdc73 --- /dev/null +++ b/tests/test_threadsafe_api.py @@ -0,0 +1,334 @@ +""" +Basic API tests for the ``xxhash.threadsafe`` module. + +Verifies that: + * the ``xxhash.threadsafe`` submodule is importable + * all 4 hash types (xxh32, xxh64, xxh3_64, xxh128/xxh3_128) work + * streaming methods (update, digest, hexdigest, intdigest, copy, reset) work + * results match the default ``xxhash`` module exactly + * ``repr()`` and ``type.__module__`` reflect the public module name +""" + +import os +import unittest +import xxhash +from xxhash import threadsafe + + +# Known good values from the default xxhash module. +XXH32_A = (b'a', 1426945110) +XXH64_A = (b'a', 15154266338359012955) +XXH3_64_A = (b'a', 16629034431890738719) +XXH3_128_A = (b'a', 225219434562328483135862406050043285023) + +class TestThreadsafeTypes(unittest.TestCase): + """Verify that all 4 hash types exist and produce correct results.""" + + def test_xxh32_object(self): + h = threadsafe.xxh32(b'a') + self.assertEqual(h.intdigest(), XXH32_A[1]) + + def test_xxh32_seed(self): + h = threadsafe.xxh32(b'a', seed=42) + self.assertEqual(h.intdigest(), threadsafe.xxh32(b'a', 42).intdigest()) + + def test_xxh32_matches_default(self): + default = xxhash.xxh32(b'hello world', 123) + safe = threadsafe.xxh32(b'hello world', 123) + self.assertEqual(default.digest(), safe.digest()) + self.assertEqual(default.hexdigest(), safe.hexdigest()) + self.assertEqual(default.intdigest(), safe.intdigest()) + + def test_xxh64_object(self): + h = threadsafe.xxh64(b'a') + self.assertEqual(h.intdigest(), XXH64_A[1]) + + def test_xxh64_matches_default(self): + default = xxhash.xxh64(b'hello world', 123) + safe = threadsafe.xxh64(b'hello world', 123) + self.assertEqual(default.digest(), safe.digest()) + self.assertEqual(default.hexdigest(), safe.hexdigest()) + self.assertEqual(default.intdigest(), safe.intdigest()) + + def test_xxh3_64_object(self): + h = threadsafe.xxh3_64(b'a') + self.assertEqual(h.intdigest(), XXH3_64_A[1]) + + def test_xxh3_64_matches_default(self): + default = xxhash.xxh3_64(b'hello world', 123) + safe = threadsafe.xxh3_64(b'hello world', 123) + self.assertEqual(default.digest(), safe.digest()) + self.assertEqual(default.hexdigest(), safe.hexdigest()) + self.assertEqual(default.intdigest(), safe.intdigest()) + + def test_xxh128_object(self): + h = threadsafe.xxh128(b'a') + self.assertEqual(h.intdigest(), XXH3_128_A[1]) + + def test_xxh128_matches_default(self): + default = xxhash.xxh128(b'hello world', 123) + safe = threadsafe.xxh128(b'hello world', 123) + self.assertEqual(default.digest(), safe.digest()) + self.assertEqual(default.hexdigest(), safe.hexdigest()) + self.assertEqual(default.intdigest(), safe.intdigest()) + + +class TestThreadsafeStreaming(unittest.TestCase): + """Verify that streaming operations work correctly.""" + + def test_update(self): + h = threadsafe.xxh32() + h.update(b'a') + self.assertEqual(h.digest(), threadsafe.xxh32(b'a').digest()) + h.update(b'b') + self.assertEqual(h.digest(), threadsafe.xxh32(b'ab').digest()) + h.update(b'c') + self.assertEqual(h.digest(), threadsafe.xxh32(b'abc').digest()) + + def test_update_chain(self): + for typ, data in [ + (threadsafe.xxh32, b'x' * 100000), + (threadsafe.xxh64, b'x' * 100000), + (threadsafe.xxh3_64, b'x' * 100000), + (threadsafe.xxh128, b'x' * 100000), + ]: + with self.subTest(typ=typ): + h = typ() + h.update(data) + self.assertEqual(h.digest(), typ(data).digest()) + + def test_reset(self): + for typ in [threadsafe.xxh32, threadsafe.xxh64, + threadsafe.xxh3_64, threadsafe.xxh128]: + with self.subTest(typ=typ): + h = typ() + initial = h.intdigest() + for _ in range(10): + h.update(os.urandom(64)) + h.reset() + self.assertEqual(initial, h.intdigest()) + + def test_copy(self): + for typ in [threadsafe.xxh32, threadsafe.xxh64, + threadsafe.xxh3_64, threadsafe.xxh128]: + with self.subTest(typ=typ): + a = typ() + a.update(b'hello world') + b = a.copy() + self.assertEqual(a.digest(), b.digest()) + self.assertEqual(a.intdigest(), b.intdigest()) + self.assertEqual(a.hexdigest(), b.hexdigest()) + b.update(b'more data') + self.assertNotEqual(a.digest(), b.digest()) + + def test_digest_hexdigest_intdigest(self): + for typ, known in [ + (threadsafe.xxh32, XXH32_A), + (threadsafe.xxh64, XXH64_A), + (threadsafe.xxh3_64, XXH3_64_A), + (threadsafe.xxh128, XXH3_128_A), + ]: + data, expected = known + with self.subTest(typ=typ): + h = typ(data) + self.assertEqual(h.intdigest(), expected) + self.assertIsInstance(h.digest(), bytes) + self.assertIsInstance(h.hexdigest(), str) + self.assertIsInstance(h.intdigest(), int) + + +class TestThreadsafeTopLevelFunctions(unittest.TestCase): + """Verify that top-level convenience functions work.""" + + def test_xxh32_digest(self): + self.assertEqual( + threadsafe.xxh32_digest(b'a'), + xxhash.xxh32_digest(b'a'), + ) + + def test_xxh32_hexdigest(self): + self.assertEqual( + threadsafe.xxh32_hexdigest(b'a'), + xxhash.xxh32_hexdigest(b'a'), + ) + + def test_xxh32_intdigest(self): + self.assertEqual( + threadsafe.xxh32_intdigest(b'a'), + xxhash.xxh32_intdigest(b'a'), + ) + + def test_xxh64_digest(self): + self.assertEqual( + threadsafe.xxh64_digest(b'a'), + xxhash.xxh64_digest(b'a'), + ) + + def test_xxh64_hexdigest(self): + self.assertEqual( + threadsafe.xxh64_hexdigest(b'a'), + xxhash.xxh64_hexdigest(b'a'), + ) + + def test_xxh64_intdigest(self): + self.assertEqual( + threadsafe.xxh64_intdigest(b'a'), + xxhash.xxh64_intdigest(b'a'), + ) + + def test_xxh3_64_digest(self): + self.assertEqual( + threadsafe.xxh3_64_digest(b'a'), + xxhash.xxh3_64_digest(b'a'), + ) + + def test_xxh3_64_hexdigest(self): + self.assertEqual( + threadsafe.xxh3_64_hexdigest(b'a'), + xxhash.xxh3_64_hexdigest(b'a'), + ) + + def test_xxh3_64_intdigest(self): + self.assertEqual( + threadsafe.xxh3_64_intdigest(b'a'), + xxhash.xxh3_64_intdigest(b'a'), + ) + + def test_xxh128_digest(self): + self.assertEqual( + threadsafe.xxh128_digest(b'a'), + xxhash.xxh128_digest(b'a'), + ) + + def test_xxh128_hexdigest(self): + self.assertEqual( + threadsafe.xxh128_hexdigest(b'a'), + xxhash.xxh128_hexdigest(b'a'), + ) + + def test_xxh128_intdigest(self): + self.assertEqual( + threadsafe.xxh128_intdigest(b'a'), + xxhash.xxh128_intdigest(b'a'), + ) + + +class TestThreadsafeRepr(unittest.TestCase): + """Verify that repr() and __module__ use the public module name.""" + + def test_repr_xxh32(self): + h = threadsafe.xxh32() + self.assertIn("xxhash.threadsafe", repr(h)) + self.assertEqual(type(h).__module__, "xxhash.threadsafe") + + def test_repr_xxh64(self): + h = threadsafe.xxh64() + self.assertIn("xxhash.threadsafe", repr(h)) + self.assertEqual(type(h).__module__, "xxhash.threadsafe") + + def test_repr_xxh3_64(self): + h = threadsafe.xxh3_64() + self.assertIn("xxhash.threadsafe", repr(h)) + self.assertEqual(type(h).__module__, "xxhash.threadsafe") + + def test_repr_xxh128(self): + h = threadsafe.xxh128() + self.assertIn("xxhash.threadsafe", repr(h)) + self.assertEqual(type(h).__module__, "xxhash.threadsafe") + + +class TestThreadsafeLargeData(unittest.TestCase): + """Verify that large data paths (≥64KB GIL release threshold) work.""" + + SIZE = 64 * 1024 # exactly the GIL release threshold + + def test_xxh32_large(self): + data = b'x' * self.SIZE + self.assertEqual( + threadsafe.xxh32(data).digest(), + xxhash.xxh32(data).digest(), + ) + + def test_xxh64_large(self): + data = b'x' * self.SIZE + self.assertEqual( + threadsafe.xxh64(data).digest(), + xxhash.xxh64(data).digest(), + ) + + def test_xxh3_64_large(self): + data = b'x' * self.SIZE + self.assertEqual( + threadsafe.xxh3_64(data).digest(), + xxhash.xxh3_64(data).digest(), + ) + + def test_xxh128_large(self): + data = b'x' * self.SIZE + self.assertEqual( + threadsafe.xxh128(data).digest(), + xxhash.xxh128(data).digest(), + ) + + def test_streaming_update_large_chunks(self): + """Verify that streaming with large chunks works.""" + h = threadsafe.xxh64() + for _ in range(5): + h.update(b'x' * self.SIZE) + expected = xxhash.xxh64(b'x' * (5 * self.SIZE)).digest() + self.assertEqual(h.digest(), expected) + + +class TestThreadsafeAttributes(unittest.TestCase): + """Verify that threadsafe types have the expected attributes.""" + + def test_name(self): + self.assertEqual(threadsafe.xxh32().name, "XXH32") + self.assertEqual(threadsafe.xxh64().name, "XXH64") + self.assertEqual(threadsafe.xxh3_64().name, "XXH3_64") + self.assertEqual(threadsafe.xxh128().name, "XXH3_128") + + def test_seed_attribute(self): + h = threadsafe.xxh32(b'test', 42) + self.assertEqual(h.seed, 42) + + def test_digest_size(self): + self.assertEqual(threadsafe.xxh32().digest_size, 4) + self.assertEqual(threadsafe.xxh64().digest_size, 8) + self.assertEqual(threadsafe.xxh3_64().digest_size, 8) + self.assertEqual(threadsafe.xxh128().digest_size, 16) + + def test_block_size(self): + self.assertEqual(threadsafe.xxh32().block_size, 16) + self.assertEqual(threadsafe.xxh64().block_size, 32) + self.assertEqual(threadsafe.xxh3_64().block_size, 32) + self.assertEqual(threadsafe.xxh128().block_size, 64) + + +class TestThreadsafeTypesAreDistinct(unittest.TestCase): + """Verify that threadsafe types are distinct from the default types. + + A regression here (e.g. both modules accidentally pointing at the same + type object) would silently break thread safety with no other test + catching it. + """ + + def test_types_are_distinct(self): + for name in ('xxh32', 'xxh64', 'xxh3_64', 'xxh128'): + with self.subTest(name=name): + default_type = getattr(xxhash, name) + safe_type = getattr(threadsafe, name) + self.assertIsNot(default_type, safe_type) + self.assertIsNot(type(default_type()), type(safe_type())) + + def test_instances_are_not_instances_of_each_other(self): + for name in ('xxh32', 'xxh64', 'xxh3_64', 'xxh128'): + with self.subTest(name=name): + default_type = getattr(xxhash, name) + safe_type = getattr(threadsafe, name) + self.assertFalse(isinstance(default_type(), safe_type)) + self.assertFalse(isinstance(safe_type(), default_type)) + + +if __name__ == '__main__': + unittest.main() diff --git a/xxhash/__init__.py b/xxhash/__init__.py index 5fe0087..d1723b5 100644 --- a/xxhash/__init__.py +++ b/xxhash/__init__.py @@ -18,7 +18,7 @@ XXHASH_VERSION, ) -from .version import VERSION, VERSION_TUPLE +from .version import VERSION xxh128 = xxh3_128 @@ -26,13 +26,13 @@ xxh128_intdigest = xxh3_128_intdigest xxh128_digest = xxh3_128_digest -algorithms_available = set([ +algorithms_available = { "xxh32", "xxh64", "xxh3_64", "xxh128", "xxh3_128", -]) +} algorithms_guaranteed = algorithms_available @@ -59,7 +59,6 @@ "xxh128_intdigest", "xxh128_hexdigest", "VERSION", - "VERSION_TUPLE", "XXHASH_VERSION", "algorithms_available", "algorithms_guaranteed", diff --git a/xxhash/__init__.pyi b/xxhash/__init__.pyi index 7490e30..e90bffe 100644 --- a/xxhash/__init__.pyi +++ b/xxhash/__init__.pyi @@ -8,8 +8,6 @@ _DataType = _Buffer VERSION: str XXHASH_VERSION: str -#: Deprecated, will be removed in the next major release -VERSION_TUPLE: tuple[int, ...] algorithms_available: set[str] algorithms_guaranteed: set[str] @@ -36,7 +34,6 @@ __all__: list[str] = [ "xxh128_intdigest", "xxh128_hexdigest", "VERSION", - "VERSION_TUPLE", "XXHASH_VERSION", "algorithms_available", "algorithms_guaranteed", @@ -79,6 +76,10 @@ def xxh32_digest(data: _DataType, seed: int = ...) -> bytes: ... def xxh32_hexdigest(data: _DataType, seed: int = ...) -> str: ... def xxh32_intdigest(data: _DataType, seed: int = ...) -> int: ... +def xxh64_digest(data: _DataType, seed: int = ...) -> bytes: ... +def xxh64_hexdigest(data: _DataType, seed: int = ...) -> str: ... +def xxh64_intdigest(data: _DataType, seed: int = ...) -> int: ... + def xxh3_64_digest(data: _DataType, seed: int = ...) -> bytes: ... def xxh3_64_hexdigest(data: _DataType, seed: int = ...) -> str: ... def xxh3_64_intdigest(data: _DataType, seed: int = ...) -> int: ... @@ -87,10 +88,6 @@ def xxh3_128_digest(data: _DataType, seed: int = ...) -> bytes: ... def xxh3_128_hexdigest(data: _DataType, seed: int = ...) -> str: ... def xxh3_128_intdigest(data: _DataType, seed: int = ...) -> int: ... -xxh64_digest = xxh3_64_digest -xxh64_hexdigest = xxh3_64_hexdigest -xxh64_intdigest = xxh3_64_intdigest - xxh128_digest = xxh3_128_digest xxh128_hexdigest = xxh3_128_hexdigest xxh128_intdigest = xxh3_128_intdigest diff --git a/xxhash/threadsafe.py b/xxhash/threadsafe.py new file mode 100644 index 0000000..0e49785 --- /dev/null +++ b/xxhash/threadsafe.py @@ -0,0 +1,62 @@ +from xxhash.version import VERSION +from xxhash._xxhash_threadsafe import ( + xxh32, + xxh32_digest, + xxh32_intdigest, + xxh32_hexdigest, + xxh64, + xxh64_digest, + xxh64_intdigest, + xxh64_hexdigest, + xxh3_64, + xxh3_64_digest, + xxh3_64_intdigest, + xxh3_64_hexdigest, + xxh3_128, + xxh3_128_digest, + xxh3_128_intdigest, + xxh3_128_hexdigest, + XXHASH_VERSION, +) + +xxh128 = xxh3_128 +xxh128_digest = xxh3_128_digest +xxh128_intdigest = xxh3_128_intdigest +xxh128_hexdigest = xxh3_128_hexdigest + +algorithms_available = { + "xxh32", + "xxh64", + "xxh3_64", + "xxh128", + "xxh3_128", +} + +algorithms_guaranteed = algorithms_available + +__all__ = [ + "xxh32", + "xxh32_digest", + "xxh32_intdigest", + "xxh32_hexdigest", + "xxh64", + "xxh64_digest", + "xxh64_intdigest", + "xxh64_hexdigest", + "xxh3_64", + "xxh3_64_digest", + "xxh3_64_intdigest", + "xxh3_64_hexdigest", + "xxh3_128", + "xxh3_128_digest", + "xxh3_128_intdigest", + "xxh3_128_hexdigest", + "xxh128", + "xxh128_digest", + "xxh128_intdigest", + "xxh128_hexdigest", + "VERSION", + "XXHASH_VERSION", + "algorithms_available", + "algorithms_guaranteed", +] diff --git a/xxhash/threadsafe.pyi b/xxhash/threadsafe.pyi new file mode 100644 index 0000000..e90bffe --- /dev/null +++ b/xxhash/threadsafe.pyi @@ -0,0 +1,93 @@ +from typing import Protocol, final + +class _Buffer(Protocol): + """Objects that support the buffer protocol (PEP 688).""" + def __buffer__(self, flags: int, /) -> memoryview: ... + +_DataType = _Buffer + +VERSION: str +XXHASH_VERSION: str + +algorithms_available: set[str] +algorithms_guaranteed: set[str] + +__all__: list[str] = [ + "xxh32", + "xxh32_digest", + "xxh32_intdigest", + "xxh32_hexdigest", + "xxh64", + "xxh64_digest", + "xxh64_intdigest", + "xxh64_hexdigest", + "xxh3_64", + "xxh3_64_digest", + "xxh3_64_intdigest", + "xxh3_64_hexdigest", + "xxh3_128", + "xxh3_128_digest", + "xxh3_128_intdigest", + "xxh3_128_hexdigest", + "xxh128", + "xxh128_digest", + "xxh128_intdigest", + "xxh128_hexdigest", + "VERSION", + "XXHASH_VERSION", + "algorithms_available", + "algorithms_guaranteed", +] + +class _Hasher: + def __init__(self, data: _DataType = ..., seed: int = ...) -> None: ... + def update(self, data: _DataType) -> None: ... + def digest(self) -> bytes: ... + def hexdigest(self) -> str: ... + def intdigest(self) -> int: ... + def copy(self) -> _Hasher: ... + def reset(self) -> None: ... + @property + def digestsize(self) -> int: ... + @property + def digest_size(self) -> int: ... + @property + def block_size(self) -> int: ... + @property + def name(self) -> str: ... + @property + def seed(self) -> int: ... + +@final +class xxh32(_Hasher): ... + +@final +class xxh64(_Hasher): ... + +@final +class xxh3_64(_Hasher): ... + +@final +class xxh3_128(_Hasher): ... + +xxh128 = xxh3_128 + +def xxh32_digest(data: _DataType, seed: int = ...) -> bytes: ... +def xxh32_hexdigest(data: _DataType, seed: int = ...) -> str: ... +def xxh32_intdigest(data: _DataType, seed: int = ...) -> int: ... + +def xxh64_digest(data: _DataType, seed: int = ...) -> bytes: ... +def xxh64_hexdigest(data: _DataType, seed: int = ...) -> str: ... +def xxh64_intdigest(data: _DataType, seed: int = ...) -> int: ... + +def xxh3_64_digest(data: _DataType, seed: int = ...) -> bytes: ... +def xxh3_64_hexdigest(data: _DataType, seed: int = ...) -> str: ... +def xxh3_64_intdigest(data: _DataType, seed: int = ...) -> int: ... + +def xxh3_128_digest(data: _DataType, seed: int = ...) -> bytes: ... +def xxh3_128_hexdigest(data: _DataType, seed: int = ...) -> str: ... +def xxh3_128_intdigest(data: _DataType, seed: int = ...) -> int: ... + +xxh128_digest = xxh3_128_digest +xxh128_hexdigest = xxh3_128_hexdigest +xxh128_intdigest = xxh3_128_intdigest diff --git a/xxhash/version.py b/xxhash/version.py index 17952b8..69cb0bd 100644 --- a/xxhash/version.py +++ b/xxhash/version.py @@ -1,3 +1 @@ -VERSION = "3.8.0.dev9" -#: Deprecated, will be removed in the next major release -VERSION_TUPLE = (3, 8, 0) +VERSION = "4.0.0.dev1"