From 9f26bc3d64f2f8b721cbfd48e85fc90a75012b33 Mon Sep 17 00:00:00 2001 From: Felix Schlepper Date: Mon, 19 Oct 2026 11:38:23 +0200 Subject: [PATCH] GPUCommonMath: double functions, 64-bit bit counts, atomic load/store Named double functions for GPUCommonMath::Sqrt,Sin,Cos,Tan,ASin,ACos,ATan,ATan2,Log,Copysign. Clz64, Ctz64, Popcount64: 64-bit counterparts of Clz, Ctz and Popcount on the same builtins and CUDA/HIP intrinsics. Named rather than uint64_t overloads, which would change existing calls with 64-bit arguments and make calls with int arguments ambiguous. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Felix Schlepper --- .../include/MathUtils/detail/basicMath.h | 19 +-- .../include/MathUtils/detail/trigonometric.h | 18 +-- GPU/Common/GPUCommonMath.h | 113 ++++++++++++++++++ 3 files changed, 133 insertions(+), 17 deletions(-) diff --git a/Common/MathUtils/include/MathUtils/detail/basicMath.h b/Common/MathUtils/include/MathUtils/detail/basicMath.h index 059dd4cca66fb..cd9ea213dd510 100644 --- a/Common/MathUtils/include/MathUtils/detail/basicMath.h +++ b/Common/MathUtils/include/MathUtils/detail/basicMath.h @@ -91,17 +91,25 @@ GPUdi() T log(T x) return o2::gpu::GPUCommonMath::Log(x); }; -#ifndef GPUCA_GPUCODE_DEVICE // Double overloads using std namespace not visible on GPU +#if !defined(GPUCA_GPUCODE_DEVICE) || defined(__CUDACC__) || defined(__HIPCC__) // Double specialisations, not for OpenCL and Metal template <> GPUhdi() double copysign(double x, double y) { - return std::copysign(x, y); + return o2::gpu::GPUCommonMath::Copysignd(x, y); } template <> GPUhdi() double sqrt(double x) { - return std::sqrt(x); + return o2::gpu::GPUCommonMath::Sqrtd(x); +}; +template <> +GPUdi() double log(double x) +{ + return o2::gpu::GPUCommonMath::Logd(x); }; +#endif + +#ifndef GPUCA_GPUCODE_DEVICE // Double overloads using std namespace not visible on GPU GPUhdi() double abs(double x) { return std::abs(x); @@ -120,11 +128,6 @@ GPUdi() bool finite(double x) return std::isfinite(x); #endif } -template <> -GPUdi() double log(double x) -{ - return std::log(x); -}; #endif } // namespace detail diff --git a/Common/MathUtils/include/MathUtils/detail/trigonometric.h b/Common/MathUtils/include/MathUtils/detail/trigonometric.h index afdfbb57b0f28..44fa8b713cb3b 100644 --- a/Common/MathUtils/include/MathUtils/detail/trigonometric.h +++ b/Common/MathUtils/include/MathUtils/detail/trigonometric.h @@ -299,7 +299,7 @@ GPUhdi() constexpr T fastATan2(T y, T x) if (xx < 0) { // p1 is in the range [Pi/4, 3*Pi/4] phi0 = Pi075; tan = y1 > T(0) ? -x1 / y1 : T(0); // yy is always >=0, hence y1>=0 - } else { // p1 is in the range [-Pi/4, Pi/4] + } else { // p1 is in the range [-Pi/4, Pi/4] phi0 = Pi025; tan = x1 > T(0) ? y1 / x1 : T(0); } @@ -376,41 +376,41 @@ GPUdi() double pi() return o2::constants::math::PI; } -#ifndef GPUCA_GPUCODE_DEVICE +#if !defined(GPUCA_GPUCODE_DEVICE) || defined(__CUDACC__) || defined(__HIPCC__) // Double specialisations, not for OpenCL and Metal template <> GPUdi() double tan(double x) { - return std::tan(x); + return o2::gpu::GPUCommonMath::Tand(x); }; template <> GPUdi() double sin(double x) { - return std::sin(x); + return o2::gpu::GPUCommonMath::Sind(x); }; template <> GPUdi() double atan2(double y, double x) { - return std::atan2(y, x); + return o2::gpu::GPUCommonMath::ATan2d(y, x); }; template <> GPUdi() double atan(double x) { - return std::atan(x); + return o2::gpu::GPUCommonMath::ATand(x); }; template <> GPUdi() double asin(double x) { - return std::asin(x); + return o2::gpu::GPUCommonMath::ASind(x); }; template <> GPUdi() double acos(double x) { - return std::acos(x); + return o2::gpu::GPUCommonMath::ACosd(x); }; template <> GPUdi() double cos(double x) { - return std::cos(x); + return o2::gpu::GPUCommonMath::Cosd(x); }; #endif diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index f2daf30842663..54b6fcfabcb9e 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -72,6 +72,7 @@ class GPUCommonMath return Max(lo, Min(v, hi)); } GPUhdni() constexpr static float Sqrt(float x); + GPUd() constexpr static double Sqrtd(double x); GPUd() static float InvSqrt(float x); template GPUdi() constexpr static T Square(T x) @@ -81,18 +82,27 @@ class GPUCommonMath template GPUhd() constexpr static T Abs(T x); GPUd() constexpr static float ASin(float x); + GPUd() constexpr static double ASind(double x); GPUd() constexpr static float ACos(float x); + GPUd() constexpr static double ACosd(double x); GPUd() constexpr static float ATan(float x); + GPUd() constexpr static double ATand(double x); GPUhd() constexpr static float ATan2(float y, float x); + GPUd() constexpr static double ATan2d(double y, double x); GPUd() constexpr static float Sin(float x); + GPUd() constexpr static double Sind(double x); GPUd() constexpr static float Cos(float x); + GPUd() constexpr static double Cosd(double x); GPUhdni() static void SinCos(float x, float& s, float& c); GPUhdni() static void SinCosd(double x, double& s, double& c); GPUd() constexpr static float Tan(float x); + GPUd() constexpr static double Tand(double x); GPUd() constexpr static float Pow(float x, float y); GPUd() constexpr static float Log(float x); + GPUd() constexpr static double Logd(double x); GPUd() constexpr static float Exp(float x); GPUhdni() constexpr static float Copysign(float x, float y) { return GPUCA_CHOICE(std::copysignf(x, y), copysignf(x, y), copysign(x, y)); } + GPUd() constexpr static double Copysignd(double x, double y); GPUd() constexpr static float TwoPi() { return 6.2831853f; } GPUd() constexpr static float Pi() { return 3.1415927f; } GPUd() constexpr static float Round(float x); @@ -110,6 +120,9 @@ class GPUCommonMath GPUd() constexpr static uint32_t Clz(uint32_t val); GPUd() constexpr static uint32_t Ctz(uint32_t val); GPUd() constexpr static uint32_t Popcount(uint32_t val); + GPUd() constexpr static uint32_t Clz64(uint64_t val); + GPUd() constexpr static uint32_t Ctz64(uint64_t val); + GPUd() constexpr static uint32_t Popcount64(uint64_t val); GPUd() static void memcpy(void* dst, const void* src, size_t size); @@ -148,6 +161,16 @@ class GPUCommonMath GPUCommonMath::AtomicMinInternal(addr, val); } template + GPUdi() static T AtomicLoad(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr) + { + return GPUCommonMath::AtomicLoadInternal(addr); + } + template + GPUdi() static void AtomicStore(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr, T val) + { + GPUCommonMath::AtomicStoreInternal(addr, val); + } + template GPUdi() static T AtomicExchShared(GPUsharedref() GPUgeneric() GPUAtomic(T) * addr, T val) { return GPUCommonMath::AtomicExchInternal(addr, val); @@ -187,6 +210,10 @@ class GPUCommonMath GPUd() static void AtomicMaxInternal(S* addr, T val); template GPUd() static void AtomicMinInternal(S* addr, T val); + template + GPUd() static T AtomicLoadInternal(S* addr); + template + GPUd() static void AtomicStoreInternal(S* addr, T val); }; typedef GPUCommonMath CAMath; @@ -267,6 +294,8 @@ GPUdi() uint32_t GPUCommonMath::Float2UIntReint(const float& x) return __float_as_uint(x); #elif defined(GPUCA_GPUCODE_DEVICE) && defined(__OPENCL__) return as_uint(x); +#elif !defined(GPUCA_GPUCODE_DEVICE) + return __builtin_bit_cast(uint32_t, x); #else return reinterpret_cast(x); #endif @@ -338,6 +367,17 @@ GPUhdi() void GPUCommonMath::SinCosd(double x, double& s, double& c) GPUCA_CHOICE((void)((s = sin(x)) + (c = cos(x))), sincos(x, &s, &c), s = sincos(x, &c)); #endif } + +GPUdi() constexpr double GPUCommonMath::Sqrtd(double x) { return GPUCA_CHOICE(std::sqrt(x), sqrt(x), sqrt(x)); } +GPUdi() constexpr double GPUCommonMath::ASind(double x) { return GPUCA_CHOICE(std::asin(x), asin(x), asin(x)); } +GPUdi() constexpr double GPUCommonMath::ACosd(double x) { return GPUCA_CHOICE(std::acos(x), acos(x), acos(x)); } +GPUdi() constexpr double GPUCommonMath::ATand(double x) { return GPUCA_CHOICE(std::atan(x), atan(x), atan(x)); } +GPUdi() constexpr double GPUCommonMath::ATan2d(double y, double x) { return GPUCA_CHOICE(std::atan2(y, x), atan2(y, x), atan2(y, x)); } +GPUdi() constexpr double GPUCommonMath::Sind(double x) { return GPUCA_CHOICE(std::sin(x), sin(x), sin(x)); } +GPUdi() constexpr double GPUCommonMath::Cosd(double x) { return GPUCA_CHOICE(std::cos(x), cos(x), cos(x)); } +GPUdi() constexpr double GPUCommonMath::Tand(double x) { return GPUCA_CHOICE(std::tan(x), tan(x), tan(x)); } +GPUdi() constexpr double GPUCommonMath::Logd(double x) { return GPUCA_CHOICE(std::log(x), log(x), log(x)); } +GPUdi() constexpr double GPUCommonMath::Copysignd(double x, double y) { return GPUCA_CHOICE(std::copysign(x, y), copysign(x, y), copysign(x, y)); } #endif GPUdi() constexpr uint32_t GPUCommonMath::Clz(uint32_t x) @@ -380,6 +420,44 @@ GPUdi() constexpr uint32_t GPUCommonMath::Popcount(uint32_t x) #endif } +GPUdi() constexpr uint32_t GPUCommonMath::Clz64(uint64_t x) +{ +#if (defined(__GNUC__) || defined(__clang__) || defined(__CUDACC__) || defined(__HIPCC__)) + return x == 0 ? 64 : GPUCA_CHOICE(__builtin_clzll(x), __clzll(x), __builtin_clzll(x)); // use builtin if available +#else + for (int32_t i = 63; i >= 0; i--) { + if (x & (1ull << i)) { + return (63 - i); + } + } + return 64; +#endif +} + +GPUdi() constexpr uint32_t GPUCommonMath::Ctz64(uint64_t x) +{ +#if (defined(__GNUC__) || defined(__clang__) || defined(__CUDACC__) || defined(__HIPCC__)) + return x == 0 ? 64 : GPUCA_CHOICE(__builtin_ctzll(x), __ffsll((long long)x) - 1, __builtin_ctzll(x)); +#else + for (uint32_t i = 0; i < 64; ++i) { + if (x & (1ull << i)) { + return i; + } + } + return 64; +#endif +} + +GPUdi() constexpr uint32_t GPUCommonMath::Popcount64(uint64_t x) +{ +#if (defined(__GNUC__) || defined(__clang__) || defined(__CUDACC__) || defined(__HIPCC__)) && !defined(__OPENCL__) // TODO: remove OPENCL when reported SPIR-V bug is fixed + // use builtin if available + return GPUCA_CHOICE(__builtin_popcountll(x), __popcll(x), __builtin_popcountll(x)); +#else + return Popcount((uint32_t)x) + Popcount((uint32_t)(x >> 32)); +#endif +} + template GPUhdi() constexpr void GPUCommonMath::Swap(T& a, T& b) { @@ -600,6 +678,41 @@ GPUdi() void GPUCommonMath::AtomicMinInternal(S* addr, T val) #endif // GPUCA_GPUCODE } +// Relaxed, as the device atomics above +template +GPUdi() T GPUCommonMath::AtomicLoadInternal(S* addr) +{ +#if defined(GPUCA_GPUCODE) && defined(__OPENCL__) && (!defined(__clang__) || defined(GPUCA_OPENCL_CLANG_C11_ATOMICS)) + return ::atomic_load_explicit(addr, memory_order_relaxed); +#elif defined(GPUCA_GPUCODE) && (defined(__OPENCL__) || defined(__CUDACC__) || defined(__HIPCC__)) + return *(volatile S*)addr; +#elif defined(GPUCA_GPUCODE) && defined(__METAL__) + return atomic_load_explicit(GPUCommonMathMetalAtomic(addr), memory_order_relaxed); +#elif defined(WITH_OPENMP) + T ret; + __atomic_load(addr, &ret, __ATOMIC_RELAXED); + return ret; +#else + return reinterpret_cast*>(addr)->load(std::memory_order_relaxed); +#endif +} + +template +GPUdi() void GPUCommonMath::AtomicStoreInternal(S* addr, T val) +{ +#if defined(GPUCA_GPUCODE) && defined(__OPENCL__) && (!defined(__clang__) || defined(GPUCA_OPENCL_CLANG_C11_ATOMICS)) + ::atomic_store_explicit(addr, val, memory_order_relaxed); +#elif defined(GPUCA_GPUCODE) && (defined(__OPENCL__) || defined(__CUDACC__) || defined(__HIPCC__)) + *(volatile S*)addr = val; +#elif defined(GPUCA_GPUCODE) && defined(__METAL__) + atomic_store_explicit(GPUCommonMathMetalAtomic(addr), val, memory_order_relaxed); +#elif defined(WITH_OPENMP) + __atomic_store(addr, &val, __ATOMIC_RELAXED); +#else + reinterpret_cast*>(addr)->store(val, std::memory_order_relaxed); +#endif +} + #if (defined(__CUDACC__) || defined(__HIPCC__)) && !defined(G__ROOT) && !defined(__CLING__) #define GPUCA_HAVE_ATOMIC_MINMAX_FLOAT template <>