Skip to content

Commit 9f26bc3

Browse files
f3schclaude
f3sch
and
claude
committed
GPUCommonMath: double functions, 64-bit bit counts, atomic load/store
Named double functions for GPUCommonMath::Sqrt,Sin,Cos,Tan,ASin,ACos,ATan,ATan2,Log,Copysign. Clz64, Ctz64, Popcount64: 64-bit counterparts of Clz, Ctz and Popcount on the same builtins and CUDA/HIP intrinsics. Named rather than uint64_t overloads, which would change existing calls with 64-bit arguments and make calls with int arguments ambiguous. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Felix Schlepper <felix.schlepper@cern.ch>
1 parent 3f159b8 commit 9f26bc3

3 files changed

Lines changed: 133 additions & 17 deletions

File tree

‎Common/MathUtils/include/MathUtils/detail/basicMath.h‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -91,17 +91,25 @@ GPUdi() T log(T x)
9191
return o2::gpu::GPUCommonMath::Log(x);
9292
};
9393

94-
#ifndef GPUCA_GPUCODE_DEVICE // Double overloads using std namespace not visible on GPU
94+
#if !defined(GPUCA_GPUCODE_DEVICE) || defined(__CUDACC__) || defined(__HIPCC__) // Double specialisations, not for OpenCL and Metal
9595
template <>
9696
GPUhdi() double copysign(double x, double y)
9797
{
98-
return std::copysign(x, y);
98+
return o2::gpu::GPUCommonMath::Copysignd(x, y);
9999
}
100100
template <>
101101
GPUhdi() double sqrt(double x)
102102
{
103-
return std::sqrt(x);
103+
return o2::gpu::GPUCommonMath::Sqrtd(x);
104+
};
105+
template <>
106+
GPUdi() double log(double x)
107+
{
108+
return o2::gpu::GPUCommonMath::Logd(x);
104109
};
110+
#endif
111+
112+
#ifndef GPUCA_GPUCODE_DEVICE // Double overloads using std namespace not visible on GPU
105113
GPUhdi() double abs(double x)
106114
{
107115
return std::abs(x);
@@ -120,11 +128,6 @@ GPUdi() bool finite(double x)
120128
return std::isfinite(x);
121129
#endif
122130
}
123-
template <>
124-
GPUdi() double log(double x)
125-
{
126-
return std::log(x);
127-
};
128131
#endif
129132

130133
} // namespace detail

‎Common/MathUtils/include/MathUtils/detail/trigonometric.h‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,7 @@ GPUhdi() constexpr T fastATan2(T y, T x)
299299
if (xx < 0) { // p1 is in the range [Pi/4, 3*Pi/4]
300300
phi0 = Pi075;
301301
tan = y1 > T(0) ? -x1 / y1 : T(0); // yy is always >=0, hence y1>=0
302-
} else { // p1 is in the range [-Pi/4, Pi/4]
302+
} else { // p1 is in the range [-Pi/4, Pi/4]
303303
phi0 = Pi025;
304304
tan = x1 > T(0) ? y1 / x1 : T(0);
305305
}
@@ -376,41 +376,41 @@ GPUdi() double pi()
376376
return o2::constants::math::PI;
377377
}
378378

379-
#ifndef GPUCA_GPUCODE_DEVICE
379+
#if !defined(GPUCA_GPUCODE_DEVICE) || defined(__CUDACC__) || defined(__HIPCC__) // Double specialisations, not for OpenCL and Metal
380380
template <>
381381
GPUdi() double tan(double x)
382382
{
383-
return std::tan(x);
383+
return o2::gpu::GPUCommonMath::Tand(x);
384384
};
385385
template <>
386386
GPUdi() double sin(double x)
387387
{
388-
return std::sin(x);
388+
return o2::gpu::GPUCommonMath::Sind(x);
389389
};
390390
template <>
391391
GPUdi() double atan2(double y, double x)
392392
{
393-
return std::atan2(y, x);
393+
return o2::gpu::GPUCommonMath::ATan2d(y, x);
394394
};
395395
template <>
396396
GPUdi() double atan(double x)
397397
{
398-
return std::atan(x);
398+
return o2::gpu::GPUCommonMath::ATand(x);
399399
};
400400
template <>
401401
GPUdi() double asin(double x)
402402
{
403-
return std::asin(x);
403+
return o2::gpu::GPUCommonMath::ASind(x);
404404
};
405405
template <>
406406
GPUdi() double acos(double x)
407407
{
408-
return std::acos(x);
408+
return o2::gpu::GPUCommonMath::ACosd(x);
409409
};
410410
template <>
411411
GPUdi() double cos(double x)
412412
{
413-
return std::cos(x);
413+
return o2::gpu::GPUCommonMath::Cosd(x);
414414
};
415415
#endif
416416

‎GPU/Common/GPUCommonMath.h‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ class GPUCommonMath
7272
return Max(lo, Min(v, hi));
7373
}
7474
GPUhdni() constexpr static float Sqrt(float x);
75+
GPUd() constexpr static double Sqrtd(double x);
7576
GPUd() static float InvSqrt(float x);
7677
template <class T>
7778
GPUdi() constexpr static T Square(T x)
@@ -81,18 +82,27 @@ class GPUCommonMath
8182
template <class T>
8283
GPUhd() constexpr static T Abs(T x);
8384
GPUd() constexpr static float ASin(float x);
85+
GPUd() constexpr static double ASind(double x);
8486
GPUd() constexpr static float ACos(float x);
87+
GPUd() constexpr static double ACosd(double x);
8588
GPUd() constexpr static float ATan(float x);
89+
GPUd() constexpr static double ATand(double x);
8690
GPUhd() constexpr static float ATan2(float y, float x);
91+
GPUd() constexpr static double ATan2d(double y, double x);
8792
GPUd() constexpr static float Sin(float x);
93+
GPUd() constexpr static double Sind(double x);
8894
GPUd() constexpr static float Cos(float x);
95+
GPUd() constexpr static double Cosd(double x);
8996
GPUhdni() static void SinCos(float x, float& s, float& c);
9097
GPUhdni() static void SinCosd(double x, double& s, double& c);
9198
GPUd() constexpr static float Tan(float x);
99+
GPUd() constexpr static double Tand(double x);
92100
GPUd() constexpr static float Pow(float x, float y);
93101
GPUd() constexpr static float Log(float x);
102+
GPUd() constexpr static double Logd(double x);
94103
GPUd() constexpr static float Exp(float x);
95104
GPUhdni() constexpr static float Copysign(float x, float y) { return GPUCA_CHOICE(std::copysignf(x, y), copysignf(x, y), copysign(x, y)); }
105+
GPUd() constexpr static double Copysignd(double x, double y);
96106
GPUd() constexpr static float TwoPi() { return 6.2831853f; }
97107
GPUd() constexpr static float Pi() { return 3.1415927f; }
98108
GPUd() constexpr static float Round(float x);
@@ -110,6 +120,9 @@ class GPUCommonMath
110120
GPUd() constexpr static uint32_t Clz(uint32_t val);
111121
GPUd() constexpr static uint32_t Ctz(uint32_t val);
112122
GPUd() constexpr static uint32_t Popcount(uint32_t val);
123+
GPUd() constexpr static uint32_t Clz64(uint64_t val);
124+
GPUd() constexpr static uint32_t Ctz64(uint64_t val);
125+
GPUd() constexpr static uint32_t Popcount64(uint64_t val);
113126

114127
GPUd() static void memcpy(void* dst, const void* src, size_t size);
115128

@@ -148,6 +161,16 @@ class GPUCommonMath
148161
GPUCommonMath::AtomicMinInternal(addr, val);
149162
}
150163
template <class T>
164+
GPUdi() static T AtomicLoad(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr)
165+
{
166+
return GPUCommonMath::AtomicLoadInternal<T>(addr);
167+
}
168+
template <class T>
169+
GPUdi() static void AtomicStore(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr, T val)
170+
{
171+
GPUCommonMath::AtomicStoreInternal(addr, val);
172+
}
173+
template <class T>
151174
GPUdi() static T AtomicExchShared(GPUsharedref() GPUgeneric() GPUAtomic(T) * addr, T val)
152175
{
153176
return GPUCommonMath::AtomicExchInternal(addr, val);
@@ -187,6 +210,10 @@ class GPUCommonMath
187210
GPUd() static void AtomicMaxInternal(S* addr, T val);
188211
template <class S, class T>
189212
GPUd() static void AtomicMinInternal(S* addr, T val);
213+
template <class T, class S>
214+
GPUd() static T AtomicLoadInternal(S* addr);
215+
template <class S, class T>
216+
GPUd() static void AtomicStoreInternal(S* addr, T val);
190217
};
191218

192219
typedef GPUCommonMath CAMath;
@@ -267,6 +294,8 @@ GPUdi() uint32_t GPUCommonMath::Float2UIntReint(const float& x)
267294
return __float_as_uint(x);
268295
#elif defined(GPUCA_GPUCODE_DEVICE) && defined(__OPENCL__)
269296
return as_uint(x);
297+
#elif !defined(GPUCA_GPUCODE_DEVICE)
298+
return __builtin_bit_cast(uint32_t, x);
270299
#else
271300
return reinterpret_cast<const uint32_t&>(x);
272301
#endif
@@ -338,6 +367,17 @@ GPUhdi() void GPUCommonMath::SinCosd(double x, double& s, double& c)
338367
GPUCA_CHOICE((void)((s = sin(x)) + (c = cos(x))), sincos(x, &s, &c), s = sincos(x, &c));
339368
#endif
340369
}
370+
371+
GPUdi() constexpr double GPUCommonMath::Sqrtd(double x) { return GPUCA_CHOICE(std::sqrt(x), sqrt(x), sqrt(x)); }
372+
GPUdi() constexpr double GPUCommonMath::ASind(double x) { return GPUCA_CHOICE(std::asin(x), asin(x), asin(x)); }
373+
GPUdi() constexpr double GPUCommonMath::ACosd(double x) { return GPUCA_CHOICE(std::acos(x), acos(x), acos(x)); }
374+
GPUdi() constexpr double GPUCommonMath::ATand(double x) { return GPUCA_CHOICE(std::atan(x), atan(x), atan(x)); }
375+
GPUdi() constexpr double GPUCommonMath::ATan2d(double y, double x) { return GPUCA_CHOICE(std::atan2(y, x), atan2(y, x), atan2(y, x)); }
376+
GPUdi() constexpr double GPUCommonMath::Sind(double x) { return GPUCA_CHOICE(std::sin(x), sin(x), sin(x)); }
377+
GPUdi() constexpr double GPUCommonMath::Cosd(double x) { return GPUCA_CHOICE(std::cos(x), cos(x), cos(x)); }
378+
GPUdi() constexpr double GPUCommonMath::Tand(double x) { return GPUCA_CHOICE(std::tan(x), tan(x), tan(x)); }
379+
GPUdi() constexpr double GPUCommonMath::Logd(double x) { return GPUCA_CHOICE(std::log(x), log(x), log(x)); }
380+
GPUdi() constexpr double GPUCommonMath::Copysignd(double x, double y) { return GPUCA_CHOICE(std::copysign(x, y), copysign(x, y), copysign(x, y)); }
341381
#endif
342382

343383
GPUdi() constexpr uint32_t GPUCommonMath::Clz(uint32_t x)
@@ -380,6 +420,44 @@ GPUdi() constexpr uint32_t GPUCommonMath::Popcount(uint32_t x)
380420
#endif
381421
}
382422

423+
GPUdi() constexpr uint32_t GPUCommonMath::Clz64(uint64_t x)
424+
{
425+
#if (defined(__GNUC__) || defined(__clang__) || defined(__CUDACC__) || defined(__HIPCC__))
426+
return x == 0 ? 64 : GPUCA_CHOICE(__builtin_clzll(x), __clzll(x), __builtin_clzll(x)); // use builtin if available
427+
#else
428+
for (int32_t i = 63; i >= 0; i--) {
429+
if (x & (1ull << i)) {
430+
return (63 - i);
431+
}
432+
}
433+
return 64;
434+
#endif
435+
}
436+
437+
GPUdi() constexpr uint32_t GPUCommonMath::Ctz64(uint64_t x)
438+
{
439+
#if (defined(__GNUC__) || defined(__clang__) || defined(__CUDACC__) || defined(__HIPCC__))
440+
return x == 0 ? 64 : GPUCA_CHOICE(__builtin_ctzll(x), __ffsll((long long)x) - 1, __builtin_ctzll(x));
441+
#else
442+
for (uint32_t i = 0; i < 64; ++i) {
443+
if (x & (1ull << i)) {
444+
return i;
445+
}
446+
}
447+
return 64;
448+
#endif
449+
}
450+
451+
GPUdi() constexpr uint32_t GPUCommonMath::Popcount64(uint64_t x)
452+
{
453+
#if (defined(__GNUC__) || defined(__clang__) || defined(__CUDACC__) || defined(__HIPCC__)) && !defined(__OPENCL__) // TODO: remove OPENCL when reported SPIR-V bug is fixed
454+
// use builtin if available
455+
return GPUCA_CHOICE(__builtin_popcountll(x), __popcll(x), __builtin_popcountll(x));
456+
#else
457+
return Popcount((uint32_t)x) + Popcount((uint32_t)(x >> 32));
458+
#endif
459+
}
460+
383461
template <typename T>
384462
GPUhdi() constexpr void GPUCommonMath::Swap(T& a, T& b)
385463
{
@@ -600,6 +678,41 @@ GPUdi() void GPUCommonMath::AtomicMinInternal(S* addr, T val)
600678
#endif // GPUCA_GPUCODE
601679
}
602680

681+
// Relaxed, as the device atomics above
682+
template <class T, class S>
683+
GPUdi() T GPUCommonMath::AtomicLoadInternal(S* addr)
684+
{
685+
#if defined(GPUCA_GPUCODE) && defined(__OPENCL__) && (!defined(__clang__) || defined(GPUCA_OPENCL_CLANG_C11_ATOMICS))
686+
return ::atomic_load_explicit(addr, memory_order_relaxed);
687+
#elif defined(GPUCA_GPUCODE) && (defined(__OPENCL__) || defined(__CUDACC__) || defined(__HIPCC__))
688+
return *(volatile S*)addr;
689+
#elif defined(GPUCA_GPUCODE) && defined(__METAL__)
690+
return atomic_load_explicit(GPUCommonMathMetalAtomic(addr), memory_order_relaxed);
691+
#elif defined(WITH_OPENMP)
692+
T ret;
693+
__atomic_load(addr, &ret, __ATOMIC_RELAXED);
694+
return ret;
695+
#else
696+
return reinterpret_cast<std::atomic<T>*>(addr)->load(std::memory_order_relaxed);
697+
#endif
698+
}
699+
700+
template <class S, class T>
701+
GPUdi() void GPUCommonMath::AtomicStoreInternal(S* addr, T val)
702+
{
703+
#if defined(GPUCA_GPUCODE) && defined(__OPENCL__) && (!defined(__clang__) || defined(GPUCA_OPENCL_CLANG_C11_ATOMICS))
704+
::atomic_store_explicit(addr, val, memory_order_relaxed);
705+
#elif defined(GPUCA_GPUCODE) && (defined(__OPENCL__) || defined(__CUDACC__) || defined(__HIPCC__))
706+
*(volatile S*)addr = val;
707+
#elif defined(GPUCA_GPUCODE) && defined(__METAL__)
708+
atomic_store_explicit(GPUCommonMathMetalAtomic(addr), val, memory_order_relaxed);
709+
#elif defined(WITH_OPENMP)
710+
__atomic_store(addr, &val, __ATOMIC_RELAXED);
711+
#else
712+
reinterpret_cast<std::atomic<T>*>(addr)->store(val, std::memory_order_relaxed);
713+
#endif
714+
}
715+
603716
#if (defined(__CUDACC__) || defined(__HIPCC__)) && !defined(G__ROOT) && !defined(__CLING__)
604717
#define GPUCA_HAVE_ATOMIC_MINMAX_FLOAT
605718
template <>

0 commit comments

Comments
 (0)