Skip to content

GPUCommonMath: double functions, 64-bit bit counts, atomic load/store… #15986

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
f3sch wants to merge 1 commit into
base: dev
Choose a base branch
from
+133 −17
Open

GPUCommonMath: double functions, 64-bit bit counts, atomic load/store… #15986

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions Common/MathUtils/include/MathUtils/detail/basicMath.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,17 +91,25 @@ GPUdi() T log(T x)
return o2::gpu::GPUCommonMath::Log(x);
};

#ifndef GPUCA_GPUCODE_DEVICE // Double overloads using std namespace not visible on GPU
#if !defined(GPUCA_GPUCODE_DEVICE) || defined(__CUDACC__) || defined(__HIPCC__) // Double specialisations, not for OpenCL and Metal
template <>
GPUhdi() double copysign(double x, double y)
{
return std::copysign(x, y);
return o2::gpu::GPUCommonMath::Copysignd(x, y);
}
template <>
GPUhdi() double sqrt(double x)
{
return std::sqrt(x);
return o2::gpu::GPUCommonMath::Sqrtd(x);
};
template <>
GPUdi() double log(double x)
{
return o2::gpu::GPUCommonMath::Logd(x);
};
#endif

#ifndef GPUCA_GPUCODE_DEVICE // Double overloads using std namespace not visible on GPU
GPUhdi() double abs(double x)
{
return std::abs(x);
Expand All @@ -120,11 +128,6 @@ GPUdi() bool finite(double x)
return std::isfinite(x);
#endif
}
template <>
GPUdi() double log(double x)
{
return std::log(x);
};
#endif

} // namespace detail
Expand Down
18 changes: 9 additions & 9 deletions Common/MathUtils/include/MathUtils/detail/trigonometric.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ GPUhdi() constexpr T fastATan2(T y, T x)
if (xx < 0) { // p1 is in the range [Pi/4, 3*Pi/4]
phi0 = Pi075;
tan = y1 > T(0) ? -x1 / y1 : T(0); // yy is always >=0, hence y1>=0
} else { // p1 is in the range [-Pi/4, Pi/4]
} else { // p1 is in the range [-Pi/4, Pi/4]
phi0 = Pi025;
tan = x1 > T(0) ? y1 / x1 : T(0);
}
Expand Down Expand Up @@ -376,41 +376,41 @@ GPUdi() double pi()
return o2::constants::math::PI;
}

#ifndef GPUCA_GPUCODE_DEVICE
#if !defined(GPUCA_GPUCODE_DEVICE) || defined(__CUDACC__) || defined(__HIPCC__) // Double specialisations, not for OpenCL and Metal
template <>
GPUdi() double tan(double x)
{
return std::tan(x);
return o2::gpu::GPUCommonMath::Tand(x);
};
template <>
GPUdi() double sin(double x)
{
return std::sin(x);
return o2::gpu::GPUCommonMath::Sind(x);
};
template <>
GPUdi() double atan2(double y, double x)
{
return std::atan2(y, x);
return o2::gpu::GPUCommonMath::ATan2d(y, x);
};
template <>
GPUdi() double atan(double x)
{
return std::atan(x);
return o2::gpu::GPUCommonMath::ATand(x);
};
template <>
GPUdi() double asin(double x)
{
return std::asin(x);
return o2::gpu::GPUCommonMath::ASind(x);
};
template <>
GPUdi() double acos(double x)
{
return std::acos(x);
return o2::gpu::GPUCommonMath::ACosd(x);
};
template <>
GPUdi() double cos(double x)
{
return std::cos(x);
return o2::gpu::GPUCommonMath::Cosd(x);
};
#endif

Expand Down
113 changes: 113 additions & 0 deletions GPU/Common/GPUCommonMath.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ class GPUCommonMath
return Max(lo, Min(v, hi));
}
GPUhdni() constexpr static float Sqrt(float x);
GPUd() constexpr static double Sqrtd(double x);
GPUd() static float InvSqrt(float x);
template <class T>
GPUdi() constexpr static T Square(T x)
Expand All @@ -81,18 +82,27 @@ class GPUCommonMath
template <class T>
GPUhd() constexpr static T Abs(T x);
GPUd() constexpr static float ASin(float x);
GPUd() constexpr static double ASind(double x);
GPUd() constexpr static float ACos(float x);
GPUd() constexpr static double ACosd(double x);
GPUd() constexpr static float ATan(float x);
GPUd() constexpr static double ATand(double x);
GPUhd() constexpr static float ATan2(float y, float x);
GPUd() constexpr static double ATan2d(double y, double x);
GPUd() constexpr static float Sin(float x);
GPUd() constexpr static double Sind(double x);
GPUd() constexpr static float Cos(float x);
GPUd() constexpr static double Cosd(double x);
GPUhdni() static void SinCos(float x, float& s, float& c);
GPUhdni() static void SinCosd(double x, double& s, double& c);
GPUd() constexpr static float Tan(float x);
GPUd() constexpr static double Tand(double x);
GPUd() constexpr static float Pow(float x, float y);
GPUd() constexpr static float Log(float x);
GPUd() constexpr static double Logd(double x);
GPUd() constexpr static float Exp(float x);
GPUhdni() constexpr static float Copysign(float x, float y) { return GPUCA_CHOICE(std::copysignf(x, y), copysignf(x, y), copysign(x, y)); }
GPUd() constexpr static double Copysignd(double x, double y);
GPUd() constexpr static float TwoPi() { return 6.2831853f; }
GPUd() constexpr static float Pi() { return 3.1415927f; }
GPUd() constexpr static float Round(float x);
Expand All @@ -110,6 +120,9 @@ class GPUCommonMath
GPUd() constexpr static uint32_t Clz(uint32_t val);
GPUd() constexpr static uint32_t Ctz(uint32_t val);
GPUd() constexpr static uint32_t Popcount(uint32_t val);
GPUd() constexpr static uint32_t Clz64(uint64_t val);
GPUd() constexpr static uint32_t Ctz64(uint64_t val);
GPUd() constexpr static uint32_t Popcount64(uint64_t val);

GPUd() static void memcpy(void* dst, const void* src, size_t size);

Expand Down Expand Up @@ -148,6 +161,16 @@ class GPUCommonMath
GPUCommonMath::AtomicMinInternal(addr, val);
}
template <class T>
GPUdi() static T AtomicLoad(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr)
{
return GPUCommonMath::AtomicLoadInternal<T>(addr);
}
template <class T>
GPUdi() static void AtomicStore(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr, T val)
{
GPUCommonMath::AtomicStoreInternal(addr, val);
}
template <class T>
GPUdi() static T AtomicExchShared(GPUsharedref() GPUgeneric() GPUAtomic(T) * addr, T val)
{
return GPUCommonMath::AtomicExchInternal(addr, val);
Expand Down Expand Up @@ -187,6 +210,10 @@ class GPUCommonMath
GPUd() static void AtomicMaxInternal(S* addr, T val);
template <class S, class T>
GPUd() static void AtomicMinInternal(S* addr, T val);
template <class T, class S>
GPUd() static T AtomicLoadInternal(S* addr);
template <class S, class T>
GPUd() static void AtomicStoreInternal(S* addr, T val);
};

typedef GPUCommonMath CAMath;
Expand Down Expand Up @@ -267,6 +294,8 @@ GPUdi() uint32_t GPUCommonMath::Float2UIntReint(const float& x)
return __float_as_uint(x);
#elif defined(GPUCA_GPUCODE_DEVICE) && defined(__OPENCL__)
return as_uint(x);
#elif !defined(GPUCA_GPUCODE_DEVICE)
return __builtin_bit_cast(uint32_t, x);
#else
return reinterpret_cast<const uint32_t&>(x);
#endif
Expand Down Expand Up @@ -338,6 +367,17 @@ GPUhdi() void GPUCommonMath::SinCosd(double x, double& s, double& c)
GPUCA_CHOICE((void)((s = sin(x)) + (c = cos(x))), sincos(x, &s, &c), s = sincos(x, &c));
#endif
}

GPUdi() constexpr double GPUCommonMath::Sqrtd(double x) { return GPUCA_CHOICE(std::sqrt(x), sqrt(x), sqrt(x)); }
GPUdi() constexpr double GPUCommonMath::ASind(double x) { return GPUCA_CHOICE(std::asin(x), asin(x), asin(x)); }
GPUdi() constexpr double GPUCommonMath::ACosd(double x) { return GPUCA_CHOICE(std::acos(x), acos(x), acos(x)); }
GPUdi() constexpr double GPUCommonMath::ATand(double x) { return GPUCA_CHOICE(std::atan(x), atan(x), atan(x)); }
GPUdi() constexpr double GPUCommonMath::ATan2d(double y, double x) { return GPUCA_CHOICE(std::atan2(y, x), atan2(y, x), atan2(y, x)); }
GPUdi() constexpr double GPUCommonMath::Sind(double x) { return GPUCA_CHOICE(std::sin(x), sin(x), sin(x)); }
GPUdi() constexpr double GPUCommonMath::Cosd(double x) { return GPUCA_CHOICE(std::cos(x), cos(x), cos(x)); }
GPUdi() constexpr double GPUCommonMath::Tand(double x) { return GPUCA_CHOICE(std::tan(x), tan(x), tan(x)); }
GPUdi() constexpr double GPUCommonMath::Logd(double x) { return GPUCA_CHOICE(std::log(x), log(x), log(x)); }
GPUdi() constexpr double GPUCommonMath::Copysignd(double x, double y) { return GPUCA_CHOICE(std::copysign(x, y), copysign(x, y), copysign(x, y)); }
#endif

GPUdi() constexpr uint32_t GPUCommonMath::Clz(uint32_t x)
Expand Down Expand Up @@ -380,6 +420,44 @@ GPUdi() constexpr uint32_t GPUCommonMath::Popcount(uint32_t x)
#endif
}

GPUdi() constexpr uint32_t GPUCommonMath::Clz64(uint64_t x)
{
#if (defined(__GNUC__) || defined(__clang__) || defined(__CUDACC__) || defined(__HIPCC__))
return x == 0 ? 64 : GPUCA_CHOICE(__builtin_clzll(x), __clzll(x), __builtin_clzll(x)); // use builtin if available
#else
for (int32_t i = 63; i >= 0; i--) {
if (x & (1ull << i)) {
return (63 - i);
}
}
return 64;
#endif
}

GPUdi() constexpr uint32_t GPUCommonMath::Ctz64(uint64_t x)
{
#if (defined(__GNUC__) || defined(__clang__) || defined(__CUDACC__) || defined(__HIPCC__))
return x == 0 ? 64 : GPUCA_CHOICE(__builtin_ctzll(x), __ffsll((long long)x) - 1, __builtin_ctzll(x));
#else
for (uint32_t i = 0; i < 64; ++i) {
if (x & (1ull << i)) {
return i;
}
}
return 64;
#endif
}

GPUdi() constexpr uint32_t GPUCommonMath::Popcount64(uint64_t x)
{
#if (defined(__GNUC__) || defined(__clang__) || defined(__CUDACC__) || defined(__HIPCC__)) && !defined(__OPENCL__) // TODO: remove OPENCL when reported SPIR-V bug is fixed
// use builtin if available
return GPUCA_CHOICE(__builtin_popcountll(x), __popcll(x), __builtin_popcountll(x));
#else
return Popcount((uint32_t)x) + Popcount((uint32_t)(x >> 32));
#endif
}

template <typename T>
GPUhdi() constexpr void GPUCommonMath::Swap(T& a, T& b)
{
Expand Down Expand Up @@ -600,6 +678,41 @@ GPUdi() void GPUCommonMath::AtomicMinInternal(S* addr, T val)
#endif // GPUCA_GPUCODE
}

// Relaxed, as the device atomics above
template <class T, class S>
GPUdi() T GPUCommonMath::AtomicLoadInternal(S* addr)
{
#if defined(GPUCA_GPUCODE) && defined(__OPENCL__) && (!defined(__clang__) || defined(GPUCA_OPENCL_CLANG_C11_ATOMICS))
return ::atomic_load_explicit(addr, memory_order_relaxed);
#elif defined(GPUCA_GPUCODE) && (defined(__OPENCL__) || defined(__CUDACC__) || defined(__HIPCC__))
return *(volatile S*)addr;
#elif defined(GPUCA_GPUCODE) && defined(__METAL__)
return atomic_load_explicit(GPUCommonMathMetalAtomic(addr), memory_order_relaxed);
#elif defined(WITH_OPENMP)
T ret;
__atomic_load(addr, &ret, __ATOMIC_RELAXED);
return ret;
#else
return reinterpret_cast<std::atomic<T>*>(addr)->load(std::memory_order_relaxed);
#endif
}

template <class S, class T>
GPUdi() void GPUCommonMath::AtomicStoreInternal(S* addr, T val)
{
#if defined(GPUCA_GPUCODE) && defined(__OPENCL__) && (!defined(__clang__) || defined(GPUCA_OPENCL_CLANG_C11_ATOMICS))
::atomic_store_explicit(addr, val, memory_order_relaxed);
#elif defined(GPUCA_GPUCODE) && (defined(__OPENCL__) || defined(__CUDACC__) || defined(__HIPCC__))
*(volatile S*)addr = val;
#elif defined(GPUCA_GPUCODE) && defined(__METAL__)
atomic_store_explicit(GPUCommonMathMetalAtomic(addr), val, memory_order_relaxed);
#elif defined(WITH_OPENMP)
__atomic_store(addr, &val, __ATOMIC_RELAXED);
#else
reinterpret_cast<std::atomic<T>*>(addr)->store(val, std::memory_order_relaxed);
#endif
}

#if (defined(__CUDACC__) || defined(__HIPCC__)) && !defined(G__ROOT) && !defined(__CLING__)
#define GPUCA_HAVE_ATOMIC_MINMAX_FLOAT
template <>
Expand Down
Loading