From f147c036b6f4699ba561a6809c2d94de7ce1fa3e Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 18 Nov 2026 11:31:46 +0100 Subject: [PATCH 1/2] GPU: Unify usage of half precision float into half_t typedef in GPUCommonMath.h --- Common/ML/include/ML/3rdparty/GPUORTFloat16.h | 887 ------------------ Common/ML/src/OrtInterface.cxx | 7 +- GPU/Common/GPUCommonMath.h | 16 + .../hip/GPUReconstructionHIPIncludesSystem.h | 1 + .../GPUReconstructionMetalIncludesHost.h | 3 - .../opencl/GPUReconstructionOCLIncludesHost.h | 2 - .../Definitions/Parameters/GPUParameters.csv | 4 +- .../Parameters/GPUParameters.json.example | 8 +- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 1 + .../TPCClusterFinder/GPUTPCNNClusterizer.cxx | 1 - .../TPCClusterFinder/GPUTPCNNClusterizer.h | 3 +- .../GPUTPCNNClusterizerHost.cxx | 1 - .../GPUTPCNNClusterizerHost.h | 5 - .../GPUTPCNNClusterizerKernels.cxx | 50 +- .../devtools/RegularSpline1D.h | 6 +- 15 files changed, 53 insertions(+), 942 deletions(-) delete mode 100644 Common/ML/include/ML/3rdparty/GPUORTFloat16.h diff --git a/Common/ML/include/ML/3rdparty/GPUORTFloat16.h b/Common/ML/include/ML/3rdparty/GPUORTFloat16.h deleted file mode 100644 index 75e146d872cd1..0000000000000 --- a/Common/ML/include/ML/3rdparty/GPUORTFloat16.h +++ /dev/null @@ -1,887 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// This code was created from: -// - https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/onnxruntime_float16.h -// - https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/onnxruntime_cxx_api.h - -#ifndef GPUORTFLOAT16_H -#define GPUORTFLOAT16_H - -#ifndef GPUCA_GPUCODE_DEVICE -#include -#include -#include -#include -#endif - -#include "GPUCommonDef.h" -#include "GPUCommonMath.h" - -namespace o2 -{ - -namespace OrtDataType -{ - -namespace detail -{ - -enum class endian { -#if defined(_WIN32) - little = 0, - big = 1, - native = little, -#elif defined(__GNUC__) || defined(__clang__) - little = __ORDER_LITTLE_ENDIAN__, - big = __ORDER_BIG_ENDIAN__, - native = __BYTE_ORDER__, -#else -#error OrtDataType::detail::endian is not implemented in this environment. -#endif -}; - -static_assert( - endian::native == endian::little || endian::native == endian::big, - "Only little-endian or big-endian native byte orders are supported."); - -} // namespace detail - -/// -/// Shared implementation between public and internal classes. CRTP pattern. -/// -template -struct Float16Impl { - protected: - /// - /// Converts from float to uint16_t float16 representation - /// - /// - /// - GPUd() constexpr static uint16_t ToUint16Impl(float v) noexcept; - - /// - /// Converts float16 to float - /// - /// float representation of float16 value - GPUd() float ToFloatImpl() const noexcept; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - GPUd() uint16_t AbsImpl() const noexcept - { - return static_cast(val & ~kSignMask); - } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - GPUd() uint16_t NegateImpl() const noexcept - { - return IsNaN() ? val : static_cast(val ^ kSignMask); - } - - public: - // uint16_t special values - static constexpr uint16_t kSignMask = 0x8000U; - static constexpr uint16_t kBiasedExponentMask = 0x7C00U; - static constexpr uint16_t kPositiveInfinityBits = 0x7C00U; - static constexpr uint16_t kNegativeInfinityBits = 0xFC00U; - static constexpr uint16_t kPositiveQNaNBits = 0x7E00U; - static constexpr uint16_t kNegativeQNaNBits = 0xFE00U; - static constexpr uint16_t kEpsilonBits = 0x4170U; - static constexpr uint16_t kMinValueBits = 0xFBFFU; // Minimum normal number - static constexpr uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number - static constexpr uint16_t kOneBits = 0x3C00U; - static constexpr uint16_t kMinusOneBits = 0xBC00U; - - uint16_t val{0}; - - GPUdDefault() Float16Impl() = default; - - /// - /// Checks if the value is negative - /// - /// true if negative - GPUd() bool IsNegative() const noexcept - { - return static_cast(val) < 0; - } - - /// - /// Tests if the value is NaN - /// - /// true if NaN - GPUd() bool IsNaN() const noexcept - { - return AbsImpl() > kPositiveInfinityBits; - } - - /// - /// Tests if the value is finite - /// - /// true if finite - GPUd() bool IsFinite() const noexcept - { - return AbsImpl() < kPositiveInfinityBits; - } - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - GPUd() bool IsPositiveInfinity() const noexcept - { - return val == kPositiveInfinityBits; - } - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - GPUd() bool IsNegativeInfinity() const noexcept - { - return val == kNegativeInfinityBits; - } - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - GPUd() bool IsInfinity() const noexcept - { - return AbsImpl() == kPositiveInfinityBits; - } - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - GPUd() bool IsNaNOrZero() const noexcept - { - auto abs = AbsImpl(); - return (abs == 0 || abs > kPositiveInfinityBits); - } - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - GPUd() bool IsNormal() const noexcept - { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) != 0); // is not subnormal (has a non-zero exponent) - } - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - GPUd() bool IsSubnormal() const noexcept - { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) == 0); // is subnormal (has a zero exponent) - } - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - GPUd() Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - GPUd() Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - GPUd() static bool AreZero(const Float16Impl& lhs, const Float16Impl& rhs) noexcept - { - return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; - } - - GPUd() bool operator==(const Float16Impl& rhs) const noexcept - { - if (IsNaN() || rhs.IsNaN()) { - // IEEE defines that NaN is not equal to anything, including itself. - return false; - } - return val == rhs.val; - } - - GPUd() bool operator!=(const Float16Impl& rhs) const noexcept { return !(*this == rhs); } - - GPUd() bool operator<(const Float16Impl& rhs) const noexcept - { - if (IsNaN() || rhs.IsNaN()) { - // IEEE defines that NaN is unordered with respect to everything, including itself. - return false; - } - - const bool left_is_negative = IsNegative(); - if (left_is_negative != rhs.IsNegative()) { - // When the signs of left and right differ, we know that left is less than right if it is - // the negative value. The exception to this is if both values are zero, in which case IEEE - // says they should be equal, even if the signs differ. - return left_is_negative && !AreZero(*this, rhs); - } - return (val != rhs.val) && ((val < rhs.val) ^ left_is_negative); - } -}; - -// The following Float16_t conversions are based on the code from -// Eigen library. - -// The conversion routines are Copyright (c) Fabian Giesen, 2016. -// The original license follows: -// -// Copyright (c) Fabian Giesen, 2016 -// All rights reserved. -// Redistribution and use in source and binary forms, with or without -// modification, are permitted. -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -namespace detail -{ -union float32_bits { - unsigned int u; - float f; -}; -}; // namespace detail - -template -GPUdi() constexpr uint16_t Float16Impl::ToUint16Impl(float v) noexcept -{ - detail::float32_bits f{}; - f.f = v; - - constexpr detail::float32_bits f32infty = {255 << 23}; - constexpr detail::float32_bits f16max = {(127 + 16) << 23}; - constexpr detail::float32_bits denorm_magic = {((127 - 15) + (23 - 10) + 1) << 23}; - constexpr unsigned int sign_mask = 0x80000000u; - uint16_t val = static_cast(0x0u); - - unsigned int sign = f.u & sign_mask; - f.u ^= sign; - - // NOTE all the integer compares in this function can be safely - // compiled into signed compares since all operands are below - // 0x80000000. Important if you want fast straight SSE2 code - // (since there's no unsigned PCMPGTD). - - if (f.u >= f16max.u) { // result is Inf or NaN (all exponent bits set) - val = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf - } else { // (De)normalized number or zero - if (f.u < (113 << 23)) { // resulting FP16 is subnormal or zero - // use a magic value to align our 10 mantissa bits at the bottom of - // the float. as long as FP addition is round-to-nearest-even this - // just works. - f.f += denorm_magic.f; - - // and one integer subtract of the bias later, we have our final float! - val = static_cast(f.u - denorm_magic.u); - } else { - unsigned int mant_odd = (f.u >> 13) & 1; // resulting mantissa is odd - - // update exponent, rounding bias part 1 - // Equivalent to `f.u += ((unsigned int)(15 - 127) << 23) + 0xfff`, but - // without arithmetic overflow. - f.u += 0xc8000fffU; - // rounding bias part 2 - f.u += mant_odd; - // take the bits! - val = static_cast(f.u >> 13); - } - } - - val |= static_cast(sign >> 16); - return val; -} - -template -GPUdi() float Float16Impl::ToFloatImpl() const noexcept -{ - constexpr detail::float32_bits magic = {113 << 23}; - constexpr unsigned int shifted_exp = 0x7c00 << 13; // exponent mask after shift - detail::float32_bits o{}; - - o.u = (val & 0x7fff) << 13; // exponent/mantissa bits - unsigned int exp = shifted_exp & o.u; // just the exponent - o.u += (127 - 15) << 23; // exponent adjust - - // handle exponent special cases - if (exp == shifted_exp) { // Inf/NaN? - o.u += (128 - 16) << 23; // extra exp adjust - } else if (exp == 0) { // Zero/Denormal? - o.u += 1 << 23; // extra exp adjust - o.f -= magic.f; // re-normalize - } - - // Attempt to workaround the Internal Compiler Error on ARM64 - // for bitwise | operator, including std::bitset -#if (defined _MSC_VER) && (defined _M_ARM || defined _M_ARM64 || defined _M_ARM64EC) - if (IsNegative()) { - return -o.f; - } -#else - // original code: - o.u |= (val & 0x8000U) << 16U; // sign bit -#endif - return o.f; -} - -/// Shared implementation between public and internal classes. CRTP pattern. -template -struct BFloat16Impl { - protected: - /// - /// Converts from float to uint16_t float16 representation - /// - /// - /// - GPUd() static uint16_t ToUint16Impl(float v) noexcept; - - /// - /// Converts bfloat16 to float - /// - /// float representation of bfloat16 value - GPUd() float ToFloatImpl() const noexcept; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - GPUd() uint16_t AbsImpl() const noexcept - { - return static_cast(val & ~kSignMask); - } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - GPUd() uint16_t NegateImpl() const noexcept - { - return IsNaN() ? val : static_cast(val ^ kSignMask); - } - - public: - // uint16_t special values - static constexpr uint16_t kSignMask = 0x8000U; - static constexpr uint16_t kBiasedExponentMask = 0x7F80U; - static constexpr uint16_t kPositiveInfinityBits = 0x7F80U; - static constexpr uint16_t kNegativeInfinityBits = 0xFF80U; - static constexpr uint16_t kPositiveQNaNBits = 0x7FC1U; - static constexpr uint16_t kNegativeQNaNBits = 0xFFC1U; - static constexpr uint16_t kSignaling_NaNBits = 0x7F80U; - static constexpr uint16_t kEpsilonBits = 0x0080U; - static constexpr uint16_t kMinValueBits = 0xFF7FU; - static constexpr uint16_t kMaxValueBits = 0x7F7FU; - static constexpr uint16_t kRoundToNearest = 0x7FFFU; - static constexpr uint16_t kOneBits = 0x3F80U; - static constexpr uint16_t kMinusOneBits = 0xBF80U; - - uint16_t val{0}; - - GPUdDefault() BFloat16Impl() = default; - - /// - /// Checks if the value is negative - /// - /// true if negative - GPUd() bool IsNegative() const noexcept - { - return static_cast(val) < 0; - } - - /// - /// Tests if the value is NaN - /// - /// true if NaN - GPUd() bool IsNaN() const noexcept - { - return AbsImpl() > kPositiveInfinityBits; - } - - /// - /// Tests if the value is finite - /// - /// true if finite - GPUd() bool IsFinite() const noexcept - { - return AbsImpl() < kPositiveInfinityBits; - } - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - GPUd() bool IsPositiveInfinity() const noexcept - { - return val == kPositiveInfinityBits; - } - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - GPUd() bool IsNegativeInfinity() const noexcept - { - return val == kNegativeInfinityBits; - } - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - GPUd() bool IsInfinity() const noexcept - { - return AbsImpl() == kPositiveInfinityBits; - } - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - GPUd() bool IsNaNOrZero() const noexcept - { - auto abs = AbsImpl(); - return (abs == 0 || abs > kPositiveInfinityBits); - } - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - GPUd() bool IsNormal() const noexcept - { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) != 0); // is not subnormal (has a non-zero exponent) - } - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - GPUd() bool IsSubnormal() const noexcept - { - auto abs = AbsImpl(); - return (abs < kPositiveInfinityBits) // is finite - && (abs != 0) // is not zero - && ((abs & kBiasedExponentMask) == 0); // is subnormal (has a zero exponent) - } - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - GPUd() Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - GPUd() Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - GPUd() static bool AreZero(const BFloat16Impl& lhs, const BFloat16Impl& rhs) noexcept - { - // IEEE defines that positive and negative zero are equal, this gives us a quick equality check - // for two values by or'ing the private bits together and stripping the sign. They are both zero, - // and therefore equivalent, if the resulting value is still zero. - return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; - } -}; - -template -GPUdi() uint16_t BFloat16Impl::ToUint16Impl(float v) noexcept -{ - uint16_t result; - if (o2::gpu::CAMath::IsNaN(v)) { - result = kPositiveQNaNBits; - } else { - auto get_msb_half = [](float fl) { - uint16_t res; -#ifdef GPUCA_GPUCODE - o2::gpu::CAMath::memcpy(&res, reinterpret_cast(&fl) + sizeof(uint16_t), sizeof(uint16_t)); -#else -#ifdef __cpp_if_constexpr - if constexpr (detail::endian::native == detail::endian::little) -#else - if (detail::endian::native == detail::endian::little) -#endif - { - std::memcpy(&res, reinterpret_cast(&fl) + sizeof(uint16_t), sizeof(uint16_t)); - } else { - std::memcpy(&res, &fl, sizeof(uint16_t)); - } -#endif - return res; - }; - - uint16_t upper_bits = get_msb_half(v); - union { - uint32_t U32; - float F32; - }; - F32 = v; - U32 += (upper_bits & 1) + kRoundToNearest; - result = get_msb_half(F32); - } - return result; -} - -template -GPUdi() float BFloat16Impl::ToFloatImpl() const noexcept -{ -#ifndef __FAST_MATH__ - if (IsNaN()) { - return o2::gpu::CAMath::QuietNaN(); - } -#endif - float result; - char* const first = reinterpret_cast(&result); - char* const second = first + sizeof(uint16_t); -#ifdef GPUCA_GPUCODE - first[0] = first[1] = 0; - o2::gpu::CAMath::memcpy(second, &val, sizeof(uint16_t)); -#else -#ifdef __cpp_if_constexpr - if constexpr (detail::endian::native == detail::endian::little) -#else - if (detail::endian::native == detail::endian::little) -#endif - { - std::memset(first, 0, sizeof(uint16_t)); - std::memcpy(second, &val, sizeof(uint16_t)); - } else { - std::memcpy(first, &val, sizeof(uint16_t)); - std::memset(second, 0, sizeof(uint16_t)); - } -#endif - return result; -} - -/** \brief IEEE 754 half-precision floating point data type - * - * \details This struct is used for converting float to float16 and back - * so the user could feed inputs and fetch outputs using these type. - * - * The size of the structure should align with uint16_t and one can freely cast - * uint16_t buffers to/from Ort::Float16_t to feed and retrieve data. - * - * \code{.unparsed} - * // This example demonstrates converion from float to float16 - * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; - * std::vector fp16_values; - * fp16_values.reserve(std::size(values)); - * std::transform(std::begin(values), std::end(values), std::back_inserter(fp16_values), - * [](float value) { return Ort::Float16_t(value); }); - * - * \endcode - */ -struct Float16_t : OrtDataType::Float16Impl { - private: - /// - /// Constructor from a 16-bit representation of a float16 value - /// No conversion is done here. - /// - /// 16-bit representation - constexpr explicit Float16_t(uint16_t v) noexcept { val = v; } - - public: - using Base = OrtDataType::Float16Impl; - - /// - /// Default constructor - /// - GPUdDefault() Float16_t() = default; - - /// - /// Explicit conversion to uint16_t representation of float16. - /// - /// uint16_t bit representation of float16 - /// new instance of Float16_t - GPUd() constexpr static Float16_t FromBits(uint16_t v) noexcept { return Float16_t(v); } - - /// - /// __ctor from float. Float is converted into float16 16-bit representation. - /// - /// float value - GPUd() explicit Float16_t(float v) noexcept { val = Base::ToUint16Impl(v); } - - /// - /// Converts float16 to float - /// - /// float representation of float16 value - GPUd() float ToFloat() const noexcept { return Base::ToFloatImpl(); } - - /// - /// Checks if the value is negative - /// - /// true if negative - using Base::IsNegative; - - /// - /// Tests if the value is NaN - /// - /// true if NaN - using Base::IsNaN; - - /// - /// Tests if the value is finite - /// - /// true if finite - using Base::IsFinite; - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - using Base::IsPositiveInfinity; - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - using Base::IsNegativeInfinity; - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - using Base::IsInfinity; - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - using Base::IsNaNOrZero; - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - using Base::IsNormal; - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - using Base::IsSubnormal; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - using Base::Abs; - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - using Base::Negate; - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - using Base::AreZero; - - /// - /// User defined conversion operator. Converts Float16_t to float. - /// - GPUdi() explicit operator float() const noexcept { return ToFloat(); } - - using Base::operator==; - using Base::operator!=; - using Base::operator<; -}; - -static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match"); - -/** \brief bfloat16 (Brain Floating Point) data type - * - * \details This struct is used for converting float to bfloat16 and back - * so the user could feed inputs and fetch outputs using these type. - * - * The size of the structure should align with uint16_t and one can freely cast - * uint16_t buffers to/from Ort::BFloat16_t to feed and retrieve data. - * - * \code{.unparsed} - * // This example demonstrates converion from float to float16 - * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; - * std::vector bfp16_values; - * bfp16_values.reserve(std::size(values)); - * std::transform(std::begin(values), std::end(values), std::back_inserter(bfp16_values), - * [](float value) { return Ort::BFloat16_t(value); }); - * - * \endcode - */ -struct BFloat16_t : OrtDataType::BFloat16Impl { - private: - /// - /// Constructor from a uint16_t representation of bfloat16 - /// used in FromBits() to escape overload resolution issue with - /// constructor from float. - /// No conversion is done. - /// - /// 16-bit bfloat16 value - constexpr explicit BFloat16_t(uint16_t v) noexcept { val = v; } - - public: - using Base = OrtDataType::BFloat16Impl; - - GPUdDefault() BFloat16_t() = default; - - /// - /// Explicit conversion to uint16_t representation of bfloat16. - /// - /// uint16_t bit representation of bfloat16 - /// new instance of BFloat16_t - GPUd() static constexpr BFloat16_t FromBits(uint16_t v) noexcept { return BFloat16_t(v); } - - /// - /// __ctor from float. Float is converted into bfloat16 16-bit representation. - /// - /// float value - GPUd() explicit BFloat16_t(float v) noexcept { val = Base::ToUint16Impl(v); } - - /// - /// Converts bfloat16 to float - /// - /// float representation of bfloat16 value - GPUd() float ToFloat() const noexcept { return Base::ToFloatImpl(); } - - /// - /// Checks if the value is negative - /// - /// true if negative - using Base::IsNegative; - - /// - /// Tests if the value is NaN - /// - /// true if NaN - using Base::IsNaN; - - /// - /// Tests if the value is finite - /// - /// true if finite - using Base::IsFinite; - - /// - /// Tests if the value represents positive infinity. - /// - /// true if positive infinity - using Base::IsPositiveInfinity; - - /// - /// Tests if the value represents negative infinity - /// - /// true if negative infinity - using Base::IsNegativeInfinity; - - /// - /// Tests if the value is either positive or negative infinity. - /// - /// True if absolute value is infinity - using Base::IsInfinity; - - /// - /// Tests if the value is NaN or zero. Useful for comparisons. - /// - /// True if NaN or zero. - using Base::IsNaNOrZero; - - /// - /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). - /// - /// True if so - using Base::IsNormal; - - /// - /// Tests if the value is subnormal (denormal). - /// - /// True if so - using Base::IsSubnormal; - - /// - /// Creates an instance that represents absolute value. - /// - /// Absolute value - using Base::Abs; - - /// - /// Creates a new instance with the sign flipped. - /// - /// Flipped sign instance - using Base::Negate; - - /// - /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check - /// for two values by or'ing the private bits together and stripping the sign. They are both zero, - /// and therefore equivalent, if the resulting value is still zero. - /// - /// first value - /// second value - /// True if both arguments represent zero - using Base::AreZero; - - /// - /// User defined conversion operator. Converts BFloat16_t to float. - /// - GPUdi() explicit operator float() const noexcept { return ToFloat(); } - - // We do not have an inherited impl for the below operators - // as the internal class implements them a little differently - bool operator==(const BFloat16_t& rhs) const noexcept; - bool operator!=(const BFloat16_t& rhs) const noexcept { return !(*this == rhs); } - bool operator<(const BFloat16_t& rhs) const noexcept; -}; - -static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match"); - -} // namespace OrtDataType - -} // namespace o2 -#endif diff --git a/Common/ML/src/OrtInterface.cxx b/Common/ML/src/OrtInterface.cxx index 9eccb9638d882..407ca91469b47 100644 --- a/Common/ML/src/OrtInterface.cxx +++ b/Common/ML/src/OrtInterface.cxx @@ -14,7 +14,12 @@ /// \brief A header library for loading ONNX models and inferencing them on CPU and GPU #include "ML/OrtInterface.h" -#include "ML/3rdparty/GPUORTFloat16.h" +#include "GPUCommonMath.h" + +namespace o2::OrtDataType +{ +using Float16_t = o2::gpu::GPUCommonMath::half_t; +} // ONNX includes #include diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index f32cf61a057ae..c55b04ddc0743 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -19,6 +19,11 @@ #if defined(__CUDACC__) && !defined(__clang__) && !defined(GPUCA_GPUCODE_COMPILEKERNELS) && !defined(GPUCA_GPUCODE_HOSTONLY) #include +#include +#endif + +#if defined(__HIPCC__) && !defined(GPUCA_GPUCODE_COMPILEKERNELS) && !defined(GPUCA_GPUCODE_HOSTONLY) +#include #endif #if !defined(GPUCA_GPUCODE_DEVICE) @@ -27,6 +32,9 @@ #include #include #include +#if __cplusplus >= 202302L && __has_include() +#include +#endif #endif #if !defined(GPUCA_GPUCODE_COMPILEKERNELS) && (!defined(GPUCA_GPUCODE_DEVICE) || defined(__CUDACC__) || defined(__HIPCC__)) @@ -48,6 +56,14 @@ namespace o2::gpu class GPUCommonMath { public: +#if (defined(GPUCA_GPUCODE_DEVICE) && !defined(__CPUDEVICE__)) || defined(__CUDACC__) + using half_t = half; +#elif __cplusplus >= 202302L && defined(__STDCPP_FLOAT16_T__) + using half_t = std::float16_t; +#else + using half_t = _Float16; +#endif + GPUd() static float2 MakeFloat2(float x, float y); // TODO: Find better appraoch that is constexpr template diff --git a/GPU/GPUTracking/Base/hip/GPUReconstructionHIPIncludesSystem.h b/GPU/GPUTracking/Base/hip/GPUReconstructionHIPIncludesSystem.h index 389c79c0e4eb9..9deb68f473299 100644 --- a/GPU/GPUTracking/Base/hip/GPUReconstructionHIPIncludesSystem.h +++ b/GPU/GPUTracking/Base/hip/GPUReconstructionHIPIncludesSystem.h @@ -25,5 +25,6 @@ #include #include #include +#include #endif // O2_GPU_RECONSTRUCTIONHIPINCLUDESSYSTEM_H diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h index 23b2e851de76f..82389e3053c25 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h @@ -22,9 +22,6 @@ #ifndef __METAL_VERSION__ // __METAL_VERSION__ is only defined in device code. #define __METAL_HOST__ -// The parameter table picks MSL's `half` for the device layouts this translation -// unit has to agree with; _Float16 is the same IEEE binary16 type. -using half = _Float16; #endif #import diff --git a/GPU/GPUTracking/Base/opencl/GPUReconstructionOCLIncludesHost.h b/GPU/GPUTracking/Base/opencl/GPUReconstructionOCLIncludesHost.h index 5432d00396434..c79b021e80308 100644 --- a/GPU/GPUTracking/Base/opencl/GPUReconstructionOCLIncludesHost.h +++ b/GPU/GPUTracking/Base/opencl/GPUReconstructionOCLIncludesHost.h @@ -25,8 +25,6 @@ #include #include -typedef int16_t half; // TODO: Fixme once we have C++23 and proper host side half support - #include "GPULogging.h" #include "GPUReconstructionOCL.h" diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv index 330c9a3fa7377..8e41173cee40a 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv @@ -109,8 +109,8 @@ TRACKLET_SELECTOR_HITS_REG_SIZE,12,0,9,27,,,,,,20,20,20,20,,20,,2,20, ALTERNATE_BORDER_SORT,1,0,1,1,,,,,,1,1,1,1,,1,,1,1, SORT_BEFORE_FIT,1,0,1,1,,,,,,1,1,1,1,,1,,1,1, NO_ATOMIC_PRECHECK,0,0,1,1,,,,,,1,1,1,1,,1,,1,1, -DEDX_STORAGE_TYPE,"""half""","""float""",,,,,,,,,,,,,,,,, -MERGER_INTERPOLATION_ERROR_TYPE,"""half""","""float""",,,,,,,,,,,,,,,,, +DEDX_STORAGE_TYPE,"""CAMath::half_t""","""float""",,,,,,,,,,,,,,,,, +MERGER_INTERPOLATION_ERROR_TYPE,"""CAMath::half_t""","""float""",,,,,,,,,,,,,,,,, COMP_GATHER_KERNEL,4,0,4,4,,,,,,4,4,4,4,,4,,4,4, COMP_GATHER_MODE,3,0,3,3,,,,,,3,3,3,3,,3,,3,3, CF_SCAN_WORKGROUP_SIZE,512,0,,,,,,,,224,,992,448,,1024,,,448, diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example index 3c6f1af1aab2f..15f905cbcc41e 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example @@ -588,10 +588,10 @@ "MERGER_INTERPOLATION_ERROR_TYPE": { "default": "float", "default_cpu": "float", - "MI100": "half", - "VEGA": "half", - "AMPERE": "half", - "TURING": "half" + "MI100": "CAMath::half_t", + "VEGA": "CAMath::half_t", + "AMPERE": "CAMath::half_t", + "TURING": "CAMath::half_t" }, "COMP_GATHER_KERNEL": { "default": 0, diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index 99c47b6228889..89b553aba9d1c 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -16,6 +16,7 @@ #define GPUTPCGMMERGERTYPES_H #include "GPUTPCDef.h" +#include "GPUCommonMath.h" #include "GPUGeneralKernels.h" namespace o2::gpu::gputpcgmmergertypes diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizer.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizer.cxx index 0e77393be1ce3..c041cfcc10121 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizer.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizer.cxx @@ -13,7 +13,6 @@ /// \author Christian Sonnabend #include "GPUReconstruction.h" -#include "ML/3rdparty/GPUORTFloat16.h" #include "GPUTPCNNClusterizer.h" #include "GPUSettings.h" #include "GPUCommonLogger.h" diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizer.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizer.h index 7aa23489eb2f4..89a28cafacf85 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizer.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizer.h @@ -15,12 +15,13 @@ #ifndef O2_GPUTPCNNCLUSTERIZER_H #define O2_GPUTPCNNCLUSTERIZER_H +#include "GPUCommonMath.h" #include "CfChargePos.h" #include "GPUProcessor.h" namespace o2::OrtDataType { -struct Float16_t; +using Float16_t = o2::gpu::GPUCommonMath::half_t; } namespace o2::gpu diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx index 96b8a1d7ed2fd..16dd786c1eb53 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx @@ -17,7 +17,6 @@ #include "GPUTPCNNClusterizerHost.h" #include "GPUTPCNNClusterizer.h" #include "GPUSettings.h" -#include "ML/3rdparty/GPUORTFloat16.h" #include "GPUReconstruction.h" #include "GPUTPCGeometry.h" #include "DataFormatsTPC/Constants.h" diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.h index 8f8465d5dca34..a70855610651b 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.h @@ -29,11 +29,6 @@ struct Env; struct MemoryInfo; } // namespace Ort -namespace o2::OrtDataType -{ -struct Float16_t; -} - namespace o2::gpu { diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx index c57c01bdda7a8..61c71666ab862 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx @@ -25,20 +25,6 @@ using namespace o2::gpu::tpccf; #include "CfConsts.h" #include "CfUtils.h" #include "ClusterAccumulator.h" -#ifdef __METAL__ -namespace o2::OrtDataType -{ -struct Float16_t { - half val; - Float16_t() = default; - explicit Float16_t(float v) : val(v) {} - float ToFloat() const { return (float)val; } - explicit operator float() const { return ToFloat(); } -}; -} // namespace o2::OrtDataType -#else -#include "ML/3rdparty/GPUORTFloat16.h" -#endif #if !defined(GPUCA_GPUCODE) #include "GPUHostDataTypes.h" @@ -266,7 +252,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread clustererNN.mNnClassThreshold); } else { - clustererNN.mOutputDataClass[glo_idx + batchStart] = (int32_t)((clustererNN.mModelProbabilities_16[glo_idx]).ToFloat() > clustererNN.mNnClassThreshold); + clustererNN.mOutputDataClass[glo_idx + batchStart] = (int32_t)((float)clustererNN.mModelProbabilities_16[glo_idx] > clustererNN.mNnClassThreshold); } } else { clustererNN.mOutputDataClass[glo_idx + batchStart] = 1; @@ -297,7 +283,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread(peak.pad()) + clustererNN.mOutputDataReg1_16[model_output_index].ToFloat(); - publishTimePosition = static_cast(peak.time()) + clustererNN.mOutputDataReg1_16[model_output_index + 1].ToFloat(); + publishPadPosition = static_cast(peak.pad()) + (float)clustererNN.mOutputDataReg1_16[model_output_index]; + publishTimePosition = static_cast(peak.time()) + (float)clustererNN.mOutputDataReg1_16[model_output_index + 1]; isBoundaryPublish(full_glo_idx, static_cast(peak.row()), publishPadPosition, publishTimePosition); - pc.setFull(central_charge * clustererNN.mOutputDataReg1_16[model_output_index + 4].ToFloat(), + pc.setFull(central_charge * (float)clustererNN.mOutputDataReg1_16[model_output_index + 4], publishPadPosition, - notSinglePad ? clustererNN.mOutputDataReg1_16[model_output_index + 2].ToFloat() : 0.f, + notSinglePad ? (float)clustererNN.mOutputDataReg1_16[model_output_index + 2] : 0.f, (clusterer.mPmemory->frag).start + publishTimePosition, - notSingleTime ? clustererNN.mOutputDataReg1_16[model_output_index + 3].ToFloat() : 0.f, + notSingleTime ? (float)clustererNN.mOutputDataReg1_16[model_output_index + 3] : 0.f, clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); } @@ -468,7 +454,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread (MAX_CHARS - 32)) { appendStr(" ..."); break; } @@ -583,14 +569,14 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread(peak.pad()) + clustererNN.mOutputDataReg2_16[model_output_index].ToFloat(); - publishTimePosition = static_cast(peak.time()) + clustererNN.mOutputDataReg2_16[model_output_index + 1].ToFloat(); + publishPadPosition = static_cast(peak.pad()) + (float)clustererNN.mOutputDataReg2_16[model_output_index]; + publishTimePosition = static_cast(peak.time()) + (float)clustererNN.mOutputDataReg2_16[model_output_index + 1]; isBoundaryPublish(full_glo_idx, static_cast(peak.row()), publishPadPosition, publishTimePosition); - pc.setFull(central_charge * clustererNN.mOutputDataReg2_16[model_output_index + 8].ToFloat(), + pc.setFull(central_charge * (float)clustererNN.mOutputDataReg2_16[model_output_index + 8], publishPadPosition, - clustererNN.mOutputDataReg2_16[model_output_index + 4].ToFloat(), + (float)clustererNN.mOutputDataReg2_16[model_output_index + 4], (clusterer.mPmemory->frag).start + publishTimePosition, - clustererNN.mOutputDataReg2_16[model_output_index + 6].ToFloat(), + (float)clustererNN.mOutputDataReg2_16[model_output_index + 6], clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); } @@ -637,14 +623,14 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread(peak.pad()) + clustererNN.mOutputDataReg2_16[model_output_index + 1].ToFloat(); - publishTimePosition = static_cast(peak.time()) + clustererNN.mOutputDataReg2_16[model_output_index + 3].ToFloat(); + publishPadPosition = static_cast(peak.pad()) + (float)clustererNN.mOutputDataReg2_16[model_output_index + 1]; + publishTimePosition = static_cast(peak.time()) + (float)clustererNN.mOutputDataReg2_16[model_output_index + 3]; isBoundaryPublish(full_glo_idx, static_cast(peak.row()), publishPadPosition, publishTimePosition); - pc.setFull(central_charge * clustererNN.mOutputDataReg2_16[model_output_index + 9].ToFloat(), + pc.setFull(central_charge * (float)clustererNN.mOutputDataReg2_16[model_output_index + 9], publishPadPosition, - clustererNN.mOutputDataReg2_16[model_output_index + 5].ToFloat(), + (float)clustererNN.mOutputDataReg2_16[model_output_index + 5], (clusterer.mPmemory->frag).start + publishTimePosition, - clustererNN.mOutputDataReg2_16[model_output_index + 7].ToFloat(), + (float)clustererNN.mOutputDataReg2_16[model_output_index + 7], clustererNN.mClusterFlags[2 * glo_idx], clustererNN.mClusterFlags[2 * glo_idx + 1]); } diff --git a/GPU/TPCFastTransformation/devtools/RegularSpline1D.h b/GPU/TPCFastTransformation/devtools/RegularSpline1D.h index 885b2e12eea0b..3a4b75ea295ec 100644 --- a/GPU/TPCFastTransformation/devtools/RegularSpline1D.h +++ b/GPU/TPCFastTransformation/devtools/RegularSpline1D.h @@ -107,14 +107,14 @@ inline T RegularSpline1D::getSpline(const int32_t iknot1, T f0, T f1, T f2, T f3 f2 -= f1; f3 -= f1; - const T half(0.5f); + const T half_value(0.5f); // Scale u so that the knot1 is at 0.0 and the knot2 is at 1.0 T x = T((mNumberOfKnots - 1) * u - iknot1); T x2 = x * x; - T z1 = half * (f2 - f0); // scaled u derivative at the knot 1 - T z2 = half * f3; // scaled u derivative at the knot 2 + T z1 = half_value * (f2 - f0); // scaled u derivative at the knot 1 + T z2 = half_value * f3; // scaled u derivative at the knot 2 // f(u) = a*u^3 + b*u^2 + c*u + d // From 20e707dccdd0f5151556220cad42e534f43f3f49 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 18 Nov 2026 13:17:08 +0100 Subject: [PATCH 2/2] ONNXRuntime: Use temporary GPUORTFloat16.h file for backward compatibility since needed by O2Physics --- Common/ML/include/ML/3rdparty/GPUORTFloat16.h | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 Common/ML/include/ML/3rdparty/GPUORTFloat16.h diff --git a/Common/ML/include/ML/3rdparty/GPUORTFloat16.h b/Common/ML/include/ML/3rdparty/GPUORTFloat16.h new file mode 100644 index 0000000000000..8fe7f7ee118c5 --- /dev/null +++ b/Common/ML/include/ML/3rdparty/GPUORTFloat16.h @@ -0,0 +1,31 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \author David Rohr +/// Temporary file to support O2Physics code that uses it, should migrate away from this. + +#ifndef GPUORTFLOAT16_H +#define GPUORTFLOAT16_H + +#if __cplusplus >= 202302L && __has_include() +#include +#endif + +namespace o2::OrtDataType +{ +#if __cplusplus >= 202302L && defined(__STDCPP_FLOAT16_T__) +using Float16_t = std::float16_t; +#else +using Float16_t = _Float16; +#endif +} + +#endif