Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
892 changes: 18 additions & 874 deletions Common/ML/include/ML/3rdparty/GPUORTFloat16.h

Large diffs are not rendered by default.

7 changes: 6 additions & 1 deletion Common/ML/src/OrtInterface.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@
/// \brief A header library for loading ONNX models and inferencing them on CPU and GPU

#include "ML/OrtInterface.h"
#include "ML/3rdparty/GPUORTFloat16.h"
#include "GPUCommonMath.h"

namespace o2::OrtDataType
{
using Float16_t = o2::gpu::GPUCommonMath::half_t;
}

// ONNX includes
#include <onnxruntime_cxx_api.h>
Expand Down
16 changes: 16 additions & 0 deletions GPU/Common/GPUCommonMath.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@

#if defined(__CUDACC__) && !defined(__clang__) && !defined(GPUCA_GPUCODE_COMPILEKERNELS) && !defined(GPUCA_GPUCODE_HOSTONLY)
#include <sm_20_atomic_functions.h>
#include <cuda_fp16.h>
#endif

#if defined(__HIPCC__) && !defined(GPUCA_GPUCODE_COMPILEKERNELS) && !defined(GPUCA_GPUCODE_HOSTONLY)
#include <hip/hip_fp16.h>
#endif

#if !defined(GPUCA_GPUCODE_DEVICE)
Expand All @@ -27,6 +32,9 @@
#include <atomic>
#include <limits>
#include <cstring>
#if __cplusplus >= 202302L && __has_include(<stdfloat>)
#include <stdfloat>
#endif
#endif

#if !defined(GPUCA_GPUCODE_COMPILEKERNELS) && (!defined(GPUCA_GPUCODE_DEVICE) || defined(__CUDACC__) || defined(__HIPCC__))
Expand All @@ -48,6 +56,14 @@ namespace o2::gpu
class GPUCommonMath
{
public:
#if (defined(GPUCA_GPUCODE_DEVICE) && !defined(__CPUDEVICE__)) || defined(__CUDACC__)
using half_t = half;
#elif __cplusplus >= 202302L && defined(__STDCPP_FLOAT16_T__)
using half_t = std::float16_t;
#else
using half_t = _Float16;
#endif

GPUd() static float2 MakeFloat2(float x, float y); // TODO: Find better appraoch that is constexpr

template <class T>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,6 @@
#include <thrust/sort.h>
#include <thrust/execution_policy.h>
#include <thrust/device_ptr.h>
#include <hip/hip_fp16.h>

#endif // O2_GPU_RECONSTRUCTIONHIPINCLUDESSYSTEM_H
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,6 @@
#ifndef __METAL_VERSION__
// __METAL_VERSION__ is only defined in device code.
#define __METAL_HOST__
// The parameter table picks MSL's `half` for the device layouts this translation
// unit has to agree with; _Float16 is the same IEEE binary16 type.
using half = _Float16;
#endif

#import <Foundation/Foundation.h>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@
#include <string>
#include <memory>

typedef int16_t half; // TODO: Fixme once we have C++23 and proper host side half support

#include "GPULogging.h"

#include "GPUReconstructionOCL.h"
Expand Down
4 changes: 2 additions & 2 deletions GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ TRACKLET_SELECTOR_HITS_REG_SIZE,12,0,9,27,,,,,,20,20,20,20,,20,,2,20,
ALTERNATE_BORDER_SORT,1,0,1,1,,,,,,1,1,1,1,,1,,1,1,
SORT_BEFORE_FIT,1,0,1,1,,,,,,1,1,1,1,,1,,1,1,
NO_ATOMIC_PRECHECK,0,0,1,1,,,,,,1,1,1,1,,1,,1,1,
DEDX_STORAGE_TYPE,"""half""","""float""",,,,,,,,,,,,,,,,,
MERGER_INTERPOLATION_ERROR_TYPE,"""half""","""float""",,,,,,,,,,,,,,,,,
DEDX_STORAGE_TYPE,"""CAMath::half_t""","""float""",,,,,,,,,,,,,,,,,
MERGER_INTERPOLATION_ERROR_TYPE,"""CAMath::half_t""","""float""",,,,,,,,,,,,,,,,,
COMP_GATHER_KERNEL,4,0,4,4,,,,,,4,4,4,4,,4,,4,4,
COMP_GATHER_MODE,3,0,3,3,,,,,,3,3,3,3,,3,,3,3,
CF_SCAN_WORKGROUP_SIZE,512,0,,,,,,,,224,,992,448,,1024,,,448,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -588,10 +588,10 @@
"MERGER_INTERPOLATION_ERROR_TYPE": {
"default": "float",
"default_cpu": "float",
"MI100": "half",
"VEGA": "half",
"AMPERE": "half",
"TURING": "half"
"MI100": "CAMath::half_t",
"VEGA": "CAMath::half_t",
"AMPERE": "CAMath::half_t",
"TURING": "CAMath::half_t"
},
"COMP_GATHER_KERNEL": {
"default": 0,
Expand Down
1 change: 1 addition & 0 deletions GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#define GPUTPCGMMERGERTYPES_H

#include "GPUTPCDef.h"
#include "GPUCommonMath.h"
#include "GPUGeneralKernels.h"

namespace o2::gpu::gputpcgmmergertypes
Expand Down
1 change: 0 additions & 1 deletion GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizer.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
/// \author Christian Sonnabend

#include "GPUReconstruction.h"
#include "ML/3rdparty/GPUORTFloat16.h"
#include "GPUTPCNNClusterizer.h"
#include "GPUSettings.h"
#include "GPUCommonLogger.h"
Expand Down
3 changes: 2 additions & 1 deletion GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizer.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,13 @@
#ifndef O2_GPUTPCNNCLUSTERIZER_H
#define O2_GPUTPCNNCLUSTERIZER_H

#include "GPUCommonMath.h"
#include "CfChargePos.h"
#include "GPUProcessor.h"

namespace o2::OrtDataType
{
struct Float16_t;
using Float16_t = o2::gpu::GPUCommonMath::half_t;
}

namespace o2::gpu
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
#include "GPUTPCNNClusterizerHost.h"
#include "GPUTPCNNClusterizer.h"
#include "GPUSettings.h"
#include "ML/3rdparty/GPUORTFloat16.h"
#include "GPUReconstruction.h"
#include "GPUTPCGeometry.h"
#include "DataFormatsTPC/Constants.h"
Expand Down
5 changes: 0 additions & 5 deletions GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,6 @@ struct Env;
struct MemoryInfo;
} // namespace Ort

namespace o2::OrtDataType
{
struct Float16_t;
}

namespace o2::gpu
{

Expand Down
50 changes: 18 additions & 32 deletions GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerKernels.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,6 @@ using namespace o2::gpu::tpccf;
#include "CfConsts.h"
#include "CfUtils.h"
#include "ClusterAccumulator.h"
#ifdef __METAL__
namespace o2::OrtDataType
{
struct Float16_t {
half val;
Float16_t() = default;
explicit Float16_t(float v) : val(v) {}
float ToFloat() const { return (float)val; }
explicit operator float() const { return ToFloat(); }
};
} // namespace o2::OrtDataType
#else
#include "ML/3rdparty/GPUORTFloat16.h"
#endif

#if !defined(GPUCA_GPUCODE)
#include "GPUHostDataTypes.h"
Expand Down Expand Up @@ -266,7 +252,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread<GPUTPCNNClusterizerKernels::det
if (dtype == 0) {
clustererNN.mOutputDataClass[glo_idx + batchStart] = (int32_t)(clustererNN.mModelProbabilities_32[glo_idx] > clustererNN.mNnClassThreshold);
} else {
clustererNN.mOutputDataClass[glo_idx + batchStart] = (int32_t)((clustererNN.mModelProbabilities_16[glo_idx]).ToFloat() > clustererNN.mNnClassThreshold);
clustererNN.mOutputDataClass[glo_idx + batchStart] = (int32_t)((float)clustererNN.mModelProbabilities_16[glo_idx] > clustererNN.mNnClassThreshold);
}
} else {
clustererNN.mOutputDataClass[glo_idx + batchStart] = 1;
Expand Down Expand Up @@ -297,7 +283,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread<GPUTPCNNClusterizerKernels::det
if (dtype == 0) {
current_max_prob = CAMath::Max(current_max_prob, clustererNN.mModelProbabilities_32[pIdx]);
} else {
current_max_prob = CAMath::Max(current_max_prob, clustererNN.mModelProbabilities_16[pIdx].ToFloat());
current_max_prob = CAMath::Max(current_max_prob, (float)clustererNN.mModelProbabilities_16[pIdx]);
}
}
}
Expand Down Expand Up @@ -397,14 +383,14 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread<GPUTPCNNClusterizerKernels::pub
clustererNN.mClusterFlags[2 * glo_idx],
clustererNN.mClusterFlags[2 * glo_idx + 1]);
} else {
publishPadPosition = static_cast<float>(peak.pad()) + clustererNN.mOutputDataReg1_16[model_output_index].ToFloat();
publishTimePosition = static_cast<float>(peak.time()) + clustererNN.mOutputDataReg1_16[model_output_index + 1].ToFloat();
publishPadPosition = static_cast<float>(peak.pad()) + (float)clustererNN.mOutputDataReg1_16[model_output_index];
publishTimePosition = static_cast<float>(peak.time()) + (float)clustererNN.mOutputDataReg1_16[model_output_index + 1];
isBoundaryPublish(full_glo_idx, static_cast<int32_t>(peak.row()), publishPadPosition, publishTimePosition);
pc.setFull(central_charge * clustererNN.mOutputDataReg1_16[model_output_index + 4].ToFloat(),
pc.setFull(central_charge * (float)clustererNN.mOutputDataReg1_16[model_output_index + 4],
publishPadPosition,
notSinglePad ? clustererNN.mOutputDataReg1_16[model_output_index + 2].ToFloat() : 0.f,
notSinglePad ? (float)clustererNN.mOutputDataReg1_16[model_output_index + 2] : 0.f,
(clusterer.mPmemory->frag).start + publishTimePosition,
notSingleTime ? clustererNN.mOutputDataReg1_16[model_output_index + 3].ToFloat() : 0.f,
notSingleTime ? (float)clustererNN.mOutputDataReg1_16[model_output_index + 3] : 0.f,
clustererNN.mClusterFlags[2 * glo_idx],
clustererNN.mClusterFlags[2 * glo_idx + 1]);
}
Expand Down Expand Up @@ -468,7 +454,7 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread<GPUTPCNNClusterizerKernels::pub
// int maxPrint = elemSize;
// for (int i = 0; i < maxPrint; ++i) {
// appendChar(' ');
// float v = (dtype == 0) ? clustererNN.mInputData_16[baseIdx + i].ToFloat()
// float v = (dtype == 0) ? (float)clustererNN.mInputData_16[baseIdx + i]
// : clustererNN.mInputData_32[baseIdx + i];
// appendFloat(v);
// if (pos > (MAX_CHARS - 32)) { appendStr(" ..."); break; }
Expand Down Expand Up @@ -583,14 +569,14 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread<GPUTPCNNClusterizerKernels::pub
clustererNN.mClusterFlags[2 * glo_idx],
clustererNN.mClusterFlags[2 * glo_idx + 1]);
} else {
publishPadPosition = static_cast<float>(peak.pad()) + clustererNN.mOutputDataReg2_16[model_output_index].ToFloat();
publishTimePosition = static_cast<float>(peak.time()) + clustererNN.mOutputDataReg2_16[model_output_index + 1].ToFloat();
publishPadPosition = static_cast<float>(peak.pad()) + (float)clustererNN.mOutputDataReg2_16[model_output_index];
publishTimePosition = static_cast<float>(peak.time()) + (float)clustererNN.mOutputDataReg2_16[model_output_index + 1];
isBoundaryPublish(full_glo_idx, static_cast<int32_t>(peak.row()), publishPadPosition, publishTimePosition);
pc.setFull(central_charge * clustererNN.mOutputDataReg2_16[model_output_index + 8].ToFloat(),
pc.setFull(central_charge * (float)clustererNN.mOutputDataReg2_16[model_output_index + 8],
publishPadPosition,
clustererNN.mOutputDataReg2_16[model_output_index + 4].ToFloat(),
(float)clustererNN.mOutputDataReg2_16[model_output_index + 4],
(clusterer.mPmemory->frag).start + publishTimePosition,
clustererNN.mOutputDataReg2_16[model_output_index + 6].ToFloat(),
(float)clustererNN.mOutputDataReg2_16[model_output_index + 6],
clustererNN.mClusterFlags[2 * glo_idx],
clustererNN.mClusterFlags[2 * glo_idx + 1]);
}
Expand Down Expand Up @@ -637,14 +623,14 @@ GPUdii() void GPUTPCNNClusterizerKernels::Thread<GPUTPCNNClusterizerKernels::pub
clustererNN.mClusterFlags[2 * glo_idx],
clustererNN.mClusterFlags[2 * glo_idx + 1]);
} else {
publishPadPosition = static_cast<float>(peak.pad()) + clustererNN.mOutputDataReg2_16[model_output_index + 1].ToFloat();
publishTimePosition = static_cast<float>(peak.time()) + clustererNN.mOutputDataReg2_16[model_output_index + 3].ToFloat();
publishPadPosition = static_cast<float>(peak.pad()) + (float)clustererNN.mOutputDataReg2_16[model_output_index + 1];
publishTimePosition = static_cast<float>(peak.time()) + (float)clustererNN.mOutputDataReg2_16[model_output_index + 3];
isBoundaryPublish(full_glo_idx, static_cast<int32_t>(peak.row()), publishPadPosition, publishTimePosition);
pc.setFull(central_charge * clustererNN.mOutputDataReg2_16[model_output_index + 9].ToFloat(),
pc.setFull(central_charge * (float)clustererNN.mOutputDataReg2_16[model_output_index + 9],
publishPadPosition,
clustererNN.mOutputDataReg2_16[model_output_index + 5].ToFloat(),
(float)clustererNN.mOutputDataReg2_16[model_output_index + 5],
(clusterer.mPmemory->frag).start + publishTimePosition,
clustererNN.mOutputDataReg2_16[model_output_index + 7].ToFloat(),
(float)clustererNN.mOutputDataReg2_16[model_output_index + 7],
clustererNN.mClusterFlags[2 * glo_idx],
clustererNN.mClusterFlags[2 * glo_idx + 1]);
}
Expand Down
6 changes: 3 additions & 3 deletions GPU/TPCFastTransformation/devtools/RegularSpline1D.h
Original file line number Diff line number Diff line change
Expand Up @@ -107,14 +107,14 @@ inline T RegularSpline1D::getSpline(const int32_t iknot1, T f0, T f1, T f2, T f3
f2 -= f1;
f3 -= f1;

const T half(0.5f);
const T half_value(0.5f);

// Scale u so that the knot1 is at 0.0 and the knot2 is at 1.0
T x = T((mNumberOfKnots - 1) * u - iknot1);
T x2 = x * x;

T z1 = half * (f2 - f0); // scaled u derivative at the knot 1
T z2 = half * f3; // scaled u derivative at the knot 2
T z1 = half_value * (f2 - f0); // scaled u derivative at the knot 1
T z2 = half_value * f3; // scaled u derivative at the knot 2

// f(u) = a*u^3 + b*u^2 + c*u + d
//
Expand Down
Loading