diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h index 45617bc00b502..e0ab1f2d5f34d 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h @@ -26,6 +26,11 @@ #include "ITStracking/LineProjection.h" #include "DetectorsBase/Propagator.h" +namespace o2::itsmft::tracking +{ +class ExternalAllocator; +} + namespace o2::its { using o2::itsmft::tracking::bounded_vector; @@ -41,7 +46,7 @@ template class IndexTableUtils; class Cluster; class TrackITSExt; -class ExternalAllocator; +using ExternalAllocator = o2::itsmft::tracking::ExternalAllocator; template struct TrackingKernels { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ExternalAllocator.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ExternalAllocator.h index e858c4bb476f9..7258213d842b5 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ExternalAllocator.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ExternalAllocator.h @@ -16,72 +16,13 @@ #ifndef TRACKINGITSU_INCLUDE_EXTERNALALLOCATOR_H_ #define TRACKINGITSU_INCLUDE_EXTERNALALLOCATOR_H_ -#include #include "GPUO2ExternalUser.h" -#include "Base/GPUMemoryResource.h" +#include "ITSMFTTracking/ExternalAllocator.h" namespace o2::its { - -class ExternalAllocator -{ - using Type = std::underlying_type_t; - - public: - virtual void deallocate(char*, size_t) = 0; - virtual void* allocate(size_t) = 0; - void* allocate(size_t s, Type type) - { - auto old = mType; - mType = type; - void* p = allocate(s); - mType = old; - return p; - } - - void* allocateStack(size_t s) - { - return allocate(s, (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); - } - virtual void pushTagOnStack(uint64_t) = 0; - virtual void popTagOffStack(uint64_t) = 0; - - void setType(Type t) noexcept { mType = t; } - Type getType() const noexcept { return mType; } - - protected: - Type mType; -}; - -class ExternalAllocatorAdaptor final : public std::pmr::memory_resource -{ - public: - explicit ExternalAllocatorAdaptor(ExternalAllocator* alloc) : mAlloc(alloc) {} - - protected: - void* do_allocate(size_t bytes, size_t alignment) override - { - void* p = mAlloc->allocate(bytes, o2::gpu::GPUMemoryResource::MemoryType::MEMORY_HOST); - if (!p) { - throw std::bad_alloc(); - } - return p; - } - - void do_deallocate(void* p, size_t bytes, size_t) override - { - mAlloc->deallocate(static_cast(p), bytes); - } - - bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override - { - return this == &other; - } - - private: - ExternalAllocator* mAlloc; -}; - +using ExternalAllocator = o2::itsmft::tracking::ExternalAllocator; +using ExternalAllocatorAdaptor = o2::itsmft::tracking::ExternalAllocatorAdaptor; } // namespace o2::its #endif diff --git a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx index cb53dae1ed905..dcf8f1a972b13 100644 --- a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx +++ b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx @@ -45,6 +45,9 @@ void TrackerDPL::init(InitContext& ic) mTimer.Reset(); o2::base::GRPGeomHelper::instance().setRequest(mGGCCDBRequest); mChainITS.reset(mRecChain->AddChain()); + if (mRecChain->Init()) { + throw std::runtime_error("GPU reconstruction initialization failed"); + } mITSTrackingInterface.setTraitsFromProvider(mChainITS->GetITSVertexerTraits(), mChainITS->GetITSTrackerTraits(), mChainITS->GetITSTimeframe()); diff --git a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ExternalAllocator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ExternalAllocator.h new file mode 100644 index 0000000000000..9dba9d68f6f0d --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ExternalAllocator.h @@ -0,0 +1,84 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +/// +/// \file ExternalAllocator.h +/// \brief +/// + +#ifndef ALICEO2_ITSMFT_TRACKING_EXTERNALALLOCATOR_H_ +#define ALICEO2_ITSMFT_TRACKING_EXTERNALALLOCATOR_H_ + +#include +#include +#include +#include +#include "Base/GPUMemoryResource.h" + +namespace o2::itsmft::tracking +{ + +class ExternalAllocator +{ + public: + using Type = std::underlying_type_t; + virtual ~ExternalAllocator() = default; + virtual void deallocate(char*, size_t) = 0; + virtual void* allocate(size_t, Type) = 0; + void* allocate(size_t size) { return allocate(size, mType); } + + void* allocateDevice(size_t size, bool stack) + { + return allocate(size, o2::gpu::GPUMemoryResource::MEMORY_GPU | (stack ? o2::gpu::GPUMemoryResource::MEMORY_STACK : 0)); + } + + void* allocateStack(size_t size) { return allocateDevice(size, true); } + virtual void pushTagOnStack(uint64_t) = 0; + virtual void popTagOffStack(uint64_t) = 0; + + void setType(Type t) noexcept { mType = t; } + Type getType() const noexcept { return mType; } + + protected: + Type mType{o2::gpu::GPUMemoryResource::MEMORY_HOST}; +}; + +class ExternalAllocatorAdaptor final : public std::pmr::memory_resource +{ + public: + explicit ExternalAllocatorAdaptor(ExternalAllocator* alloc) : mAlloc(alloc) {} + + protected: + void* do_allocate(size_t bytes, size_t alignment) override + { + void* p = mAlloc->allocate(bytes, o2::gpu::GPUMemoryResource::MemoryType::MEMORY_HOST); + if (!p) { + throw std::bad_alloc(); + } + return p; + } + + void do_deallocate(void* p, size_t bytes, size_t) override + { + mAlloc->deallocate(static_cast(p), bytes); + } + + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override + { + return this == &other; + } + + private: + ExternalAllocator* mAlloc; +}; + +} // namespace o2::itsmft::tracking + +#endif diff --git a/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/GPUExternalAllocator.h b/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/GPUExternalAllocator.h index e873931a5a46c..b16f237ad6619 100644 --- a/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/GPUExternalAllocator.h +++ b/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/GPUExternalAllocator.h @@ -30,7 +30,7 @@ class GPUExternalAllocator final : public o2::its::ExternalAllocator GPUExternalAllocator() = default; ~GPUExternalAllocator(); - void* allocate(size_t size) override; + void* allocate(size_t size, Type type) override; void deallocate(char* ptr, size_t size) override; void pushTagOnStack(uint64_t tag) override; void popTagOffStack(uint64_t tag) override; diff --git a/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/src/GPUExternalAllocator.cu b/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/src/GPUExternalAllocator.cu index c7b5f1cee50f5..4534a60670014 100644 --- a/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/src/GPUExternalAllocator.cu +++ b/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/src/GPUExternalAllocator.cu @@ -37,9 +37,8 @@ GPUExternalAllocator::~GPUExternalAllocator() releaseAll(); } -void* GPUExternalAllocator::allocate(size_t size) +void* GPUExternalAllocator::allocate(size_t size, Type type) { - const auto type = static_cast(getType()); const bool useHost = (type & static_cast(o2::gpu::GPUMemoryResource::MEMORY_HOST)) != 0; const bool useStack = (type & static_cast(o2::gpu::GPUMemoryResource::MEMORY_STACK)) != 0; diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt index cc44b2003a01d..5addfdadcc5ba 100644 --- a/GPU/GPUTracking/CMakeLists.txt +++ b/GPU/GPUTracking/CMakeLists.txt @@ -379,6 +379,7 @@ if(ALIGPU_BUILD_TYPE STREQUAL "O2") O2::TOFBase O2::ITSBase O2::ITStracking + O2::ITSMFTTracking O2::TPCFastTransformation O2::DetectorsRaw O2::Steer diff --git a/GPU/GPUTracking/Global/GPUChainITS.cxx b/GPU/GPUTracking/Global/GPUChainITS.cxx index 598f7a61cac1a..95ed30924e38e 100644 --- a/GPU/GPUTracking/Global/GPUChainITS.cxx +++ b/GPU/GPUTracking/Global/GPUChainITS.cxx @@ -14,36 +14,31 @@ #include "GPUChainITS.h" #include "GPUConstantMem.h" +#include "GPUDefParametersConstants.h" #include "DataFormatsITS/TrackITS.h" -#include "ITStracking/ExternalAllocator.h" +#include "ITSMFTTracking/ExternalAllocator.h" #include "GPUReconstructionIncludesITS.h" using namespace o2::gpu; -namespace o2::its +namespace { -class GPUFrameworkExternalAllocator final : public o2::its::ExternalAllocator +class GPUFrameworkExternalAllocator final : public o2::itsmft::tracking::ExternalAllocator { public: - void* allocate(size_t size) final + explicit GPUFrameworkExternalAllocator(GPUReconstruction* fwr) : mFWReco(fwr) {} + void* allocate(size_t size, Type type) final { - return mFWReco->AllocateDirectMemory(size, mType); + return mFWReco->AllocateDirectMemory(size, type); } - void deallocate(char* ptr, size_t size) final {} // this is a simple no-op - void pushTagOnStack(uint64_t tag) final - { - mFWReco->PushNonPersistentMemory(tag); - } - void popTagOffStack(uint64_t tag) final - { - mFWReco->PopNonPersistentMemory(gpudatatypes::RecoStep::ITSTracking, tag); - } - void setReconstructionFramework(o2::gpu::GPUReconstruction* fwr) { mFWReco = fwr; } + void deallocate(char*, size_t) final {} + void pushTagOnStack(uint64_t tag) final { mFWReco->PushNonPersistentMemory(tag); } + void popTagOffStack(uint64_t tag) final { mFWReco->PopNonPersistentMemory(gpudatatypes::RecoStep::ITSTracking, tag); } private: - o2::gpu::GPUReconstruction* mFWReco; + GPUReconstruction* mFWReco; }; -} // namespace o2::its +} // namespace GPUChainITS::~GPUChainITS() = default; @@ -51,6 +46,12 @@ GPUChainITS::GPUChainITS(GPUReconstruction* rec) : GPUChain(rec) {} int32_t GPUChainITS::Init() { return 0; } +void GPUChainITS::MemorySize(size_t& gpuMem, size_t& pageLockedHostMem) +{ + gpuMem = constants::GPU_DEFAULT_MEMORY_SIZE; + pageLockedHostMem = constants::GPU_DEFAULT_HOST_MEMORY_SIZE; +} + o2::its::TrackerTraits<7>* GPUChainITS::GetITSTrackerTraits() { if (mITSTrackerTraits == nullptr) { @@ -74,14 +75,20 @@ o2::its::TimeFrame<7>* GPUChainITS::GetITSTimeframe() } #if !defined(GPUCA_STANDALONE) if (mITSTimeFrame->isGPU()) { - mFrameworkAllocator.reset(new o2::its::GPUFrameworkExternalAllocator()); - mFrameworkAllocator->setReconstructionFramework(rec()); - mITSTimeFrame->setFrameworkAllocator(mFrameworkAllocator.get()); + mITSTimeFrame->setFrameworkAllocator(GetITSMFTFrameworkAllocator()); } #endif return mITSTimeFrame.get(); } +o2::itsmft::tracking::ExternalAllocator* GPUChainITS::GetITSMFTFrameworkAllocator() +{ + if (mFrameworkAllocator == nullptr && mRec->IsGPU()) { + mFrameworkAllocator = std::make_unique(rec()); + } + return mFrameworkAllocator.get(); +} + int32_t GPUChainITS::PrepareEvent() { return 0; } int32_t GPUChainITS::Finalize() { return 0; } diff --git a/GPU/GPUTracking/Global/GPUChainITS.h b/GPU/GPUTracking/Global/GPUChainITS.h index ee466365a157d..b848cd151104f 100644 --- a/GPU/GPUTracking/Global/GPUChainITS.h +++ b/GPU/GPUTracking/Global/GPUChainITS.h @@ -21,8 +21,11 @@ namespace o2::its struct Cluster; struct TrackingFrameInfo; class TrackITSExt; -class GPUFrameworkExternalAllocator; } // namespace o2::its +namespace o2::itsmft::tracking +{ +class ExternalAllocator; +} // namespace o2::itsmft::tracking namespace o2::gpu { @@ -39,15 +42,16 @@ class GPUChainITS final : public GPUChain void RegisterPermanentMemoryAndProcessors() final {}; void RegisterGPUProcessors() final {}; - void MemorySize(size_t&, size_t&) final {}; + void MemorySize(size_t& gpuMem, size_t& pageLockedHostMem) final; o2::its::TrackerTraits<7>* GetITSTrackerTraits(); o2::its::VertexerTraits<7>* GetITSVertexerTraits(); o2::its::TimeFrame<7>* GetITSTimeframe(); + o2::itsmft::tracking::ExternalAllocator* GetITSMFTFrameworkAllocator(); protected: GPUChainITS(GPUReconstruction* rec); - std::unique_ptr mFrameworkAllocator; + std::unique_ptr mFrameworkAllocator; std::unique_ptr> mITSTimeFrame; std::unique_ptr> mITSTrackerTraits; std::unique_ptr> mITSVertexerTraits; diff --git a/GPU/GPUTracking/Interface/GPUO2Interface.cxx b/GPU/GPUTracking/Interface/GPUO2Interface.cxx index ca22df5b95695..d6bce0448d0ad 100644 --- a/GPU/GPUTracking/Interface/GPUO2Interface.cxx +++ b/GPU/GPUTracking/Interface/GPUO2Interface.cxx @@ -261,6 +261,11 @@ void GPUO2Interface::setErrorCodeOutput(std::vector>* v) } } +o2::itsmft::tracking::ExternalAllocator* GPUO2Interface::GetITSMFTFrameworkAllocator() +{ + return mChainITS->GetITSMFTFrameworkAllocator(); +} + void GPUO2Interface::GetITSTraits(o2::its::TrackerTraits<7>*& trackerTraits, o2::its::VertexerTraits<7>*& vertexerTraits, o2::its::TimeFrame<7>*& timeFrame) { trackerTraits = mChainITS->GetITSTrackerTraits(); diff --git a/GPU/GPUTracking/Interface/GPUO2Interface.h b/GPU/GPUTracking/Interface/GPUO2Interface.h index eed7c119f5328..11d801ddcb477 100644 --- a/GPU/GPUTracking/Interface/GPUO2Interface.h +++ b/GPU/GPUTracking/Interface/GPUO2Interface.h @@ -45,6 +45,10 @@ class VertexerTraits; template class TimeFrame; } // namespace o2::its +namespace o2::itsmft::tracking +{ +class ExternalAllocator; +} // namespace o2::itsmft::tracking namespace o2::gpu { @@ -78,6 +82,7 @@ class GPUO2Interface void DumpEvent(int32_t nEvent, GPUTrackingInOutPointers* data, uint32_t iThread, const char* dir = ""); void DumpSettings(uint32_t iThread, const char* dir = ""); + o2::itsmft::tracking::ExternalAllocator* GetITSMFTFrameworkAllocator(); void GetITSTraits(o2::its::TrackerTraits<7>*& trackerTraits, o2::its::VertexerTraits<7>*& vertexerTraits, o2::its::TimeFrame<7>*& timeFrame); const o2::base::Propagator* GetDeviceO2Propagator(int32_t iThread = 0) const; void UseGPUPolynomialFieldInPropagator(o2::base::Propagator* prop) const; diff --git a/GPU/GPUTracking/Standalone/CMakeLists.txt b/GPU/GPUTracking/Standalone/CMakeLists.txt index a4c5817d1f7f9..8fd213fe137aa 100644 --- a/GPU/GPUTracking/Standalone/CMakeLists.txt +++ b/GPU/GPUTracking/Standalone/CMakeLists.txt @@ -188,6 +188,7 @@ target_include_directories(standalone_support PUBLIC ${O2_DIR}/Detectors/Base/include ${O2_DIR}/Detectors/Base/src ${O2_DIR}/Detectors/ITSMFT/common/base/include + ${O2_DIR}/Detectors/ITSMFT/common/tracking/include ${O2_DIR}/Detectors/ITSMFT/ITS/base/include ${O2_DIR}/Detectors/ITSMFT/ITS/tracking/include ${O2_DIR}/Detectors/ITSMFT/ITS/tracking/GPU