diff --git a/Detectors/CMakeLists.txt b/Detectors/CMakeLists.txt index 289fb3fcd0fa7..9094bca1a2d2e 100644 --- a/Detectors/CMakeLists.txt +++ b/Detectors/CMakeLists.txt @@ -25,6 +25,7 @@ add_subdirectory(HMPID) add_subdirectory(TOF) add_subdirectory(ZDC) +add_subdirectory(GenericTracking) add_subdirectory(ITSMFT) add_subdirectory(External) # sensitive external (CAD-derived) detectors; uses ITSMFT hit type add_subdirectory(TRD) diff --git a/Detectors/GenericTracking/CMakeLists.txt b/Detectors/GenericTracking/CMakeLists.txt new file mode 100644 index 0000000000000..2c865e20d88a5 --- /dev/null +++ b/Detectors/GenericTracking/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. +o2_add_library(GenericTracking + SOURCES src/BoundedAllocator.cxx + PRIVATE_LINK_LIBRARIES O2::GPUCommon O2::FrameworkLogger) + +add_subdirectory(test) diff --git a/Detectors/GenericTracking/include/GenericTracking/BoundedAllocator.h b/Detectors/GenericTracking/include/GenericTracking/BoundedAllocator.h new file mode 100644 index 0000000000000..d523a10bb2c63 --- /dev/null +++ b/Detectors/GenericTracking/include/GenericTracking/BoundedAllocator.h @@ -0,0 +1,199 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +/// +/// \file BoundedAllocator.h +/// \brief Shared budgeted host allocation and bounded-vector utilities. +/// +/// Reconciles GTRK's overflow-safe allocator, profiling counters and vector +/// rebuilds with the owned-resource, mutable-budget and stage-peak contracts +/// used by the upstream legacy ITS tracker and vertexer. +/// + +#ifndef ALICEO2_GENERICTRACKING_BOUNDEDALLOCATOR_H_ +#define ALICEO2_GENERICTRACKING_BOUNDEDALLOCATOR_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace o2::gtrk +{ + +// #define BOUNDED_MR_STATS +class BoundedMemoryResource final : public std::pmr::memory_resource +{ + public: + class MemoryLimitExceeded final : public std::bad_alloc + { + public: + MemoryLimitExceeded(size_t attempted, size_t used, size_t max); + const char* what() const noexcept final; + + private: + std::string mMsg; + }; + + static std::pmr::memory_resource* cachingUpstream(); + + BoundedMemoryResource(size_t maxBytes = std::numeric_limits::max(), + std::pmr::memory_resource* upstream = nullptr); + + BoundedMemoryResource(std::unique_ptr upstream, + size_t maxBytes = std::numeric_limits::max()); + + [[nodiscard]] size_t getUsedMemory() const noexcept; + [[nodiscard]] size_t getMaxMemory() const noexcept; + [[nodiscard]] size_t getThrowCount() const noexcept; + [[nodiscard]] size_t getPeakMemory() const noexcept; + [[nodiscard]] size_t getPeakMemoryDelta() const noexcept; + + // Reset the stage peak/baseline, returning the previous peak for nested profiling. + size_t resetPeakMemory() noexcept; + void raisePeakMemory(size_t peak) noexcept; + [[nodiscard]] size_t getAllocationCount() const noexcept { return mAllocations.load(std::memory_order_relaxed); } + [[nodiscard]] size_t getAllocatedBytes() const noexcept { return mAllocatedBytes.load(std::memory_order_relaxed); } + void setMaxMemory(size_t max); + +#if !defined(__HIPCC__) && !defined(__CUDACC__) + std::string asString() const; + void print() const; +#endif + + private: + void* do_allocate(size_t bytes, size_t alignment) final; + void do_deallocate(void* p, size_t bytes, size_t alignment) final; + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final; + + std::atomic mMaxMemory{std::numeric_limits::max()}; + std::atomic mCountThrow{0}; + std::atomic mUsedMemory{0}; + std::atomic mAllocations{0}; + std::atomic mAllocatedBytes{0}; + std::atomic mPeakUsedMemory{0}; + std::atomic mPeakBaselineMemory{0}; + std::unique_ptr mOwnedUpstream; + std::pmr::memory_resource* mUpstream{nullptr}; + +#ifdef BOUNDED_MR_STATS + struct Stats { + std::atomic peak{0}; + std::atomic live{0}; + std::atomic nAlloc{0}; + std::atomic nFree{0}; + std::atomic totalAlloc{0}; + std::atomic totalFreed{0}; + std::atomic maxAlign{0}; + std::atomic upstreamFailures{0}; + }; + Stats mStats{}; +#endif +}; + +template +using BoundedVector = std::pmr::vector; + +template +using bounded_vector = BoundedVector; + +template +inline void deepVectorClear(std::vector& vec) +{ + std::vector().swap(vec); +} + +template +inline void deepVectorClear(bounded_vector& vec, std::pmr::memory_resource* mr = nullptr) +{ + std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource(); + std::destroy_at(&vec); + new (&vec) bounded_vector(std::pmr::polymorphic_allocator{tmr}); +} + +template +inline void deepVectorClear(std::vector>& vec, std::pmr::memory_resource* mr = nullptr) +{ + for (auto& v : vec) { + deepVectorClear(v, mr); + } +} + +template +inline void deepVectorClear(std::array, S>& arr, std::pmr::memory_resource* mr = nullptr) +{ + for (size_t i{0}; i < S; ++i) { + deepVectorClear(arr[i], mr); + } +} + +template +inline void clearResizeBoundedVector(bounded_vector& vec, size_t sz, std::pmr::memory_resource* mr = nullptr, T def = T()) +{ + // Rebuild a valid empty vector before allocation or element construction can + // throw. The old allocation is released first, as required by bounded pools. + deepVectorClear(vec, mr); + vec.assign(sz, def); +} + +// GTRK's capacity-reusing resize: retained storage is intended for arrays +// rewritten by the caller. A non-null different resource requires a rebuild. +template +inline void resizeBoundedVector(BoundedVector& vec, size_t sz, std::pmr::memory_resource* mr = nullptr) +{ + if (sz > vec.capacity() || (mr != nullptr && mr != vec.get_allocator().resource())) { + clearResizeBoundedVector(vec, sz, mr); + return; + } + vec.resize(sz); +} + +// Release several bounded vectors onto one resource. +template +inline void deepVectorClear(std::pmr::memory_resource* mr, BoundedVector&... vecs) +{ + (deepVectorClear(vecs, mr), ...); +} + +template +inline void clearResizeBoundedVector(std::vector>& vec, size_t size, std::pmr::memory_resource* mr) +{ + vec.clear(); + vec.reserve(size); + for (size_t i = 0; i < size; ++i) { + vec.emplace_back(std::pmr::polymorphic_allocator>{mr}); + } +} + +template +inline void clearResizeBoundedArray(std::array, S>& arr, size_t size, std::pmr::memory_resource* mr = nullptr, T def = T()) +{ + for (size_t i{0}; i < S; ++i) { + clearResizeBoundedVector(arr[i], size, mr, def); + } +} + +template +inline std::vector toSTDVector(const bounded_vector& b) +{ + std::vector t(b.size()); + std::copy(b.cbegin(), b.cend(), t.begin()); + return t; +} + +} // namespace o2::gtrk + +#endif /* ALICEO2_GENERICTRACKING_BOUNDEDALLOCATOR_H_ */ diff --git a/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx b/Detectors/GenericTracking/src/BoundedAllocator.cxx similarity index 80% rename from Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx rename to Detectors/GenericTracking/src/BoundedAllocator.cxx index 152d3c8d7dd51..111e6fe5ce6e0 100644 --- a/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx +++ b/Detectors/GenericTracking/src/BoundedAllocator.cxx @@ -9,16 +9,19 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include #include #include "GPUCommonLogger.h" -#include "ITSMFTTracking/Constants.h" -namespace o2::itsmft::tracking +namespace o2::gtrk { +namespace +{ +constexpr double kGigabyte = 1024. * 1024. * 1024.; +} BoundedMemoryResource::MemoryLimitExceeded::MemoryLimitExceeded(size_t attempted, size_t used, size_t max) { @@ -56,13 +59,19 @@ void* BoundedMemoryResource::do_allocate(size_t bytes, size_t alignment) { size_t newUsed{0}; size_t currentUsed{mUsedMemory.load(std::memory_order_relaxed)}; - do { - newUsed = currentUsed + bytes; - if (newUsed > mMaxMemory.load(std::memory_order_relaxed)) { + for (;;) { + const size_t limit = mMaxMemory.load(std::memory_order_relaxed); + // Test before adding, so even SIZE_MAX requests cannot wrap under budget. + if (currentUsed > limit || bytes > limit - currentUsed) { + const size_t attempted = bytes > std::numeric_limits::max() - currentUsed ? std::numeric_limits::max() : currentUsed + bytes; mCountThrow.fetch_add(1, std::memory_order_relaxed); - throw MemoryLimitExceeded(newUsed, currentUsed, mMaxMemory.load(std::memory_order_relaxed)); + throw MemoryLimitExceeded(attempted, currentUsed, limit); + } + newUsed = currentUsed + bytes; + if (mUsedMemory.compare_exchange_weak(currentUsed, newUsed, std::memory_order_acq_rel, std::memory_order_relaxed)) { + break; } - } while (!mUsedMemory.compare_exchange_weak(currentUsed, newUsed, std::memory_order_acq_rel, std::memory_order_relaxed)); + } void* p{nullptr}; try { @@ -75,6 +84,9 @@ void* BoundedMemoryResource::do_allocate(size_t bytes, size_t alignment) throw; } + mAllocations.fetch_add(1, std::memory_order_relaxed); + mAllocatedBytes.fetch_add(bytes, std::memory_order_relaxed); + size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); while (newUsed > peak && !mPeakUsedMemory.compare_exchange_weak(peak, newUsed, std::memory_order_relaxed)) { } @@ -137,11 +149,18 @@ size_t BoundedMemoryResource::getPeakMemoryDelta() const noexcept return peak > baseline ? peak - baseline : 0; } -void BoundedMemoryResource::resetPeakMemory() noexcept +size_t BoundedMemoryResource::resetPeakMemory() noexcept { const size_t used = mUsedMemory.load(std::memory_order_acquire); mPeakBaselineMemory.store(used, std::memory_order_release); - mPeakUsedMemory.store(used, std::memory_order_release); + return mPeakUsedMemory.exchange(used, std::memory_order_acq_rel); +} + +void BoundedMemoryResource::raisePeakMemory(size_t peak) noexcept +{ + size_t current = mPeakUsedMemory.load(std::memory_order_relaxed); + while (peak > current && !mPeakUsedMemory.compare_exchange_weak(current, peak, std::memory_order_relaxed)) { + } } void BoundedMemoryResource::setMaxMemory(size_t max) @@ -174,18 +193,18 @@ std::string BoundedMemoryResource::asString() const const auto maxMemory = mMaxMemory.load(std::memory_order_relaxed); std::string result; if (maxMemory == std::numeric_limits::max()) { - result += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, used / o2::its::constants::GB, peak / o2::its::constants::GB, peakDelta / o2::its::constants::GB); + result += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, used / kGigabyte, peak / kGigabyte, peakDelta / kGigabyte); } else { - result += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, static_cast(maxMemory) / o2::its::constants::GB, used / o2::its::constants::GB, 100.0 * used / static_cast(maxMemory), peak / o2::its::constants::GB, peakDelta / o2::its::constants::GB); + result += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, static_cast(maxMemory) / kGigabyte, used / kGigabyte, 100.0 * used / static_cast(maxMemory), peak / kGigabyte, peakDelta / kGigabyte); } #ifdef BOUNDED_MR_STATS result += std::format(" peak={:.2f} GB live={} nAlloc={} nFree={} totalAlloc={:.2f} GB totalFreed={:.2f} GB maxAlign={} upstreamFail={}", - static_cast(mStats.peak.load(std::memory_order_relaxed)) / o2::its::constants::GB, + static_cast(mStats.peak.load(std::memory_order_relaxed)) / kGigabyte, mStats.live.load(std::memory_order_relaxed), mStats.nAlloc.load(std::memory_order_relaxed), mStats.nFree.load(std::memory_order_relaxed), - static_cast(mStats.totalAlloc.load(std::memory_order_relaxed)) / o2::its::constants::GB, - static_cast(mStats.totalFreed.load(std::memory_order_relaxed)) / o2::its::constants::GB, + static_cast(mStats.totalAlloc.load(std::memory_order_relaxed)) / kGigabyte, + static_cast(mStats.totalFreed.load(std::memory_order_relaxed)) / kGigabyte, mStats.maxAlign.load(std::memory_order_relaxed), mStats.upstreamFailures.load(std::memory_order_relaxed)); #endif @@ -197,4 +216,4 @@ void BoundedMemoryResource::print() const LOGP(info, "{}", asString()); } -} // namespace o2::itsmft::tracking +} // namespace o2::gtrk diff --git a/Detectors/GenericTracking/test/CMakeLists.txt b/Detectors/GenericTracking/test/CMakeLists.txt new file mode 100644 index 0000000000000..988c8811540da --- /dev/null +++ b/Detectors/GenericTracking/test/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. +o2_add_test(bounded-allocator + SOURCES testBoundedAllocator.cxx + COMPONENT_NAME generic-tracking + LABELS "generic-tracking" + PUBLIC_LINK_LIBRARIES O2::GenericTracking) diff --git a/Detectors/GenericTracking/test/testBoundedAllocator.cxx b/Detectors/GenericTracking/test/testBoundedAllocator.cxx new file mode 100644 index 0000000000000..d68134c488bc3 --- /dev/null +++ b/Detectors/GenericTracking/test/testBoundedAllocator.cxx @@ -0,0 +1,156 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE GenericTrackingBoundedAllocator +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK +#include + +#include "GenericTracking/BoundedAllocator.h" + +#include + +using namespace o2::gtrk; + +namespace +{ +class ThrowingResource final : public std::pmr::memory_resource +{ + void* do_allocate(size_t, size_t) final { throw std::bad_alloc{}; } + void do_deallocate(void*, size_t, size_t) final {} + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final { return this == &other; } +}; + +struct ThrowingCopy { + static inline int copiesBeforeFailure = -1; + ThrowingCopy() = default; + ThrowingCopy(const ThrowingCopy&) + { + if (copiesBeforeFailure == 0) { + throw std::runtime_error{"copy"}; + } + if (copiesBeforeFailure > 0) { + --copiesBeforeFailure; + } + } + ThrowingCopy& operator=(const ThrowingCopy&) = default; +}; +} // namespace + +BOOST_AUTO_TEST_CASE(huge_request_cannot_wrap_past_budget) +{ + BoundedMemoryResource pool; + void* held = pool.allocate(8); + BOOST_CHECK_THROW(static_cast(pool.allocate(std::numeric_limits::max())), BoundedMemoryResource::MemoryLimitExceeded); + BOOST_CHECK_EQUAL(pool.getUsedMemory(), 8u); + BOOST_CHECK_EQUAL(pool.getThrowCount(), 1u); + pool.deallocate(held, 8); + BOOST_CHECK_EQUAL(pool.getUsedMemory(), 0u); +} + +BOOST_AUTO_TEST_CASE(upstream_failure_rolls_back_reserved_bytes) +{ + ThrowingResource upstream; + BoundedMemoryResource pool{1024, &upstream}; + BOOST_CHECK_THROW(static_cast(pool.allocate(16)), std::bad_alloc); + BOOST_CHECK_EQUAL(pool.getUsedMemory(), 0u); + BOOST_CHECK_EQUAL(pool.getAllocationCount(), 0u); + BOOST_CHECK_EQUAL(pool.getAllocatedBytes(), 0u); +} + +BOOST_AUTO_TEST_CASE(legacy_mutable_budget_and_stage_peak_are_preserved) +{ + BoundedMemoryResource pool{64}; + void* first = pool.allocate(16); + BOOST_CHECK_THROW(pool.setMaxMemory(8), BoundedMemoryResource::MemoryLimitExceeded); + BOOST_CHECK_EQUAL(pool.getMaxMemory(), 64u); + const size_t previousPeak = pool.resetPeakMemory(); + BOOST_CHECK_EQUAL(previousPeak, 16u); + BOOST_CHECK_EQUAL(pool.getPeakMemoryDelta(), 0u); + pool.setMaxMemory(24); + void* second = pool.allocate(8); + BOOST_CHECK_EQUAL(pool.getPeakMemoryDelta(), 8u); + BOOST_CHECK_THROW(static_cast(pool.allocate(1)), BoundedMemoryResource::MemoryLimitExceeded); + pool.deallocate(second, 8); + pool.deallocate(first, 16); + BOOST_CHECK_EQUAL(pool.getAllocationCount(), 2u); + BOOST_CHECK_EQUAL(pool.getAllocatedBytes(), 24u); + BOOST_CHECK_EQUAL(pool.resetPeakMemory(), 24u); + pool.raisePeakMemory(24); + BOOST_CHECK_EQUAL(pool.getPeakMemory(), 24u); +} + +BOOST_AUTO_TEST_CASE(failed_rebuild_leaves_valid_vector_on_requested_resource) +{ + BoundedMemoryResource pool{64}; + BoundedVector values{&pool}; + values.assign(8, 7); + BOOST_CHECK_THROW(clearResizeBoundedVector(values, 32, &pool, 3), BoundedMemoryResource::MemoryLimitExceeded); + BOOST_CHECK(values.empty()); + BOOST_CHECK_EQUAL(pool.getUsedMemory(), 0u); + BOOST_CHECK(values.get_allocator().resource() == &pool); + values.push_back(9); + BOOST_CHECK_EQUAL(values.front(), 9); +} + +BOOST_AUTO_TEST_CASE(element_constructor_failure_leaves_vector_destructible) +{ + BoundedMemoryResource pool; + BoundedVector values{&pool}; + values.resize(2); + ThrowingCopy sample; + ThrowingCopy::copiesBeforeFailure = 1; + // The by-value default argument succeeds; assign throws after rebuilding. + BOOST_CHECK_THROW(clearResizeBoundedVector(values, 4, &pool, sample), std::runtime_error); + ThrowingCopy::copiesBeforeFailure = -1; + BOOST_CHECK(values.empty()); + deepVectorClear(values); + BOOST_CHECK_EQUAL(pool.getUsedMemory(), 0u); + values.resize(1); +} + +BOOST_AUTO_TEST_CASE(resize_reuses_capacity_and_honors_resource_changes) +{ + BoundedMemoryResource first, second; + BoundedVector values{&first}; + values.reserve(16); + const auto allocations = first.getAllocationCount(); + auto* storage = values.data(); + resizeBoundedVector(values, 8); + BOOST_CHECK(values.data() == storage); + BOOST_CHECK_EQUAL(first.getAllocationCount(), allocations); + resizeBoundedVector(values, 4, &second); + BOOST_CHECK(values.get_allocator().resource() == &second); + BOOST_CHECK_EQUAL(first.getUsedMemory(), 0u); +} + +BOOST_AUTO_TEST_CASE(nested_vectors_and_arrays_retain_legacy_clear_semantics) +{ + BoundedMemoryResource first, second; + std::vector> nested; + clearResizeBoundedVector(nested, 3, &first); + for (auto& values : nested) { + values.assign(4, 5); + } + deepVectorClear(nested, &second); + BOOST_CHECK_EQUAL(nested.size(), 3u); + BOOST_CHECK_EQUAL(first.getUsedMemory(), 0u); + for (const auto& values : nested) { + BOOST_CHECK(values.empty()); + BOOST_CHECK(values.get_allocator().resource() == &second); + } + std::array, 2> array; + clearResizeBoundedArray(array, 4, &second, 7); + BOOST_CHECK_EQUAL(array[0][0], 7); + deepVectorClear(array, &first); + BOOST_CHECK_EQUAL(second.getUsedMemory(), 0u); + BOOST_CHECK(array[0].get_allocator().resource() == &first); +} diff --git a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt index 6560f25966a63..333f44878af18 100644 --- a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt @@ -25,6 +25,7 @@ o2_add_library(ITStracking src/VertexerTraits.cxx PUBLIC_LINK_LIBRARIES O2::GPUCommon + O2::GenericTracking Microsoft.GSL::GSL O2::CommonConstants O2::DataFormatsITSMFT diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h index fe10b6c31ecdc..f067098cec991 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h @@ -16,7 +16,7 @@ #include #include -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/Configuration.h" #include "ITStracking/TrackExtensionHypothesis.h" diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h index 5c25bdc833890..f2c98509fd580 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h @@ -16,7 +16,7 @@ #include #include -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "ITSMFTTracking/CapacityEstimator.h" #include "ITSMFTTracking/ROFLookupTables.h" #include "ITStracking/TrackingTopology.h" @@ -28,7 +28,7 @@ namespace o2::its { -using o2::itsmft::tracking::bounded_vector; +using o2::gtrk::bounded_vector; using o2::itsmft::tracking::CapacityEstimator; class CellSeed; diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu index aa2578a9bb6ad..3f74f71ed3adf 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu @@ -22,7 +22,7 @@ #include "ITStrackingGPU/TimeFrameGPU.h" #include "ITSMFTTracking/Constants.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "ITStrackingGPU/Utils.h" #include "ITStracking/ClusterLines.h" #include "ITStracking/LineProjection.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h index 8c1beb9ca5c8b..3568ffeb3e2f3 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h @@ -15,14 +15,14 @@ #include #include -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "ITStracking/ClusterLines.h" namespace o2::its::line_vertexer { -using o2::itsmft::tracking::bounded_vector; -using o2::itsmft::tracking::BoundedMemoryResource; +using o2::gtrk::bounded_vector; +using o2::gtrk::BoundedMemoryResource; struct Settings { float beamX = 0.f; diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h index 64facfc87807d..b3bd3f226b0ab 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h @@ -32,7 +32,7 @@ #include "ITStracking/Tracklet.h" #include "ITStracking/IndexTableUtils.h" #include "ITStracking/ExternalAllocator.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "ITSMFTTracking/ROFLookupTables.h" #include "ITStracking/TrackingTopology.h" #include "SimulationDataFormat/MCCompLabel.h" @@ -58,8 +58,8 @@ class ROFRecord; namespace its { -using o2::itsmft::tracking::bounded_vector; -using o2::itsmft::tracking::BoundedMemoryResource; +using o2::gtrk::bounded_vector; +using o2::gtrk::BoundedMemoryResource; using o2::itsmft::tracking::CapacityEstimator; namespace gpu diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h index fc93c53165bcb..ce3dc6845ce93 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h @@ -33,7 +33,7 @@ #include "ITStracking/Definitions.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/TrackerTraits.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" namespace o2 { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h index a50dc2c6ee132..7ae0fcc09e3f0 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h @@ -26,7 +26,7 @@ #include "ITSMFTTracking/CapacityEstimator.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/Cell.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "ITStracking/TrackExtensionHypothesis.h" #include "ITStracking/TrackFollower.h" #include "ITStracking/TrackITSInternal.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h index 5f5c1477acb88..920e815299541 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h @@ -19,7 +19,7 @@ #include "ITStracking/TrackerTraits.h" #include "ITStracking/Vertexer.h" #include "ITStracking/VertexerTraits.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "DataFormatsParameters/GRPObject.h" #include "DataFormatsITSMFT/TopologyDictionary.h" #include "DataFormatsCalibration/MeanVertexObject.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h index 76647d923e2aa..d27ea5694b75e 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h @@ -30,7 +30,7 @@ #include "ITStracking/Configuration.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/VertexerTraits.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" namespace o2::its { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h index 7be8e816e4944..13034b388eb6a 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h @@ -21,7 +21,7 @@ #include #include -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "ITStracking/Cluster.h" #include "ITStracking/ClusterLines.h" #include "ITStracking/Configuration.h" diff --git a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx index f3e68cf766f6c..89343ff55a4b6 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx @@ -26,7 +26,7 @@ #include "DataFormatsITSMFT/TopologyDictionary.h" #include "ITSBase/GeometryTGeo.h" #include "ITSMFTBase/SegmentationAlpide.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" namespace { @@ -41,8 +41,8 @@ struct ClusterHelper { namespace o2::its { -using o2::itsmft::tracking::clearResizeBoundedVector; -using o2::itsmft::tracking::deepVectorClear; +using o2::gtrk::clearResizeBoundedVector; +using o2::gtrk::deepVectorClear; constexpr float DefClusErrorRow = o2::itsmft::SegmentationAlpide::PitchRow * 0.5; constexpr float DefClusErrorCol = o2::itsmft::SegmentationAlpide::PitchCol * 0.5; diff --git a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx index f8834e655dd99..ebd9162a00ad2 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx @@ -14,7 +14,7 @@ /// #include "ITStracking/Tracker.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "ITSMFTTracking/Constants.h" #include "ITStracking/TrackerTraits.h" #include "ITSMFTTracking/ITSTrackingConfigParam.h" diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx index e751f6152b033..b29da0e7d22d3 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx @@ -35,7 +35,7 @@ #include "DetectorsBase/Propagator.h" #include "GPUCommonMath.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "ITSMFTTracking/MathUtils.h" #include "ITStracking/Cell.h" #include "ITSMFTTracking/Constants.h" @@ -54,7 +54,7 @@ namespace o2::its { -using o2::itsmft::tracking::deepVectorClear; +using o2::gtrk::deepVectorClear; using o2::itsmft::tracking::GroupedSlabSink; using o2::itsmft::tracking::SlabSinkStats; using o2::itsmft::tracking::SlabSite; diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx index 1dbab0e9f6170..b71c4219f6084 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx @@ -38,7 +38,7 @@ #include "DataFormatsTRD/TriggerRecord.h" #include "CommonDataFormat/IRFrame.h" #include "DetectorsBase/GRPGeomHelper.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "Framework/InputRecordWalker.h" #include "Framework/DataRefUtils.h" #include "Framework/DeviceSpec.h" diff --git a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx index 356229bc201c5..18b4791f474d9 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx @@ -14,7 +14,7 @@ /// #include "ITStracking/Vertexer.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "ITStracking/VertexerTraits.h" #include "ITSMFTTracking/ITSTrackingConfigParam.h" diff --git a/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx index 4c5ff68eddb0c..e40088525d7d3 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx @@ -21,7 +21,7 @@ #include #include "ITStracking/VertexerTraits.h" -#include "ITSMFTTracking/BoundedAllocator.h" +#include "GenericTracking/BoundedAllocator.h" #include "ITStracking/ClusterLines.h" #include "ITStracking/Definitions.h" #include "ITStracking/LineVertexerHelpers.h" @@ -35,7 +35,7 @@ namespace o2::its { -using o2::itsmft::tracking::deepVectorClear; +using o2::gtrk::deepVectorClear; namespace { diff --git a/Detectors/ITSMFT/common/tracking/CMakeLists.txt b/Detectors/ITSMFT/common/tracking/CMakeLists.txt index 45deb3d5b4c4c..9df4122cec929 100644 --- a/Detectors/ITSMFT/common/tracking/CMakeLists.txt +++ b/Detectors/ITSMFT/common/tracking/CMakeLists.txt @@ -20,8 +20,7 @@ o2_target_root_dictionary(ITSMFTTrackingParams o2_add_library(ITSMFTTracking TARGETVARNAME targetName - SOURCES src/BoundedAllocator.cxx - src/CapacityEstimator.cxx + SOURCES src/CapacityEstimator.cxx src/ITSTrackingConfigParam.cxx src/SlabBumpAllocator.cxx src/IOUtils.cxx @@ -38,6 +37,7 @@ o2_add_library(ITSMFTTracking src/Tracker.cxx PUBLIC_LINK_LIBRARIES O2::ITSMFTTrackingParams + O2::GenericTracking O2::GPUCommon O2::CommonConstants O2::CommonDataFormat diff --git a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h index 4f5c634a6f50b..5a032ea7156ad 100644 --- a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h @@ -8,159 +8,23 @@ // In applying this license CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -/// -/// \file BoundedAllocator.h -/// \brief -/// - +// Compatibility imports for existing ITSMFT consumers. Implementation is owned +// by GenericTracking; new detector-independent code should include it directly. #ifndef ALICEO2_ITSMFT_TRACKING_BOUNDEDALLOCATOR_H_ #define ALICEO2_ITSMFT_TRACKING_BOUNDEDALLOCATOR_H_ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "GenericTracking/BoundedAllocator.h" namespace o2::itsmft::tracking { - -// #define BOUNDED_MR_STATS -class BoundedMemoryResource final : public std::pmr::memory_resource -{ - public: - class MemoryLimitExceeded final : public std::bad_alloc - { - public: - MemoryLimitExceeded(size_t attempted, size_t used, size_t max); - const char* what() const noexcept final; - - private: - std::string mMsg; - }; - - static std::pmr::memory_resource* cachingUpstream(); - - BoundedMemoryResource(size_t maxBytes = std::numeric_limits::max(), - std::pmr::memory_resource* upstream = nullptr); - - BoundedMemoryResource(std::unique_ptr upstream, - size_t maxBytes = std::numeric_limits::max()); - - [[nodiscard]] size_t getUsedMemory() const noexcept; - [[nodiscard]] size_t getMaxMemory() const noexcept; - [[nodiscard]] size_t getThrowCount() const noexcept; - [[nodiscard]] size_t getPeakMemory() const noexcept; - [[nodiscard]] size_t getPeakMemoryDelta() const noexcept; - - void resetPeakMemory() noexcept; - void setMaxMemory(size_t max); - -#if !defined(__HIPCC__) && !defined(__CUDACC__) - std::string asString() const; - void print() const; -#endif - - private: - void* do_allocate(size_t bytes, size_t alignment) final; - void do_deallocate(void* p, size_t bytes, size_t alignment) final; - bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final; - - std::atomic mMaxMemory{std::numeric_limits::max()}; - std::atomic mCountThrow{0}; - std::atomic mUsedMemory{0}; - std::atomic mPeakUsedMemory{0}; - std::atomic mPeakBaselineMemory{0}; - std::unique_ptr mOwnedUpstream; - std::pmr::memory_resource* mUpstream{nullptr}; - -#ifdef BOUNDED_MR_STATS - struct Stats { - std::atomic peak{0}; - std::atomic live{0}; - std::atomic nAlloc{0}; - std::atomic nFree{0}; - std::atomic totalAlloc{0}; - std::atomic totalFreed{0}; - std::atomic maxAlign{0}; - std::atomic upstreamFailures{0}; - }; - Stats mStats{}; -#endif -}; - -template -using bounded_vector = std::pmr::vector; - -template -inline void deepVectorClear(std::vector& vec) -{ - std::vector().swap(vec); -} - -template -inline void deepVectorClear(bounded_vector& vec, std::pmr::memory_resource* mr = nullptr) -{ - std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource(); - vec.~bounded_vector(); - new (&vec) bounded_vector(std::pmr::polymorphic_allocator{tmr}); -} - -template -inline void deepVectorClear(std::vector>& vec, std::pmr::memory_resource* mr = nullptr) -{ - for (auto& v : vec) { - deepVectorClear(v, mr); - } -} - -template -inline void deepVectorClear(std::array, S>& arr, std::pmr::memory_resource* mr = nullptr) -{ - for (size_t i{0}; i < S; ++i) { - deepVectorClear(arr[i], mr); - } -} - -template -inline void clearResizeBoundedVector(bounded_vector& vec, size_t sz, std::pmr::memory_resource* mr = nullptr, T def = T()) -{ - std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource(); - vec.~bounded_vector(); - new (&vec) bounded_vector(sz, def, std::pmr::polymorphic_allocator{tmr}); -} - -template -inline void clearResizeBoundedVector(std::vector>& vec, size_t size, std::pmr::memory_resource* mr) -{ - vec.clear(); - vec.reserve(size); - for (size_t i = 0; i < size; ++i) { - vec.emplace_back(std::pmr::polymorphic_allocator>{mr}); - } -} - -template -inline void clearResizeBoundedArray(std::array, S>& arr, size_t size, std::pmr::memory_resource* mr = nullptr, T def = T()) -{ - for (size_t i{0}; i < S; ++i) { - clearResizeBoundedVector(arr[i], size, mr, def); - } -} - -template -inline std::vector toSTDVector(const bounded_vector& b) -{ - std::vector t(b.size()); - std::copy(b.cbegin(), b.cend(), t.begin()); - return t; -} - +using o2::gtrk::bounded_vector; +using o2::gtrk::BoundedMemoryResource; +using o2::gtrk::BoundedVector; +using o2::gtrk::clearResizeBoundedArray; +using o2::gtrk::clearResizeBoundedVector; +using o2::gtrk::deepVectorClear; +using o2::gtrk::resizeBoundedVector; +using o2::gtrk::toSTDVector; } // namespace o2::itsmft::tracking -#endif /* ALICEO2_ITSMFT_TRACKING_BOUNDEDALLOCATOR_H_ */ +#endif